From f138e3f750bbd961fe428e494642e48464c99a3a Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 19:36:55 +0200 Subject: [PATCH 01/30] testing: port fixtures.py to _pytest.ensemble 158 of 205 pytester-based tests now build their sources as real python objects and assert on typed records. The largest single unlock was that a rootdir conftest is exactly reproducible as a plugin object in extra_plugins - fixture visibility, override chains, getfixturevalue super-lookup and autouse all behave identically - which carried ~30 tests on its own. 47 tests stay on pytester, each with an `# ensemble:` reason: conftest directory scoping (10), --fixtures/--markers rendering which is served from pytest_cmdline_main (13), assertions on a fixture's file:line, which is host-anchored (8), package layout and scope="package" (7), errors raised at module import (4), real plugin modules and example trees (2), subprocess (2). Two outcome assertions were wrong in opposite directions and are now corrected with a note at each site: a failing setup is `errors`, not `failed`, and one test asserting only `ret == 0` was really `passed=1, xfailed=2`. --- testing/python/fixtures.py | 6343 +++++++++++++++++------------------- 1 file changed, 3075 insertions(+), 3268 deletions(-) diff --git a/testing/python/fixtures.py b/testing/python/fixtures.py index f68fe3be39f..facf41d0c2e 100644 --- a/testing/python/fixtures.py +++ b/testing/python/fixtures.py @@ -9,10 +9,16 @@ from _pytest.compat import getfuncargnames from _pytest.config import ExitCode +from _pytest.ensemble import build_module +from _pytest.ensemble import collect_tests +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import Ensemble from _pytest.ensemble import run_tests from _pytest.fixtures import deduplicate_names from _pytest.fixtures import ParamValueKey from _pytest.fixtures import TopRequest +from _pytest.mark.structures import Mark +from _pytest.mark.structures import MarkDecorator from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import get_public_names from _pytest.pytester import Pytester @@ -20,6 +26,16 @@ import pytest +def unregistered_mark(name: str, *args: object, **kwargs: object) -> MarkDecorator: + """Build a mark decorator without consulting the host configuration. + + ``pytest.mark.`` resolves against the *host* config at decoration + time, and this suite runs with strict markers, so a deliberately + unregistered mark applied to an ensemble source has to be built directly. + """ + return MarkDecorator(Mark(name, args, kwargs, _ispytest=True), _ispytest=True) + + def test_getfuncargnames_functions(): """Test getfuncargnames for normal functions""" @@ -135,46 +151,95 @@ class T: @pytest.mark.pytester_example_path("fixtures/fill_fixtures") class TestFillFixtures: - def test_funcarg_lookupfails(self, pytester: Pytester) -> None: - pytester.copy_example() - result = pytester.runpytest() # "--collect-only") - assert result.ret != 0 - result.stdout.fnmatch_lines( - """ - *def test_func(some)* - *fixture*some*not found* - *xyzsomething* - """ + def test_funcarg_lookupfails(self, tmp_path: Path) -> None: + @pytest.fixture + def xyzsomething(request): + return 42 + + def test_func(some): + pass + + record = run_tests( + xyzsomething, test_func, rootpath=tmp_path, capture_output=True + ) + # A fixture missing at setup is an error, not a failure. + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( + [ + "*def test_func(some)*", + "*fixture*some*not found*", + "*xyzsomething*", + ] ) - def test_detect_recursive_dependency_error(self, pytester: Pytester) -> None: - pytester.copy_example() - result = pytester.runpytest() - result.stdout.fnmatch_lines( + def test_detect_recursive_dependency_error(self, tmp_path: Path) -> None: + @pytest.fixture + def fix1(fix2): + return 1 + + @pytest.fixture + def fix2(fix1): + return 1 + + def test(fix1): + pass + + record = run_tests(fix1, fix2, test, rootpath=tmp_path, capture_output=True) + record.stdout.fnmatch_lines( ["*recursive dependency involving fixture 'fix1' detected*"] ) - def test_funcarg_basic(self, pytester: Pytester) -> None: - pytester.copy_example() - item = pytester.getitem(Path("test_funcarg_basic.py")) - assert isinstance(item, Function) - # Execute's item's setup, which fills fixtures. - item.session._setupstate.setup(item) - del item.funcargs["request"] - assert len(get_public_names(item.funcargs)) == 2 - assert item.funcargs["some"] == "test_func" - assert item.funcargs["other"] == 42 - - def test_funcarg_lookup_modulelevel(self, pytester: Pytester) -> None: - pytester.copy_example() - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + def test_funcarg_basic(self, tmp_path: Path) -> None: + @pytest.fixture + def some(request): + return request.function.__name__ - def test_funcarg_lookup_classlevel(self, pytester: Pytester) -> None: - p = pytester.copy_example() - result = pytester.runpytest(p) - result.stdout.fnmatch_lines(["*1 passed*"]) + @pytest.fixture + def other(request): + return 42 + + def test_func(some, other): + pass + + with Ensemble(some, other, test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + assert isinstance(item, Function) + # Execute's item's setup, which fills fixtures. + item.session._setupstate.setup(item) + del item.funcargs["request"] + assert len(get_public_names(item.funcargs)) == 2 + assert item.funcargs["some"] == "test_func" + assert item.funcargs["other"] == 42 + + def test_funcarg_lookup_modulelevel(self, tmp_path: Path) -> None: + @pytest.fixture + def something(request): + return request.function.__name__ + + class TestClass: + def test_method(self, something): + assert something == "test_method" + + def test_func(something): + assert something == "test_func" + record = run_tests(something, TestClass, test_func, rootpath=tmp_path) + record.assert_outcomes(passed=2) + + def test_funcarg_lookup_classlevel(self, tmp_path: Path) -> None: + class TestClass: + @pytest.fixture + def something(self, request): + return request.instance + + def test_method(self, something): + assert something is self + + record = run_tests(TestClass, rootpath=tmp_path) + record.assert_outcomes(passed=1) + + # ensemble: conftest visibility is per-directory, and ensembles have no + # directory tree below the rootdir to scope conftests to. def test_conftest_funcargs_only_available_in_subdir( self, pytester: Pytester ) -> None: @@ -182,20 +247,46 @@ def test_conftest_funcargs_only_available_in_subdir( result = pytester.runpytest("-v") result.assert_outcomes(passed=2) - def test_extend_fixture_module_class(self, pytester: Pytester) -> None: - testfile = pytester.copy_example() - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 passed*"]) - result = pytester.runpytest(testfile) - result.stdout.fnmatch_lines(["*1 passed*"]) + def test_extend_fixture_module_class(self, tmp_path: Path) -> None: + @pytest.fixture + def spam(): + return "spam" - def test_extend_fixture_conftest_module(self, pytester: Pytester) -> None: - p = pytester.copy_example() - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 passed*"]) - result = pytester.runpytest(str(next(Path(str(p)).rglob("test_*.py")))) - result.stdout.fnmatch_lines(["*1 passed*"]) + class TestSpam: + @pytest.fixture + def spam(self, spam): + return spam * 2 + + def test_spam(self, spam): + assert spam == "spamspam" + + record = run_tests(spam, TestSpam, rootpath=tmp_path) + record.assert_outcomes(passed=1) + + def test_extend_fixture_conftest_module(self, tmp_path: Path) -> None: + # The rootdir conftest is reproduced as a plugin object; the second + # run of the original (passing the test file directly) only covered + # conftest collection for an explicit file argument, which an + # ensemble has no equivalent of. + class ConftestPlugin: + @pytest.fixture + def spam(self): + return "spam" + + @pytest.fixture + def spam(spam): + return spam * 2 + def test_spam(spam): + assert spam == "spamspam" + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_extend", spam=spam, test_spam=test_spam), spec=spec + ) + record.assert_outcomes(passed=1) + + # ensemble: two conftests at different directory levels. def test_extend_fixture_conftest_conftest(self, pytester: Pytester) -> None: p = pytester.copy_example() result = pytester.runpytest() @@ -203,6 +294,7 @@ def test_extend_fixture_conftest_conftest(self, pytester: Pytester) -> None: result = pytester.runpytest(str(next(Path(str(p)).rglob("test_*.py")))) result.stdout.fnmatch_lines(["*1 passed*"]) + # ensemble: needs a real importable plugin module named in `pytest_plugins`. def test_extend_fixture_conftest_plugin(self, pytester: Pytester) -> None: pytester.makepyfile( testplugin=""" @@ -234,68 +326,50 @@ def test_foo(foo): result = pytester.runpytest("-s") assert result.ret == 0 - def test_extend_fixture_plugin_plugin(self, pytester: Pytester) -> None: + def test_extend_fixture_plugin_plugin(self, tmp_path: Path) -> None: # Two plugins should extend each order in loading order - pytester.makepyfile( - testplugin0=""" - import pytest - + class TestPlugin0: @pytest.fixture - def foo(): + def foo(self): return 7 - """ - ) - pytester.makepyfile( - testplugin1=""" - import pytest + class TestPlugin1: @pytest.fixture - def foo(foo): + def foo(self, foo): return foo + 7 - """ - ) - pytester.syspathinsert() - pytester.makepyfile( - """ - pytest_plugins = ['testplugin0', 'testplugin1'] - def test_foo(foo): - assert foo == 14 - """ + def test_foo(foo): + assert foo == 14 + + spec = ConfigSpec( + rootpath=tmp_path, extra_plugins=(TestPlugin0(), TestPlugin1()) ) - result = pytester.runpytest() - assert result.ret == 0 + run_tests(test_foo, spec=spec).assert_outcomes(passed=1) def test_override_parametrized_fixture_conftest_module( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test override of the parametrized fixture with non-parametrized one on the test module level.""" - pytester.makeconftest( - """ - import pytest + class ConftestPlugin: @pytest.fixture(params=[1, 2, 3]) - def spam(request): + def spam(self, request): return request.param - """ - ) - testfile = pytester.makepyfile( - """ - import pytest - @pytest.fixture - def spam(): - return 'spam' + @pytest.fixture + def spam(): + return "spam" - def test_spam(spam): - assert spam == 'spam' - """ + def test_spam(spam): + assert spam == "spam" + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_override", spam=spam, test_spam=test_spam), spec=spec ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 passed*"]) - result = pytester.runpytest(testfile) - result.stdout.fnmatch_lines(["*1 passed*"]) + record.assert_outcomes(passed=1) + # ensemble: the override lives in a subdirectory conftest. def test_override_parametrized_fixture_conftest_conftest( self, pytester: Pytester ) -> None: @@ -338,38 +412,34 @@ def test_spam(spam): result.stdout.fnmatch_lines(["*1 passed*"]) def test_override_non_parametrized_fixture_conftest_module( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test override of the non-parametrized fixture with parametrized one on the test module level.""" - pytester.makeconftest( - """ - import pytest + class ConftestPlugin: @pytest.fixture - def spam(): - return 'spam' - """ - ) - testfile = pytester.makepyfile( - """ - import pytest + def spam(self): + return "spam" - @pytest.fixture(params=[1, 2, 3]) - def spam(request): - return request.param + @pytest.fixture(params=[1, 2, 3]) + def spam(request): + return request.param - params = {'spam': 1} + # The module-level ``params`` dict of the original becomes a closure: + # a source function keeps the *host* module's globals. + params = {"spam": 1} - def test_spam(spam): - assert spam == params['spam'] - params['spam'] += 1 - """ + def test_spam(spam): + assert spam == params["spam"] + params["spam"] += 1 + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_override", spam=spam, test_spam=test_spam), spec=spec ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*3 passed*"]) - result = pytester.runpytest(testfile) - result.stdout.fnmatch_lines(["*3 passed*"]) + record.assert_outcomes(passed=3) + # ensemble: the override lives in a subdirectory conftest. def test_override_non_parametrized_fixture_conftest_conftest( self, pytester: Pytester ) -> None: @@ -414,6 +484,7 @@ def test_spam(spam): result = pytester.runpytest(testfile) result.stdout.fnmatch_lines(["*3 passed*"]) + # ensemble: the override lives in a subdirectory conftest. def test_override_autouse_fixture_with_parametrized_fixture_conftest_conftest( self, pytester: Pytester ) -> None: @@ -461,224 +532,181 @@ def test_spam(spam): result.stdout.fnmatch_lines(["*3 passed*"]) def test_override_fixture_reusing_super_fixture_parametrization( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Override a fixture at a lower level, reusing the higher-level fixture that is parametrized (#1953). """ - pytester.makeconftest( - """ - import pytest + class ConftestPlugin: @pytest.fixture(params=[1, 2]) - def foo(request): + def foo(self, request): return request.param - """ - ) - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def foo(foo): - return foo * 2 + @pytest.fixture + def foo(foo): + return foo * 2 - def test_spam(foo): - assert foo in (2, 4) - """ + def test_spam(foo): + assert foo in (2, 4) + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_override", foo=foo, test_spam=test_spam), spec=spec ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*2 passed*"]) + record.assert_outcomes(passed=2) - def test_override_parametrize_fixture_and_indirect( - self, pytester: Pytester - ) -> None: + def test_override_parametrize_fixture_and_indirect(self, tmp_path: Path) -> None: """Override a fixture at a lower level, reusing the higher-level fixture that is parametrized, while also using indirect parametrization. """ - pytester.makeconftest( - """ - import pytest + class ConftestPlugin: @pytest.fixture(params=[1, 2]) - def foo(request): + def foo(self, request): return request.param - """ - ) - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def foo(foo): - return foo * 2 + @pytest.fixture + def foo(foo): + return foo * 2 - @pytest.fixture - def bar(request): - return request.param * 100 + @pytest.fixture + def bar(request): + return request.param * 100 - @pytest.mark.parametrize("bar", [42], indirect=True) - def test_spam(bar, foo): - assert bar == 4200 - assert foo in (2, 4) - """ + @pytest.mark.parametrize("bar", [42], indirect=True) + def test_spam(bar, foo): + assert bar == 4200 + assert foo in (2, 4) + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_override", foo=foo, bar=bar, test_spam=test_spam), + spec=spec, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*2 passed*"]) + record.assert_outcomes(passed=2) def test_override_top_level_fixture_reusing_super_fixture_parametrization( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Same as the above test, but with another level of overwriting.""" - pytester.makeconftest( - """ - import pytest - @pytest.fixture(params=['unused', 'unused']) - def foo(request): + class ConftestPlugin: + @pytest.fixture(params=["unused", "unused"]) + def foo(self, request): return request.param - """ - ) - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=[1, 2]) - def foo(request): - return request.param + @pytest.fixture(params=[1, 2]) + def foo(request): + return request.param - class Test: + class Test: + @pytest.fixture + def foo(self, foo): + return foo * 2 - @pytest.fixture - def foo(self, foo): - return foo * 2 + def test_spam(self, foo): + assert foo in (2, 4) - def test_spam(self, foo): - assert foo in (2, 4) - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*2 passed*"]) + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(build_module("test_override", foo=foo, Test=Test), spec=spec) + record.assert_outcomes(passed=2) def test_override_parametrized_fixture_with_new_parametrized_fixture( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Overriding a parametrized fixture, while also parametrizing the new fixture and simultaneously requesting the overwritten fixture as parameter, yields the same value as ``request.param``. """ - pytester.makeconftest( - """ - import pytest - @pytest.fixture(params=['ignored', 'ignored']) - def foo(request): + class ConftestPlugin: + @pytest.fixture(params=["ignored", "ignored"]) + def foo(self, request): return request.param - """ - ) - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=[10, 20]) - def foo(foo, request): - assert request.param == foo - return foo * 2 + @pytest.fixture(params=[10, 20]) + def foo(foo, request): + assert request.param == foo + return foo * 2 - def test_spam(foo): - assert foo in (20, 40) - """ + def test_spam(foo): + assert foo in (20, 40) + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_override", foo=foo, test_spam=test_spam), spec=spec ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*2 passed*"]) + record.assert_outcomes(passed=2) @pytest.mark.xfail(reason="not handled currently") def test_override_parametrized_fixture_via_transitive_fixture( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test that overriding a parametrized fixture works even the super fixture is requested only transitively. Regression test for #7737. """ - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=[1, 2]) - def foo(request): - return request.param + @pytest.fixture(params=[1, 2]) + def foo(request): + return request.param + + @pytest.fixture + def bar(foo): + return foo + class TestIt: @pytest.fixture - def bar(foo): - return foo + def foo(self, bar): + return bar * 2 - class TestIt: - @pytest.fixture - def foo(self, bar): - return bar * 2 + def test_it(self, foo): + pass - def test_it(self, foo): - pass - """ - ) - result = pytester.runpytest() - assert result.ret == ExitCode.OK - result.assert_outcomes(passed=2) + record = run_tests(foo, bar, TestIt, rootpath=tmp_path) + record.assert_outcomes(passed=2) - def test_autouse_fixture_plugin(self, pytester: Pytester) -> None: + def test_autouse_fixture_plugin(self, tmp_path: Path) -> None: # A fixture from a plugin has no baseid set, which screwed up # the autouse fixture handling. - pytester.makepyfile( - testplugin=""" - import pytest - + class TestPlugin: @pytest.fixture(autouse=True) - def foo(request): + def foo(self, request): request.function.foo = 7 - """ - ) - pytester.syspathinsert() - pytester.makepyfile( - """ - pytest_plugins = 'testplugin' - def test_foo(request): - assert request.function.foo == 7 - """ - ) - result = pytester.runpytest() - assert result.ret == 0 + def test_foo(request): + assert request.function.foo == 7 - def test_funcarg_lookup_error(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(TestPlugin(),)) + run_tests(test_foo, spec=spec).assert_outcomes(passed=1) + def test_funcarg_lookup_error(self, tmp_path: Path) -> None: + class ConftestPlugin: @pytest.fixture - def a_fixture(): pass + def a_fixture(self): ... @pytest.fixture - def b_fixture(): pass + def b_fixture(self): ... @pytest.fixture - def c_fixture(): pass + def c_fixture(self): ... @pytest.fixture - def d_fixture(): pass - """ - ) - pytester.makepyfile( - """ - def test_lookup_error(unknown): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ + def d_fixture(self): ... + + def test_lookup_error(unknown): + pass + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(test_lookup_error, spec=spec, capture_output=True) + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( + [ "*ERROR at setup of test_lookup_error*", - " def test_lookup_error(unknown):*", + # indentation is host-anchored: the source lives in this file + "*def test_lookup_error(unknown):*", "E fixture 'unknown' not found", "> available fixtures:*a_fixture,*b_fixture,*c_fixture,*d_fixture*monkeypatch,*", # sorted @@ -686,91 +714,80 @@ def test_lookup_error(unknown): "*1 error*", ] ) - result.stdout.no_fnmatch_line("*INTERNAL*") + record.stdout.no_fnmatch_line("*INTERNAL*") - def test_fixture_excinfo_leak(self, pytester: Pytester) -> None: + def test_fixture_excinfo_leak(self, tmp_path: Path) -> None: # on python2 sys.excinfo would leak into fixture executions - pytester.makepyfile( - """ - import sys - import traceback - import pytest + import traceback - @pytest.fixture - def leak(): - if sys.exc_info()[0]: # python3 bug :) - traceback.print_exc() - #fails - assert sys.exc_info() == (None, None, None) - - def test_leak(leak): - if sys.exc_info()[0]: # python3 bug :) - traceback.print_exc() - assert sys.exc_info() == (None, None, None) - """ - ) - result = pytester.runpytest() - assert result.ret == 0 + @pytest.fixture + def leak(): + if sys.exc_info()[0]: # python3 bug :) + traceback.print_exc() + # fails + assert sys.exc_info() == (None, None, None) + def test_leak(leak): + if sys.exc_info()[0]: # python3 bug :) + traceback.print_exc() + assert sys.exc_info() == (None, None, None) + + run_tests(leak, test_leak, rootpath=tmp_path).assert_outcomes(passed=1) -class TestRequestBasic: - def test_request_attributes(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest +class TestRequestBasic: + def test_request_attributes(self, tmp_path: Path) -> None: + @pytest.fixture + def something(request): ... + + def test_func(something): ... + + with Ensemble(something, test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + assert isinstance(item, Function) + req = TopRequest(item, _ispytest=True) + assert req.function == item.obj + assert req.keywords == item.keywords + assert hasattr(req.module, "test_func") + assert req.cls is None + assert req.function.__name__ == "test_func" + assert req.config == item.config + assert repr(req).find(req.function.__name__) != -1 + + def test_request_attributes_method(self, tmp_path: Path) -> None: + class TestB: @pytest.fixture - def something(request): pass - def test_func(something): pass - """ - ) - assert isinstance(item, Function) - req = TopRequest(item, _ispytest=True) - assert req.function == item.obj - assert req.keywords == item.keywords - assert hasattr(req.module, "test_func") - assert req.cls is None - assert req.function.__name__ == "test_func" - assert req.config == item.config - assert repr(req).find(req.function.__name__) != -1 - - def test_request_attributes_method(self, pytester: Pytester) -> None: - (item,) = pytester.getitems( - """ - import pytest - class TestB(object): + def something(self, request): + return 1 - @pytest.fixture - def something(self, request): - return 1 - def test_func(self, something): - pass - """ - ) - assert isinstance(item, Function) - req = item._request - assert req.cls.__name__ == "TestB" - assert req.instance.__class__ == req.cls + def test_func(self, something): + pass - def test_request_contains_funcarg_arg2fixturedefs(self, pytester: Pytester) -> None: - modcol = pytester.getmodulecol( - """ - import pytest - @pytest.fixture - def something(request): + with Ensemble(TestB, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + assert isinstance(item, Function) + req = item._request + assert req.cls.__name__ == "TestB" + assert req.instance.__class__ == req.cls + + def test_request_contains_funcarg_arg2fixturedefs(self, tmp_path: Path) -> None: + @pytest.fixture + def something(request): + pass + + class TestClass: + def test_method(self, something): pass - class TestClass(object): - def test_method(self, something): - pass - """ - ) - (item1,) = pytester.genitems([modcol]) - assert isinstance(item1, Function) - assert item1.name == "test_method" - arg2fixturedefs = TopRequest(item1, _ispytest=True)._arg2fixturedefs - assert len(arg2fixturedefs) == 1 - assert arg2fixturedefs["something"][0].argname == "something" + with Ensemble(something, TestClass, rootpath=tmp_path) as ensemble: + (item1,) = ensemble.collect() + assert isinstance(item1, Function) + assert item1.name == "test_method" + arg2fixturedefs = TopRequest(item1, _ispytest=True)._arg2fixturedefs + assert len(arg2fixturedefs) == 1 + assert arg2fixturedefs["something"][0].argname == "something" + + # ensemble: needs a subprocess run (gc debug state is process-global). @pytest.mark.skipif( hasattr(sys, "pypy_version_info"), reason="this method of test doesn't work on pypy", @@ -811,31 +828,27 @@ def test_func(): result = pytester.runpytest_subprocess() result.stdout.fnmatch_lines(["* 1 passed in *"]) - def test_getfixturevalue_recursive(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - + def test_getfixturevalue_recursive(self, tmp_path: Path) -> None: + class ConftestPlugin: @pytest.fixture - def something(request): + def something(self, request): return 1 - """ - ) - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def something(request): - return request.getfixturevalue("something") + 1 - def test_func(something): - assert something == 2 - """ + @pytest.fixture + def something(request): + return request.getfixturevalue("something") + 1 + + def test_func(something): + assert something == 2 + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_recursive", something=something, test_func=test_func), + spec=spec, ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + record.assert_outcomes(passed=1) - def test_getfixturevalue_teardown(self, pytester: Pytester) -> None: + def test_getfixturevalue_teardown(self, tmp_path: Path) -> None: """ Issue #1895 @@ -846,35 +859,31 @@ def test_getfixturevalue_teardown(self, pytester: Pytester) -> None: `inner` dependent on `resource` when it is used via `getfixturevalue`: `test_func` will then cause the `resource`'s finalizer to be called first because of this. """ - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope='session') - def resource(): - r = ['value'] - yield r - r.pop() + @pytest.fixture(scope="session") + def resource(): + r = ["value"] + yield r + r.pop() - @pytest.fixture(scope='session') - def inner(request): - resource = request.getfixturevalue('resource') - assert resource == ['value'] - yield - assert resource == ['value'] + @pytest.fixture(scope="session") + def inner(request): + resource = request.getfixturevalue("resource") + assert resource == ["value"] + yield + assert resource == ["value"] - def test_inner(inner): - pass + def test_inner(inner): + pass - def test_func(resource): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 2 passed in *"]) + def test_func(resource): + pass + + record = run_tests(resource, inner, test_inner, test_func, rootpath=tmp_path) + record.assert_outcomes(passed=2) def test_getfixturevalue_teardown_previously_requested_does_not_warn( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test that requesting a fixture during teardown that was previously requested is OK (#12882). @@ -882,24 +891,22 @@ def test_getfixturevalue_teardown_previously_requested_does_not_warn( Note: this is still kinda dubious so don't let this test lock you in to allowing this behavior forever... """ - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def fix(request, tmp_path): - yield - assert request.getfixturevalue("tmp_path") == tmp_path + @pytest.fixture + def fix(request, tmp_path): + yield + assert request.getfixturevalue("tmp_path") == tmp_path - def test_it(fix): - pass - """ - ) - result = pytester.runpytest("-Werror") - result.assert_outcomes(passed=1) + def test_it(fix): + pass + + # -Werror of the original: any warning would fail the run. + spec = ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["error"]}) + record = run_tests(fix, test_it, spec=spec) + record.assert_outcomes(passed=1, warnings=0) def test_getfixturevalue_teardown_new_fixture_deprecated( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test that requesting a fixture during teardown that was not previously requested raises a deprecation warning (#12882). @@ -907,236 +914,230 @@ def test_getfixturevalue_teardown_new_fixture_deprecated( Note: this is a case that previously worked but will become a hard error after the deprecation is completed. """ - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope="session") - def resource(): - return "value" + @pytest.fixture(scope="session") + def resource(): + return "value" - @pytest.fixture - def fix(request): - yield - with pytest.warns( - pytest.PytestRemovedIn10Warning, - match=r'Calling request\\.getfixturevalue\\("resource"\\) during teardown is deprecated', - ): - assert request.getfixturevalue("resource") == "value" + @pytest.fixture + def fix(request): + yield + with pytest.warns( + pytest.PytestRemovedIn10Warning, + match=r'Calling request\.getfixturevalue\("resource"\) during teardown is deprecated', + ): + assert request.getfixturevalue("resource") == "value" - def test_it(fix): - pass - """ - ) - result = pytester.runpytest() - result.assert_outcomes(passed=1) + def test_it(fix): + pass + + record = run_tests(resource, fix, test_it, rootpath=tmp_path) + record.assert_outcomes(passed=1) def test_getfixturevalue_teardown_new_inactive_fixture_errors( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test that requesting a fixture during teardown that was not previously requested raises an error (#12882).""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def fix(request): - yield - request.getfixturevalue("tmp_path") + @pytest.fixture + def fix(request): + yield + request.getfixturevalue("tmp_path") - def test_it(fix): - pass - """ - ) - result = pytester.runpytest() - result.assert_outcomes(passed=1, errors=1) - result.stdout.fnmatch_lines( - [ - ( - '*The fixture value for "tmp_path" is not available during ' - "teardown because it was not previously requested.*" - ), - ] + def test_it(fix): + pass + + record = run_tests(fix, test_it, rootpath=tmp_path) + # The call phase passes; the teardown error is a separate report. + record.assert_outcomes(passed=1, errors=1) + teardown = record["test_it"].teardown + assert teardown is not None + assert ( + 'The fixture value for "tmp_path" is not available during ' + "teardown because it was not previously requested." in teardown.longreprtext ) def test_getfixturevalue_teardown_new_inactive_fixture_errors_top_request( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test that requesting a fixture during teardown that was not previously requested raises an error (tricky case) (#12882).""" - pytester.makepyfile( - """ - def test_it(request): - request.addfinalizer(lambda: request.getfixturevalue("tmp_path")) - """ - ) - result = pytester.runpytest() - result.assert_outcomes(passed=1, errors=1) - result.stdout.fnmatch_lines( - [ - ( - '*The fixture value for "tmp_path" is not available during ' - "teardown because it was not previously requested.*" - ), - ] + + def test_it(request): + request.addfinalizer(lambda: request.getfixturevalue("tmp_path")) + + record = run_tests(test_it, rootpath=tmp_path) + record.assert_outcomes(passed=1, errors=1) + teardown = record["test_it"].teardown + assert teardown is not None + assert ( + 'The fixture value for "tmp_path" is not available during ' + "teardown because it was not previously requested." in teardown.longreprtext ) - def test_getfixturevalue(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest + def test_getfixturevalue(self, tmp_path: Path) -> None: + @pytest.fixture + def something(request): + return 1 - @pytest.fixture - def something(request): - return 1 + # A module-level ``values`` list of the original: a source function + # would read the *host* module's globals, so it becomes a closure. + values = [2] - values = [2] - @pytest.fixture - def other(request): - return values.pop() + @pytest.fixture + def other(request): + return values.pop() + + def test_func(something): ... + + with Ensemble(something, other, test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + assert isinstance(item, Function) + req = item._request + + # Execute item's setup. + item.session._setupstate.setup(item) + + with pytest.raises(pytest.FixtureLookupError): + req.getfixturevalue("notexists") + val = req.getfixturevalue("something") + assert val == 1 + val = req.getfixturevalue("something") + assert val == 1 + val2 = req.getfixturevalue("other") + assert val2 == 2 + val2 = req.getfixturevalue("other") # see about caching + assert val2 == 2 + assert item.funcargs["something"] == 1 + assert len(get_public_names(item.funcargs)) == 2 + assert "request" in item.funcargs + + def test_request_addfinalizer(self, tmp_path: Path) -> None: + teardownlist: list[int] = [] - def test_func(something): pass - """ - ) - assert isinstance(item, Function) - req = item._request - - # Execute item's setup. - item.session._setupstate.setup(item) - - with pytest.raises(pytest.FixtureLookupError): - req.getfixturevalue("notexists") - val = req.getfixturevalue("something") - assert val == 1 - val = req.getfixturevalue("something") - assert val == 1 - val2 = req.getfixturevalue("other") - assert val2 == 2 - val2 = req.getfixturevalue("other") # see about caching - assert val2 == 2 - assert item.funcargs["something"] == 1 - assert len(get_public_names(item.funcargs)) == 2 - assert "request" in item.funcargs - - def test_request_addfinalizer(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - teardownlist = [] - @pytest.fixture - def something(request): - request.addfinalizer(lambda: teardownlist.append(1)) - def test_func(something): pass - """ - ) - assert isinstance(item, Function) - item.session._setupstate.setup(item) - item._request._fillfixtures() - # successively check finalization calls - parent = item.getparent(pytest.Module) - assert parent is not None - teardownlist = parent.obj.teardownlist - ss = item.session._setupstate - assert not teardownlist - ss.teardown_exact(None) - print(ss.stack) - assert teardownlist == [1] - - def test_request_addfinalizer_failing_setup(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [1] - @pytest.fixture - def myfix(request): - request.addfinalizer(values.pop) - assert 0 - def test_fix(myfix): - pass - def test_finalizer_ran(): - assert not values - """ - ) - reprec = pytester.inline_run("-s") - reprec.assertoutcome(failed=1, passed=1) + @pytest.fixture + def something(request): + request.addfinalizer(lambda: teardownlist.append(1)) + + def test_func(something): ... + + module = build_module( + "test_addfinalizer", + something=something, + test_func=test_func, + teardownlist=teardownlist, + ) + with Ensemble(module, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + assert isinstance(item, Function) + item.session._setupstate.setup(item) + item._request._fillfixtures() + # successively check finalization calls + parent = item.getparent(pytest.Module) + assert parent is not None + assert parent.obj.teardownlist is teardownlist + ss = item.session._setupstate + assert not teardownlist + ss.teardown_exact(None) + print(ss.stack) + assert teardownlist == [1] + + def test_request_addfinalizer_failing_setup(self, tmp_path: Path) -> None: + values = [1] - def test_request_addfinalizer_failing_setup_module( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( - """ - import pytest - values = [1, 2] - @pytest.fixture(scope="module") - def myfix(request): - request.addfinalizer(values.pop) - request.addfinalizer(values.pop) - assert 0 - def test_fix(myfix): - pass - """ - ) - reprec = pytester.inline_run("-s") - mod = reprec.getcalls("pytest_runtest_setup")[0].item.module - assert not mod.values + @pytest.fixture + def myfix(request): + request.addfinalizer(values.pop) + assert 0 - def test_request_addfinalizer_partial_setup_failure( - self, pytester: Pytester - ) -> None: - p = pytester.makepyfile( - """ - import pytest - values = [] - @pytest.fixture - def something(request): - request.addfinalizer(lambda: values.append(None)) - def test_func(something, missingarg): - pass - def test_second(): - assert len(values) == 1 - """ - ) - result = pytester.runpytest(p) - result.stdout.fnmatch_lines( - ["*1 error*"] # XXX the whole module collection fails - ) + def test_fix(myfix): + pass - def test_request_subrequest_addfinalizer_exceptions( - self, pytester: Pytester - ) -> None: + def test_finalizer_ran(): + assert not values + + record = run_tests(myfix, test_fix, test_finalizer_ran, rootpath=tmp_path) + # A failing *setup* is an error in terminal categories, where the + # original's assertoutcome(failed=1) counted the failed report. + record.assert_outcomes(errors=1, passed=1) + + def test_request_addfinalizer_failing_setup_module(self, tmp_path: Path) -> None: + values = [1, 2] + + @pytest.fixture(scope="module") + def myfix(request): + request.addfinalizer(values.pop) + request.addfinalizer(values.pop) + assert 0 + + def test_fix(myfix): + pass + + run_tests(myfix, test_fix, rootpath=tmp_path) + assert not values + + def test_request_addfinalizer_partial_setup_failure(self, tmp_path: Path) -> None: + values: list[None] = [] + + @pytest.fixture + def something(request): + request.addfinalizer(lambda: values.append(None)) + + def test_func(something, missingarg): + pass + + def test_second(): + assert len(values) == 1 + + record = run_tests(something, test_func, test_second, rootpath=tmp_path) + record.assert_outcomes(errors=1, passed=1) + assert record["test_func"].outcome == "error" + assert record["test_second"].passed + + def test_request_subrequest_addfinalizer_exceptions(self, tmp_path: Path) -> None: """ Ensure exceptions raised during teardown by finalizers are suppressed until all finalizers are called, then re-raised together in an exception group (#2440) """ - pytester.makepyfile( - """ - import pytest - values = [] - def _excepts(where): - raise Exception('Error in %s fixture' % where) - @pytest.fixture - def subrequest(request): - return request - @pytest.fixture - def something(subrequest): - subrequest.addfinalizer(lambda: values.append(1)) - subrequest.addfinalizer(lambda: values.append(2)) - subrequest.addfinalizer(lambda: _excepts('something')) - @pytest.fixture - def excepts(subrequest): - subrequest.addfinalizer(lambda: _excepts('excepts')) - subrequest.addfinalizer(lambda: values.append(3)) - def test_first(something, excepts): - pass - def test_second(): - assert values == [3, 2, 1] - """ - ) - result = pytester.runpytest() - result.assert_outcomes(passed=2, errors=1) - result.stdout.fnmatch_lines( + values: list[int] = [] + + def _excepts(where): + raise Exception(f"Error in {where} fixture") + + @pytest.fixture + def subrequest(request): + return request + + @pytest.fixture + def something(subrequest): + subrequest.addfinalizer(lambda: values.append(1)) + subrequest.addfinalizer(lambda: values.append(2)) + subrequest.addfinalizer(lambda: _excepts("something")) + + @pytest.fixture + def excepts(subrequest): + subrequest.addfinalizer(lambda: _excepts("excepts")) + subrequest.addfinalizer(lambda: values.append(3)) + + def test_first(something, excepts): + pass + + def test_second(): + assert values == [3, 2, 1] + + record = run_tests( + subrequest, + something, + excepts, + test_first, + test_second, + rootpath=tmp_path, + capture_output=True, + ) + record.assert_outcomes(passed=2, errors=1) + record.stdout.fnmatch_lines( [ ' | *ExceptionGroup: errors while tearing down fixture "subrequest" of (2 sub-exceptions)', # noqa: E501 " +-+---------------- 1 ----------------", @@ -1147,47 +1148,69 @@ def test_second(): ], ) - def test_request_getmodulepath(self, pytester: Pytester) -> None: - modcol = pytester.getmodulecol("def test_somefunc(): pass") - (item,) = pytester.genitems([modcol]) - assert isinstance(item, Function) - req = TopRequest(item, _ispytest=True) - assert req.path == modcol.path + def test_request_getmodulepath(self, tmp_path: Path) -> None: + def test_somefunc(): ... - def test_request_fixturenames(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - from _pytest.pytester import get_public_names - @pytest.fixture() - def arg1(): - pass - @pytest.fixture() - def farg(arg1): - pass - @pytest.fixture(autouse=True) - def sarg(tmp_path): - pass - def test_function(request, farg): - assert set(get_public_names(request.fixturenames)) == \ - set(["sarg", "arg1", "request", "farg", - "tmp_path", "tmp_path_factory"]) - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + with Ensemble(test_somefunc, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + assert isinstance(item, Function) + modcol = item.getparent(pytest.Module) + assert modcol is not None + req = TopRequest(item, _ispytest=True) + assert req.path == modcol.path - def test_request_fixturenames_dynamic_fixture(self, pytester: Pytester) -> None: - """Regression test for #3057""" - pytester.copy_example("fixtures/test_getfixturevalue_dynamic.py") - result = pytester.runpytest("-vv") - result.stdout.fnmatch_lines(["*1 passed*"]) + def test_request_fixturenames(self, tmp_path: Path) -> None: + @pytest.fixture + def arg1(): + pass - def test_setupdecorator_and_xunit(self, tmp_path: Path) -> None: - values: list[str] = [] + @pytest.fixture + def farg(arg1): + pass - @pytest.fixture(scope="module", autouse=True) - def setup_module(): + @pytest.fixture(autouse=True) + def sarg(tmp_path): + pass + + def test_function(request, farg): + assert set(get_public_names(request.fixturenames)) == { + "sarg", + "arg1", + "request", + "farg", + "tmp_path", + "tmp_path_factory", + } + + record = run_tests(arg1, farg, sarg, test_function, rootpath=tmp_path) + record.assert_outcomes(passed=1) + + def test_request_fixturenames_dynamic_fixture(self, tmp_path: Path) -> None: + """Regression test for #3057""" + + @pytest.fixture + def dynamic(): + pass + + @pytest.fixture + def a(request): + request.getfixturevalue("dynamic") + + @pytest.fixture + def b(a): + pass + + def test(b, request): + assert request.fixturenames == ["b", "a", "request", "dynamic"] + + record = run_tests(dynamic, a, b, test, rootpath=tmp_path) + record.assert_outcomes(passed=1) + + def test_setupdecorator_and_xunit(self, tmp_path: Path) -> None: + values: list[str] = [] + + @pytest.fixture(scope="module", autouse=True) + def setup_module(): values.append("module") @pytest.fixture(autouse=True) @@ -1216,6 +1239,8 @@ def test_method(self): record.assert_outcomes(passed=2) assert values == ["module", "function", "class", "function", "method"] + # ensemble: --fixtures runs through pytest_cmdline_main, which an + # ensemble never reaches, and the subject is a subdirectory conftest. def test_fixtures_sub_subdir_normalize_sep(self, pytester: Pytester) -> None: # this tests that normalization of nodeids takes place b = pytester.path.joinpath("tests", "unit") @@ -1242,42 +1267,35 @@ def arg1(): """ ) + # ensemble: --fixtures runs through pytest_cmdline_main, unreachable. def test_show_fixtures_color_yes(self, pytester: Pytester) -> None: pytester.makepyfile("def test_this(): assert 1") result = pytester.runpytest("--color=yes", "--fixtures") assert "\x1b[32mtmp_path" in result.stdout.str() - def test_newstyle_with_request(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture() - def arg(request): - pass - def test_1(arg): - pass - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + def test_newstyle_with_request(self, tmp_path: Path) -> None: + @pytest.fixture + def arg(request): + pass - def test_setupcontext_no_param(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=[1,2]) - def arg(request): - return request.param + def test_1(arg): + pass - @pytest.fixture(autouse=True) - def mysetup(request, arg): - assert not hasattr(request, "param") - def test_1(arg): - assert arg in (1,2) - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + run_tests(arg, test_1, rootpath=tmp_path).assert_outcomes(passed=1) + + def test_setupcontext_no_param(self, tmp_path: Path) -> None: + @pytest.fixture(params=[1, 2]) + def arg(request): + return request.param + + @pytest.fixture(autouse=True) + def mysetup(request, arg): + assert not hasattr(request, "param") + + def test_1(arg): + assert arg in (1, 2) + + run_tests(arg, mysetup, test_1, rootpath=tmp_path).assert_outcomes(passed=2) class TestRequestSessionScoped: @@ -1295,114 +1313,98 @@ def test_session_scoped_unavailable_attributes(self, session_request, name): class TestRequestMarking: - def test_applymarker(self, pytester: Pytester) -> None: - item1, _item2 = pytester.getitems( - """ - import pytest + def test_applymarker(self, tmp_path: Path) -> None: + @pytest.fixture + def something(request): + pass - @pytest.fixture - def something(request): + class TestClass: + def test_func1(self, something): pass - class TestClass(object): - def test_func1(self, something): - pass - def test_func2(self, something): - pass - """ - ) - assert isinstance(item1, Function) - req1 = TopRequest(item1, _ispytest=True) - assert "xfail" not in item1.keywords - req1.applymarker(pytest.mark.xfail) - assert "xfail" in item1.keywords - assert "skipif" not in item1.keywords - req1.applymarker(pytest.mark.skipif) - assert "skipif" in item1.keywords - with pytest.raises(ValueError): - req1.applymarker(42) # type: ignore[arg-type] - def test_accesskeywords(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture() - def keywords(request): - return request.keywords - @pytest.mark.XYZ - def test_function(keywords): - assert keywords["XYZ"] - assert "abc" not in keywords - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + def test_func2(self, something): + pass - def test_accessmarker_dynamic(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - @pytest.fixture() - def keywords(request): + with Ensemble(something, TestClass, rootpath=tmp_path) as ensemble: + item1, _item2 = ensemble.collect() + assert isinstance(item1, Function) + req1 = TopRequest(item1, _ispytest=True) + assert "xfail" not in item1.keywords + req1.applymarker(pytest.mark.xfail) + assert "xfail" in item1.keywords + assert "skipif" not in item1.keywords + req1.applymarker(pytest.mark.skipif) + assert "skipif" in item1.keywords + with pytest.raises(ValueError): + req1.applymarker(42) # type: ignore[arg-type] + + def test_accesskeywords(self, tmp_path: Path) -> None: + @pytest.fixture + def keywords(request): + return request.keywords + + @unregistered_mark("XYZ") + def test_function(keywords): + assert keywords["XYZ"] + assert "abc" not in keywords + + run_tests(keywords, test_function, rootpath=tmp_path).assert_outcomes(passed=1) + + def test_accessmarker_dynamic(self, tmp_path: Path) -> None: + class ConftestPlugin: + @pytest.fixture + def keywords(self, request): return request.keywords @pytest.fixture(scope="class", autouse=True) - def marking(request): + def marking(self, request): request.applymarker(pytest.mark.XYZ("hello")) - """ - ) - pytester.makepyfile( - """ - import pytest - def test_fun1(keywords): - assert keywords["XYZ"] is not None - assert "abc" not in keywords - def test_fun2(keywords): - assert keywords["XYZ"] is not None - assert "abc" not in keywords - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + + def test_fun1(keywords): + assert keywords["XYZ"] is not None + assert "abc" not in keywords + + def test_fun2(keywords): + assert keywords["XYZ"] is not None + assert "abc" not in keywords + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + run_tests(test_fun1, test_fun2, spec=spec).assert_outcomes(passed=2) class TestFixtureUsages: - def test_noargfixturedec(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def arg1(): - return 1 + def test_noargfixturedec(self, tmp_path: Path) -> None: + @pytest.fixture + def arg1(): + return 1 - def test_func(arg1): - assert arg1 == 1 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + def test_func(arg1): + assert arg1 == 1 - def test_receives_funcargs(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture() - def arg1(): - return 1 + run_tests(arg1, test_func, rootpath=tmp_path).assert_outcomes(passed=1) - @pytest.fixture() - def arg2(arg1): - return arg1 + 1 + def test_receives_funcargs(self, tmp_path: Path) -> None: + @pytest.fixture + def arg1(): + return 1 - def test_add(arg2): - assert arg2 == 2 - def test_all(arg1, arg2): - assert arg1 == 1 - assert arg2 == 2 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + @pytest.fixture + def arg2(arg1): + return arg1 + 1 + + def test_add(arg2): + assert arg2 == 2 + + def test_all(arg1, arg2): + assert arg1 == 1 + assert arg2 == 2 + record = run_tests(arg1, arg2, test_add, test_all, rootpath=tmp_path) + record.assert_outcomes(passed=2) + + # ensemble: asserts the fixtures' file:line, which is host-anchored for + # in-memory sources (see test_receives_funcargs_scope_mismatch_issue660 + # for the same failure asserted without locations). def test_receives_funcargs_scope_mismatch(self, pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1430,26 +1432,20 @@ def test_add(arg2): ] ) - def test_receives_funcargs_scope_mismatch_issue660( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope="function") - def arg1(): - return 1 + def test_receives_funcargs_scope_mismatch_issue660(self, tmp_path: Path) -> None: + @pytest.fixture(scope="function") + def arg1(): + return 1 - @pytest.fixture(scope="module") - def arg2(arg1): - return arg1 + 1 + @pytest.fixture(scope="module") + def arg2(arg1): + return arg1 + 1 - def test_add(arg1, arg2): - assert arg2 == 2 - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + def test_add(arg1, arg2): + assert arg2 == 2 + + record = run_tests(arg1, arg2, test_add, rootpath=tmp_path, capture_output=True) + record.stdout.fnmatch_lines( [ "*ScopeMismatch*Requesting fixture stack*", "* def arg2(arg1)", @@ -1459,105 +1455,108 @@ def test_add(arg1, arg2): ], ) - def test_invalid_scope(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope="functions") - def badscope(): - pass + def test_invalid_scope(self, tmp_path: Path) -> None: + @pytest.fixture(scope="functions") # type: ignore[call-overload] + def badscope(): + pass - def test_nothing(badscope): - pass - """ + def test_nothing(badscope): + pass + + record = run_tests( + build_module("test_invalid_scope", badscope, test_nothing), + rootpath=tmp_path, + capture_output=True, ) - result = pytester.runpytest_inprocess() - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( "*Fixture 'badscope' from test_invalid_scope.py got an unexpected scope value 'functions'" ) @pytest.mark.parametrize("scope", ["function", "session"]) - def test_parameters_without_eq_semantics(self, scope, pytester: Pytester) -> None: - pytester.makepyfile( - f""" - class NoEq1: # fails on `a == b` statement - def __eq__(self, _): - raise RuntimeError - - class NoEq2: # fails on `if a == b:` statement - def __eq__(self, _): - class NoBool: - def __bool__(self): - raise RuntimeError - return NoBool() + def test_parameters_without_eq_semantics(self, scope, tmp_path: Path) -> None: + class NoEq1: # fails on `a == b` statement + def __eq__(self, _): + raise RuntimeError - import pytest - @pytest.fixture(params=[NoEq1(), NoEq2()], scope={scope!r}) - def no_eq(request): - return request.param + class NoEq2: # fails on `if a == b:` statement + def __eq__(self, _): + class NoBool: + def __bool__(self): + raise RuntimeError - def test1(no_eq): - pass + return NoBool() - def test2(no_eq): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*4 passed*"]) + @pytest.fixture(params=[NoEq1(), NoEq2()], scope=scope) + def no_eq(request): + return request.param - def test_funcarg_parametrized_and_used_twice(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [] - @pytest.fixture(params=[1,2]) - def arg1(request): - values.append(1) - return request.param + def test1(no_eq): + pass - @pytest.fixture() - def arg2(arg1): - return arg1 + 1 + def test2(no_eq): + pass - def test_add(arg1, arg2): - assert arg2 == arg1 + 1 - assert len(values) == arg1 - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*2 passed*"]) + record = run_tests(no_eq, test1, test2, rootpath=tmp_path) + record.assert_outcomes(passed=4) + + def test_funcarg_parametrized_and_used_twice(self, tmp_path: Path) -> None: + values: list[int] = [] + + @pytest.fixture(params=[1, 2]) + def arg1(request): + values.append(1) + return request.param + + @pytest.fixture + def arg2(arg1): + return arg1 + 1 + + def test_add(arg1, arg2): + assert arg2 == arg1 + 1 + assert len(values) == arg1 + + record = run_tests(arg1, arg2, test_add, rootpath=tmp_path) + record.assert_outcomes(passed=2) def test_factory_uses_unknown_funcarg_as_dependency_error( - self, pytester: Pytester + self, tmp_path: Path ) -> None: - pytester.makepyfile( - """ - import pytest + @pytest.fixture + def fail(missing): + return - @pytest.fixture() - def fail(missing): - return + @pytest.fixture + def call_fail(fail): + return - @pytest.fixture() - def call_fail(fail): - return + def test_missing(call_fail): + pass - def test_missing(call_fail): - pass - """ + # ``fail`` carries the fixture wrapper's name, so it is passed by + # keyword to keep its own name in the synthesized module. + record = run_tests( + build_module( + "test_unknown_dependency", + call_fail, + test_missing, + fail=fail, + ), + rootpath=tmp_path, + capture_output=True, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - """ - *pytest.fixture()* - *def call_fail(fail)* - *pytest.fixture()* - *def fail* - *fixture*'missing'*not found* - """ + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( + [ + "*pytest.fixture*", + "*def call_fail(fail)*", + "*pytest.fixture*", + "*def fail*", + "*fixture*'missing'*not found*", + ] ) + # ensemble: the subject is an exception raised while *importing* the test + # module; ensembles serve a preset module object and never import. def test_factory_setup_as_classes_fails(self, pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1573,77 +1572,70 @@ def __init__(self, request): values = reprec.getfailedcollections() assert len(values) == 1 - def test_usefixtures_marker(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_usefixtures_marker(self, tmp_path: Path) -> None: + values: list[int] = [] - values = [] + @pytest.fixture(scope="class") + def myfix(request): + request.cls.hello = "world" + values.append(1) - @pytest.fixture(scope="class") - def myfix(request): - request.cls.hello = "world" - values.append(1) + class TestClass: + hello: str # set by the ``myfix`` fixture - class TestClass(object): - def test_one(self): - assert self.hello == "world" - assert len(values) == 1 - def test_two(self): - assert self.hello == "world" - assert len(values) == 1 - pytest.mark.usefixtures("myfix")(TestClass) - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + def test_one(self): + assert self.hello == "world" + assert len(values) == 1 + + def test_two(self): + assert self.hello == "world" + assert len(values) == 1 + + pytest.mark.usefixtures("myfix")(TestClass) - def test_empty_usefixtures_marker(self, pytester: Pytester) -> None: + record = run_tests(myfix, TestClass, rootpath=tmp_path) + record.assert_outcomes(passed=2) + + def test_empty_usefixtures_marker(self, tmp_path: Path) -> None: """Empty usefixtures() marker issues a warning (#12439).""" - pytester.makepyfile( - """ - import pytest - @pytest.mark.usefixtures() - def test_one(): - assert 1 == 1 - """ + @pytest.mark.usefixtures() + def test_one(): + assert 1 == 1 + + spec = ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["always"]}) + record = run_tests( + build_module("test_empty_usefixtures_marker", test_one), spec=spec ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - "*PytestWarning: usefixtures() in test_empty_usefixtures_marker.py::test_one" + record.assert_outcomes(passed=1, warnings=1) + assert str(record.warnings[0].message) == ( + "usefixtures() in test_empty_usefixtures_marker.py::test_one" " without arguments has no effect" ) - def test_usefixtures_ini(self, pytester: Pytester) -> None: - pytester.makeini( - """ - [pytest] - usefixtures = myfix - """ - ) - pytester.makeconftest( - """ - import pytest - + def test_usefixtures_ini(self, tmp_path: Path) -> None: + class ConftestPlugin: @pytest.fixture(scope="class") - def myfix(request): + def myfix(self, request): request.cls.hello = "world" - """ - ) - pytester.makepyfile( - """ - class TestClass(object): - def test_one(self): - assert self.hello == "world" - def test_two(self): - assert self.hello == "world" - """ + class TestClass: + hello: str # set by the ``myfix`` fixture + + def test_one(self): + assert self.hello == "world" + + def test_two(self): + assert self.hello == "world" + + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={"usefixtures": ["myfix"]}, + extra_plugins=(ConftestPlugin(),), ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + run_tests(TestClass, spec=spec).assert_outcomes(passed=2) + # ensemble: --markers runs through pytest_cmdline_main, unreachable. def test_usefixtures_seen_in_showmarkers(self, pytester: Pytester) -> None: result = pytester.runpytest("--markers") result.stdout.fnmatch_lines( @@ -1652,179 +1644,172 @@ def test_usefixtures_seen_in_showmarkers(self, pytester: Pytester) -> None: """ ) - def test_request_instance_issue203(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_request_instance_issue203(self, tmp_path: Path) -> None: + class TestClass: + @pytest.fixture + def setup1(self, request): + assert self == request.instance + self.arg1 = 1 - class TestClass(object): - @pytest.fixture - def setup1(self, request): - assert self == request.instance - self.arg1 = 1 - def test_hello(self, setup1): - assert self.arg1 == 1 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + def test_hello(self, setup1): + assert self.arg1 == 1 - def test_fixture_parametrized_with_iterator(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + run_tests(TestClass, rootpath=tmp_path).assert_outcomes(passed=1) - values = [] - def f(): - yield 1 - yield 2 - dec = pytest.fixture(scope="module", params=f()) + def test_fixture_parametrized_with_iterator(self, tmp_path: Path) -> None: + values: list[int] = [] - @dec - def arg(request): - return request.param - @dec - def arg2(request): - return request.param + def f(): + yield 1 + yield 2 - def test_1(arg): - values.append(arg) - def test_2(arg2): - values.append(arg2*10) - """ - ) - reprec = pytester.inline_run("-v") - reprec.assertoutcome(passed=4) - values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + dec = pytest.fixture(scope="module", params=f()) + + @dec + def arg(request): + return request.param + + @dec + def arg2(request): + return request.param + + def test_1(arg): + values.append(arg) + + def test_2(arg2): + values.append(arg2 * 10) + + record = run_tests(arg, arg2, test_1, test_2, rootpath=tmp_path) + record.assert_outcomes(passed=4) assert values == [1, 2, 10, 20] - def test_setup_functions_as_fixtures(self, pytester: Pytester) -> None: + def test_setup_functions_as_fixtures(self, tmp_path: Path) -> None: """Ensure setup_* methods obey fixture scope rules (#517, #3094).""" - pytester.makepyfile( - """ - import pytest - - DB_INITIALIZED = None + # The original's module global becomes a one-element list, since a + # source function would see the host module's globals. + db_initialized: list[bool | None] = [None] - @pytest.fixture(scope="session", autouse=True) - def db(): - global DB_INITIALIZED - DB_INITIALIZED = True - yield - DB_INITIALIZED = False + @pytest.fixture(scope="session", autouse=True) + def db(): + db_initialized[0] = True + yield + db_initialized[0] = False - def setup_module(): - assert DB_INITIALIZED + def setup_module(): + assert db_initialized[0] - def teardown_module(): - assert DB_INITIALIZED + def teardown_module(): + assert db_initialized[0] - class TestClass(object): + class TestClass: + def setup_method(self, method): + assert db_initialized[0] - def setup_method(self, method): - assert DB_INITIALIZED + def teardown_method(self, method): + assert db_initialized[0] - def teardown_method(self, method): - assert DB_INITIALIZED + def test_printer_1(self): + pass - def test_printer_1(self): - pass + def test_printer_2(self): + pass - def test_printer_2(self): - pass - """ + module = build_module( + "test_setup_functions_as_fixtures", + db, + TestClass, + setup_module=setup_module, + teardown_module=teardown_module, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 2 passed in *"]) + run_tests(module, rootpath=tmp_path).assert_outcomes(passed=2) - def test_parameterized_fixture_caching(self, pytester: Pytester) -> None: + def test_parameterized_fixture_caching(self, tmp_path: Path) -> None: """Regression test for #12600.""" - pytester.makepyfile( - """ - import pytest - from itertools import count + from itertools import count + + cache_misses = count(0) - CACHE_MISSES = count(0) + def pytest_generate_tests(metafunc): + if "my_fixture" in metafunc.fixturenames: + # Use unique objects for parametrization (as opposed to small strings + # and small integers which are singletons). + metafunc.parametrize("my_fixture", [[1], [2]], indirect=True) - def pytest_generate_tests(metafunc): - if "my_fixture" in metafunc.fixturenames: - # Use unique objects for parametrization (as opposed to small strings - # and small integers which are singletons). - metafunc.parametrize("my_fixture", [[1], [2]], indirect=True) + @pytest.fixture(scope="session") + def my_fixture(request): + next(cache_misses) - @pytest.fixture(scope='session') - def my_fixture(request): - next(CACHE_MISSES) + def test1(my_fixture): + pass - def test1(my_fixture): - pass + def test2(my_fixture): + pass - def test2(my_fixture): - pass + def teardown_module(): + assert next(cache_misses) == 2 - def teardown_module(): - assert next(CACHE_MISSES) == 2 - """ + module = build_module( + "test_parameterized_fixture_caching", + my_fixture, + test1, + test2, + pytest_generate_tests=pytest_generate_tests, + teardown_module=teardown_module, ) - result = pytester.runpytest() - result.stdout.no_fnmatch_line("* ERROR at teardown *") + # A failing teardown_module would show up as an error, so asserting + # the exact outcomes subsumes the original's "no ERROR at teardown". + run_tests(module, rootpath=tmp_path).assert_outcomes(passed=4) - def test_unwrapping_pytest_fixture(self, pytester: Pytester) -> None: + def test_unwrapping_pytest_fixture(self, tmp_path: Path) -> None: """Ensure the unwrap method on `FixtureFunctionDefinition` correctly wraps and unwraps methods and functions""" - pytester.makepyfile( - """ - import pytest - import inspect - - class FixtureFunctionDefTestClass: - def __init__(self) -> None: - self.i = 10 - - @pytest.fixture - def fixture_function_def_test_method(self): - return self.i + import inspect + class FixtureFunctionDefTestClass: + def __init__(self) -> None: + self.i = 10 @pytest.fixture - def fixture_function_def_test_func(): - return 9 + def fixture_function_def_test_method(self): + return self.i + @pytest.fixture + def fixture_function_def_test_func(): + return 9 - def test_get_wrapped_func_returns_method(): - obj = FixtureFunctionDefTestClass() - wrapped_function_result = ( - obj.fixture_function_def_test_method._get_wrapped_function() - ) - assert inspect.ismethod(wrapped_function_result) - assert wrapped_function_result() == 10 + def test_get_wrapped_func_returns_method(): + obj = FixtureFunctionDefTestClass() + wrapped_function_result = ( + obj.fixture_function_def_test_method._get_wrapped_function() + ) + assert inspect.ismethod(wrapped_function_result) + assert wrapped_function_result() == 10 + def test_get_wrapped_func_returns_function(): + assert fixture_function_def_test_func._get_wrapped_function()() == 9 - def test_get_wrapped_func_returns_function(): - assert fixture_function_def_test_func._get_wrapped_function()() == 9 - """ + record = run_tests( + test_get_wrapped_func_returns_method, + test_get_wrapped_func_returns_function, + rootpath=tmp_path, ) - result = pytester.runpytest() - result.assert_outcomes(passed=2) + record.assert_outcomes(passed=2) - def test_fixture_wrapped_looks_liked_wrapped_function( - self, pytester: Pytester - ) -> None: + def test_fixture_wrapped_looks_liked_wrapped_function(self, tmp_path: Path) -> None: """Ensure that `FixtureFunctionDefinition` behaves like the function it wrapped.""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def fixture_function_def_test_func(): - return 9 - fixture_function_def_test_func.__doc__ = "documentation" + @pytest.fixture + def fixture_function_def_test_func(): + return 9 - def test_fixture_has_same_doc(): - assert fixture_function_def_test_func.__doc__ == "documentation" - """ + fixture_function_def_test_func.__doc__ = "documentation" + + def test_fixture_has_same_doc(): + assert fixture_function_def_test_func.__doc__ == "documentation" + + record = run_tests( + fixture_function_def_test_func, test_fixture_has_same_doc, rootpath=tmp_path ) - result = pytester.runpytest() - result.assert_outcomes(passed=1) + record.assert_outcomes(passed=1) class TestFixtureManagerParseFactories: @@ -1849,95 +1834,114 @@ def item(request): ) return pytester - def test_parsefactories_evil_objects_issue214(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - class A(object): - def __call__(self): - pass - def __getattr__(self, name): - raise RuntimeError() - a = A() - def test_hello(): + @pytest.fixture + def spec(self, tmp_path: Path) -> ConfigSpec: + """The rootdir conftest of this class, as a plugin object.""" + + class ConftestPlugin: + @pytest.fixture + def hello(self, request): + return "conftest" + + @pytest.fixture + def fm(self, request): + return request._fixturemanager + + @pytest.fixture + def item(self, request): + return request._pyfuncitem + + return ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + + def test_parsefactories_evil_objects_issue214(self, spec: ConfigSpec) -> None: + class A: + def __call__(self): pass - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1, failed=0) - def test_parsefactories_conftest(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_hello(item, fm): - for name in ("fm", "hello", "item"): - faclist = fm.getfixturedefs(name, item) - assert len(faclist) == 1 - fac = faclist[0] - assert fac.func.__name__ == name - """ - ) - reprec = pytester.inline_run("-s") - reprec.assertoutcome(passed=1) + def __getattr__(self, name): + raise RuntimeError() + + def test_hello(): + pass + + record = run_tests(build_module("test_evil", test_hello, a=A()), spec=spec) + record.assert_outcomes(passed=1, failed=0) + + def test_parsefactories_conftest(self, spec: ConfigSpec) -> None: + def test_hello(item, fm): + for name in ("fm", "hello", "item"): + faclist = fm.getfixturedefs(name, item) + assert len(faclist) == 1 + fac = faclist[0] + assert fac.func.__name__ == name + + run_tests(test_hello, spec=spec).assert_outcomes(passed=1) def test_parsefactories_conftest_and_module_and_class( - self, pytester: Pytester + self, spec: ConfigSpec ) -> None: - pytester.makepyfile( - """\ - import pytest + @pytest.fixture + def hello(request): + return "module" + class TestClass: @pytest.fixture - def hello(request): - return "module" - class TestClass(object): - @pytest.fixture - def hello(self, request): - return "class" - def test_hello(self, item, fm): - faclist = fm.getfixturedefs("hello", item) - print(faclist) - assert len(faclist) == 3 - - assert faclist[0].func(item._request) == "conftest" - assert faclist[1].func(item._request) == "module" - assert faclist[2].func(item._request) == "class" - """ + def hello(self, request): + return "class" + + def test_hello(self, item, fm): + faclist = fm.getfixturedefs("hello", item) + print(faclist) + assert len(faclist) == 3 + + assert faclist[0].func(item._request) == "conftest" + assert faclist[1].func(item._request) == "module" + assert faclist[2].func(item._request) == "class" + + record = run_tests( + build_module("test_three_levels", TestClass, hello=hello), spec=spec ) - reprec = pytester.inline_run("-s") - reprec.assertoutcome(passed=1) + record.assert_outcomes(passed=1) - def test_register_fixture_ordered_by_visibility(self, pytester: Pytester) -> None: + def test_register_fixture_ordered_by_visibility(self, tmp_path: Path) -> None: """A fixturedef registered for a more specific node takes precedence over one registered for a more general (ancestor) node, regardless of the order in which they were registered (#14513).""" - pytester.makeconftest( - """ - import pytest + class ConftestPlugin: @pytest.hookimpl(wrapper=True) - def pytest_collection(session): + def pytest_collection(self, session): result = yield item = session.items[0] - pytest.register_fixture(name="fix", func=lambda: "session1", node=session) + pytest.register_fixture( + name="fix", func=lambda: "session1", node=session + ) # For coverage; can be removed once nodeid= deprecation is over. fm = session._fixturemanager - fm._register_fixture(name="fix", func=lambda: "session-legacy", nodeid="") - fm._register_fixture(name="fix", func=lambda: "broken-legacy", nodeid="broken") - pytest.register_fixture(name="fix", func=lambda fix: f"item1-{fix}", node=item) - pytest.register_fixture(name="fix", func=lambda fix: f"item2-{fix}", node=item) - pytest.register_fixture(name="fix", func=lambda: "session2", node=session) + fm._register_fixture( + name="fix", func=lambda: "session-legacy", nodeid="" + ) + fm._register_fixture( + name="fix", func=lambda: "broken-legacy", nodeid="broken" + ) + pytest.register_fixture( + name="fix", func=lambda fix: f"item1-{fix}", node=item + ) + pytest.register_fixture( + name="fix", func=lambda fix: f"item2-{fix}", node=item + ) + pytest.register_fixture( + name="fix", func=lambda: "session2", node=session + ) return result - """ - ) - pytester.makepyfile( - """ - def test(fix): - assert fix == "item2-item1-session2" - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + def test(fix): + assert fix == "item2-item1-session2" + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + run_tests(test, spec=spec).assert_outcomes(passed=1) + + # ensemble: conftests in sibling directories, run from a third one. def test_parsefactories_relative_node_ids( self, pytester: Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -1995,6 +1999,7 @@ def test_x(one): reprec = pytester.inline_run("..") reprec.assertoutcome(passed=2) + # ensemble: package layout (__init__.py, relative imports). def test_package_xunit_fixture(self, pytester: Pytester) -> None: pytester.makepyfile( __init__="""\ @@ -2050,6 +2055,7 @@ def test_x(): reprec = pytester.inline_run() reprec.assertoutcome(passed=2) + # ensemble: package layout and package-scoped fixtures. def test_package_fixture_complex(self, pytester: Pytester) -> None: pytester.makepyfile( __init__="""\ @@ -2093,6 +2099,8 @@ def test_package(one): reprec = pytester.inline_run() reprec.assertoutcome(passed=2) + # ensemble: the example is a directory tree with a conftest defining + # custom collectors for non-python files. def test_collect_custom_items(self, pytester: Pytester) -> None: pytester.copy_example("fixtures/custom_item") result = pytester.runpytest("foo") @@ -2127,60 +2135,83 @@ def item(request): ) return pytester - def test_parsefactories_conftest(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - from _pytest.pytester import get_public_names - def test_check_setup(item, fm): - autousenames = list(fm._getautousenames(item)) - assert len(get_public_names(autousenames)) == 2 - assert "perfunction2" in autousenames - assert "perfunction" in autousenames - """ - ) - reprec = pytester.inline_run("-s") - reprec.assertoutcome(passed=1) + @pytest.fixture + def spec(self, tmp_path: Path) -> ConfigSpec: + """The rootdir conftest of this class, as a plugin object.""" - def test_two_classes_separated_autouse(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - class TestA(object): - values = [] - @pytest.fixture(autouse=True) - def setup1(self): - self.values.append(1) - def test_setup1(self): - assert self.values == [1] - class TestB(object): - values = [] - @pytest.fixture(autouse=True) - def setup2(self): - self.values.append(1) - def test_setup2(self): - assert self.values == [1] - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + class ConftestPlugin: + @pytest.fixture(autouse=True) + def perfunction(self, request, tmp_path): + pass - def test_setup_at_classlevel(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - class TestClass(object): - @pytest.fixture(autouse=True) - def permethod(self, request): - request.instance.funcname = request.function.__name__ - def test_method1(self): - assert self.funcname == "test_method1" - def test_method2(self): - assert self.funcname == "test_method2" - """ - ) - reprec = pytester.inline_run("-s") - reprec.assertoutcome(passed=2) + @pytest.fixture + def arg1(self, tmp_path): + pass + + @pytest.fixture(autouse=True) + def perfunction2(self, arg1): + pass + + @pytest.fixture + def fm(self, request): + return request._fixturemanager + + @pytest.fixture + def item(self, request): + return request._pyfuncitem + + return ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + + def test_parsefactories_conftest(self, spec: ConfigSpec) -> None: + def test_check_setup(item, fm): + autousenames = list(fm._getautousenames(item)) + assert len(get_public_names(autousenames)) == 2 + assert "perfunction2" in autousenames + assert "perfunction" in autousenames + + run_tests(test_check_setup, spec=spec).assert_outcomes(passed=1) + + def test_two_classes_separated_autouse(self, tmp_path: Path) -> None: + class TestA: + values: list[int] = [] + + @pytest.fixture(autouse=True) + def setup1(self): + self.values.append(1) + + def test_setup1(self): + assert self.values == [1] + class TestB: + values: list[int] = [] + + @pytest.fixture(autouse=True) + def setup2(self): + self.values.append(1) + + def test_setup2(self): + assert self.values == [1] + + run_tests(TestA, TestB, rootpath=tmp_path).assert_outcomes(passed=2) + + def test_setup_at_classlevel(self, tmp_path: Path) -> None: + class TestClass: + funcname: str # set by the ``permethod`` fixture + + @pytest.fixture(autouse=True) + def permethod(self, request): + request.instance.funcname = request.function.__name__ + + def test_method1(self): + assert self.funcname == "test_method1" + + def test_method2(self): + assert self.funcname == "test_method2" + + run_tests(TestClass, rootpath=tmp_path).assert_outcomes(passed=2) + + # ensemble: `pytest.fixture(enabled=...)` is rejected at decoration time + # by the host, so the unimplemented feature cannot be spelled inline. @pytest.mark.xfail(reason="'enabled' feature not implemented") def test_setup_enabled_functionnode(self, pytester: Pytester) -> None: pytester.makepyfile( @@ -2209,23 +2240,28 @@ def test_func2(request): reprec = pytester.inline_run("-s") reprec.assertoutcome(passed=2) - def test_callables_nocode(self, pytester: Pytester) -> None: + def test_callables_nocode(self, tmp_path: Path) -> None: """An imported mock.call would break setup/factory discovery due to it being callable and __code__ not being a code object.""" - pytester.makepyfile( - """ - class _call(tuple): - def __call__(self, *k, **kw): - pass - def __getattr__(self, k): - return self - call = _call() - """ + class _call(tuple[object, ...]): + def __call__(self, *k, **kw): + pass + + def __getattr__(self, k): + return self + + # collect_tests raises rather than returning an empty list if + # collection blew up, so this really means "collected nothing". + assert ( + collect_tests( + build_module("test_callables_nocode", call=_call()), rootpath=tmp_path + ) + == [] ) - reprec = pytester.inline_run("-s") - reprec.assertoutcome(failed=0, passed=0) + # ensemble: an autouse fixture in one subdirectory's conftest must not + # reach a sibling directory - that is directory scoping. def test_autouse_in_conftests(self, pytester: Pytester) -> None: a = pytester.mkdir("a") b = pytester.mkdir("a1") @@ -2251,33 +2287,34 @@ def hello(): """ ) - def test_autouse_in_module_and_two_classes(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [] + def test_autouse_in_module_and_two_classes(self, tmp_path: Path) -> None: + values: list[str] = [] + + @pytest.fixture(autouse=True) + def append1(): + values.append("module") + + def test_x(): + assert values == ["module"] + + class TestA: @pytest.fixture(autouse=True) - def append1(): - values.append("module") - def test_x(): - assert values == ["module"] + def append2(self): + values.append("A") - class TestA(object): - @pytest.fixture(autouse=True) - def append2(self): - values.append("A") - def test_hello(self): - assert values == ["module", "module", "A"], values - class TestA2(object): - def test_world(self): - assert values == ["module", "module", "A", "module"], values - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=3) + def test_hello(self): + assert values == ["module", "module", "A"], values + + class TestA2: + def test_world(self): + assert values == ["module", "module", "A", "module"], values + + record = run_tests(append1, test_x, TestA, TestA2, rootpath=tmp_path) + record.assert_outcomes(passed=3) class TestAutouseManagement: + # ensemble: the conftest sits in a directory between rootdir and the test. def test_autouse_conftest_mid_directory(self, pytester: Pytester) -> None: pkgdir = pytester.mkpydir("xyz123") pkgdir.joinpath("conftest.py").write_text( @@ -2309,542 +2346,523 @@ def test_app(): reprec = pytester.inline_run("-s") reprec.assertoutcome(passed=1) - def test_funcarg_and_setup(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [] - @pytest.fixture(scope="module") - def arg(): - values.append(1) - return 0 - @pytest.fixture(scope="module", autouse=True) - def something(arg): - values.append(2) + def test_funcarg_and_setup(self, tmp_path: Path) -> None: + values: list[int] = [] - def test_hello(arg): - assert len(values) == 2 - assert values == [1,2] - assert arg == 0 + @pytest.fixture(scope="module") + def arg(): + values.append(1) + return 0 - def test_hello2(arg): - assert len(values) == 2 - assert values == [1,2] - assert arg == 0 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + @pytest.fixture(scope="module", autouse=True) + def something(arg): + values.append(2) - def test_uses_parametrized_resource(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [] - @pytest.fixture(params=[1,2]) - def arg(request): - return request.param + def test_hello(arg): + assert len(values) == 2 + assert values == [1, 2] + assert arg == 0 - @pytest.fixture(autouse=True) - def something(arg): - values.append(arg) + def test_hello2(arg): + assert len(values) == 2 + assert values == [1, 2] + assert arg == 0 - def test_hello(): - if len(values) == 1: - assert values == [1] - elif len(values) == 2: - assert values == [1, 2] - else: - 0/0 + record = run_tests(arg, something, test_hello, test_hello2, rootpath=tmp_path) + record.assert_outcomes(passed=2) - """ - ) - reprec = pytester.inline_run("-s") - reprec.assertoutcome(passed=2) + def test_uses_parametrized_resource(self, tmp_path: Path) -> None: + values: list[int] = [] - def test_session_parametrized_function(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + @pytest.fixture(params=[1, 2]) + def arg(request): + return request.param - values = [] + @pytest.fixture(autouse=True) + def something(arg): + values.append(arg) - @pytest.fixture(scope="session", params=[1,2]) - def arg(request): - return request.param + def test_hello(): + if len(values) == 1: + assert values == [1] + elif len(values) == 2: + assert values == [1, 2] + else: + 0 / 0 # noqa: B018 - @pytest.fixture(scope="function", autouse=True) - def append(request, arg): - if request.function.__name__ == "test_some": - values.append(arg) + record = run_tests(arg, something, test_hello, rootpath=tmp_path) + record.assert_outcomes(passed=2) - def test_some(): - pass + def test_session_parametrized_function(self, tmp_path: Path) -> None: + values: list[int] = [] - def test_result(arg): - assert len(values) == arg - assert values[:arg] == [1,2][:arg] - """ - ) - reprec = pytester.inline_run("-v", "-s") - reprec.assertoutcome(passed=4) + @pytest.fixture(scope="session", params=[1, 2]) + def arg(request): + return request.param - def test_class_function_parametrization_finalization( - self, pytester: Pytester - ) -> None: - p = pytester.makeconftest( - """ - import pytest - import pprint + @pytest.fixture(scope="function", autouse=True) + def append(request, arg): + if request.function.__name__ == "test_some": + values.append(arg) - values = [] + def test_some(): + pass + + def test_result(arg): + assert len(values) == arg + assert values[:arg] == [1, 2][:arg] + + record = run_tests(arg, append, test_some, test_result, rootpath=tmp_path) + record.assert_outcomes(passed=4) + + def test_class_function_parametrization_finalization(self, tmp_path: Path) -> None: + values: list[str] = [] - @pytest.fixture(scope="function", params=[1,2]) - def farg(request): + class ConftestPlugin: + @pytest.fixture(scope="function", params=[1, 2]) + def farg(self, request): return request.param @pytest.fixture(scope="class", params=list("ab")) - def carg(request): + def carg(self, request): return request.param @pytest.fixture(scope="function", autouse=True) - def append(request, farg, carg): + def append(self, request, farg, carg): def fin(): - values.append("fin_%s%s" % (carg, farg)) + values.append(f"fin_{carg}{farg}") + request.addfinalizer(fin) - """ - ) - pytester.makepyfile( - """ - import pytest - class TestClass(object): - def test_1(self): - pass - class TestClass2(object): - def test_2(self): - pass - """ - ) - reprec = pytester.inline_run("-v", "-s", "--confcutdir", pytester.path) - reprec.assertoutcome(passed=8) - config = reprec.getcalls("pytest_unconfigure")[0].config - values = config.pluginmanager._getconftestmodules(p)[0].values + class TestClass: + def test_1(self): + pass + + class TestClass2: + def test_2(self): + pass + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(TestClass, TestClass2, spec=spec) + record.assert_outcomes(passed=8) assert values == ["fin_a1", "fin_a2", "fin_b1", "fin_b2"] * 2 - def test_scope_ordering(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [] - @pytest.fixture(scope="function", autouse=True) - def fappend2(): - values.append(2) - @pytest.fixture(scope="class", autouse=True) - def classappend3(): - values.append(3) - @pytest.fixture(scope="module", autouse=True) - def mappend(): - values.append(1) + def test_scope_ordering(self, tmp_path: Path) -> None: + values: list[int] = [] - class TestHallo(object): - def test_method(self): - assert values == [1,3,2] - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + @pytest.fixture(scope="function", autouse=True) + def fappend2(): + values.append(2) - def test_parametrization_setup_teardown_ordering(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + @pytest.fixture(scope="class", autouse=True) + def classappend3(): + values.append(3) - values = [] + @pytest.fixture(scope="module", autouse=True) + def mappend(): + values.append(1) - def pytest_generate_tests(metafunc): - if metafunc.cls is None: - assert metafunc.function is test_finish - if metafunc.cls is not None: - metafunc.parametrize("item", [1,2], scope="class") - - class TestClass: - @pytest.fixture(scope="class", autouse=True) - @classmethod - def setup_teardown(cls, item): - values.append("setup-%d" % item) - yield - values.append("teardown-%d" % item) - - def test_step1(self, item): - values.append("step1-%d" % item) - - def test_step2(self, item): - values.append("step2-%d" % item) - - def test_finish(): - assert values == [ - "setup-1", - "step1-1", - "step2-1", - "teardown-1", - "setup-2", - "step1-2", - "step2-2", - "teardown-2", - ] - """ + class TestHallo: + def test_method(self): + assert values == [1, 3, 2] + + record = run_tests( + fappend2, classappend3, mappend, TestHallo, rootpath=tmp_path ) - result = pytester.inline_run("-vv") - result.assertoutcome(passed=5) + record.assert_outcomes(passed=1) - def test_ordering_autouse_before_explicit(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrization_setup_teardown_ordering(self, tmp_path: Path) -> None: + values: list[str] = [] - values = [] - @pytest.fixture(autouse=True) - def fix1(): - values.append(1) - @pytest.fixture() - def arg1(): - values.append(2) - def test_hello(arg1): - assert values == [1,2] - """ + def pytest_generate_tests(metafunc): + if metafunc.cls is None: + assert metafunc.function is test_finish + if metafunc.cls is not None: + metafunc.parametrize("item", [1, 2], scope="class") + + class TestClass: + @pytest.fixture(scope="class", autouse=True) + @classmethod + def setup_teardown(cls, item): + values.append(f"setup-{item}") + yield + values.append(f"teardown-{item}") + + def test_step1(self, item): + values.append(f"step1-{item}") + + def test_step2(self, item): + values.append(f"step2-{item}") + + def test_finish(): + assert values == [ + "setup-1", + "step1-1", + "step2-1", + "teardown-1", + "setup-2", + "step1-2", + "step2-2", + "teardown-2", + ] + + module = build_module( + "test_setup_teardown_ordering", + TestClass, + test_finish, + pytest_generate_tests=pytest_generate_tests, ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + run_tests(module, rootpath=tmp_path).assert_outcomes(passed=5) + + def test_ordering_autouse_before_explicit(self, tmp_path: Path) -> None: + values: list[int] = [] - @pytest.mark.parametrize("param1", ["", "params=[1]"], ids=["p00", "p01"]) - @pytest.mark.parametrize("param2", ["", "params=[1]"], ids=["p10", "p11"]) + @pytest.fixture(autouse=True) + def fix1(): + values.append(1) + + @pytest.fixture + def arg1(): + values.append(2) + + def test_hello(arg1): + assert values == [1, 2] + + run_tests(fix1, arg1, test_hello, rootpath=tmp_path).assert_outcomes(passed=1) + + @pytest.mark.parametrize("param1", [{}, {"params": [1]}], ids=["p00", "p01"]) + @pytest.mark.parametrize("param2", [{}, {"params": [1]}], ids=["p10", "p11"]) def test_ordering_dependencies_torndown_first( - self, pytester: Pytester, param1, param2 + self, tmp_path: Path, param1, param2 ) -> None: """#226""" - pytester.makepyfile( - f""" - import pytest - values = [] - @pytest.fixture({param1}) - def arg1(request): - request.addfinalizer(lambda: values.append("fin1")) - values.append("new1") - @pytest.fixture({param2}) - def arg2(request, arg1): - request.addfinalizer(lambda: values.append("fin2")) - values.append("new2") - - def test_arg(arg2): - pass - def test_check(): - assert values == ["new1", "new2", "fin2", "fin1"] - """ - ) - reprec = pytester.inline_run("-s") - reprec.assertoutcome(passed=2) + values: list[str] = [] + + @pytest.fixture(**param1) + def arg1(request): + request.addfinalizer(lambda: values.append("fin1")) + values.append("new1") + + @pytest.fixture(**param2) + def arg2(request, arg1): + request.addfinalizer(lambda: values.append("fin2")) + values.append("new2") + + def test_arg(arg2): + pass + + def test_check(): + assert values == ["new1", "new2", "fin2", "fin1"] - def test_reordering_catastrophic_performance(self, pytester: Pytester) -> None: + record = run_tests(arg1, arg2, test_arg, test_check, rootpath=tmp_path) + record.assert_outcomes(passed=2) + + def test_reordering_catastrophic_performance(self, tmp_path: Path) -> None: """Check that a certain high-scope parametrization pattern doesn't cause a catasrophic slowdown. Regression test for #12355. """ - pytester.makepyfile(""" - import pytest + params = tuple("abcdefghijklmnopqrstuvwxyz") + + @pytest.mark.parametrize(params, [range(len(params))] * 3, scope="module") + def test_parametrize( + a, + b, + c, + d, + e, + f, + g, + h, + i, + j, + k, + l, # noqa: E741 + m, + n, + o, + p, + q, + r, + s, + t, + u, + v, + w, + x, + y, + z, + ): + pass - params = tuple("abcdefghijklmnopqrstuvwxyz") - @pytest.mark.parametrize(params, [range(len(params))] * 3, scope="module") - def test_parametrize(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z): - pass - """) + run_tests(test_parametrize, rootpath=tmp_path).assert_outcomes(passed=3) - result = pytester.runpytest() - result.assert_outcomes(passed=3) +class TestFixtureMarker: + def test_parametrize(self, tmp_path: Path) -> None: + values: list[str] = [] + @pytest.fixture(params=["a", "b", "c"]) + def arg(request): + return request.param -class TestFixtureMarker: - def test_parametrize(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=["a", "b", "c"]) - def arg(request): - return request.param - values = [] - def test_param(arg): - values.append(arg) - def test_result(): - assert values == list("abc") - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=4) + def test_param(arg): + values.append(arg) - def test_multiple_parametrization_issue_736(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_result(): + assert values == list("abc") - @pytest.fixture(params=[1,2,3]) - def foo(request): - return request.param + record = run_tests(arg, test_param, test_result, rootpath=tmp_path) + record.assert_outcomes(passed=4) - @pytest.mark.parametrize('foobar', [4,5,6]) - def test_issue(foo, foobar): - assert foo in [1,2,3] - assert foobar in [4,5,6] - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=9) + def test_multiple_parametrization_issue_736(self, tmp_path: Path) -> None: + @pytest.fixture(params=[1, 2, 3]) + def foo(request): + return request.param + + @pytest.mark.parametrize("foobar", [4, 5, 6]) + def test_issue(foo, foobar): + assert foo in [1, 2, 3] + assert foobar in [4, 5, 6] + + run_tests(foo, test_issue, rootpath=tmp_path).assert_outcomes(passed=9) @pytest.mark.parametrize( "param_args", - ["'fixt, val'", "'fixt,val'", "['fixt', 'val']", "('fixt', 'val')"], + ["fixt, val", "fixt,val", ["fixt", "val"], ("fixt", "val")], ) def test_override_parametrized_fixture_issue_979( - self, pytester: Pytester, param_args + self, tmp_path: Path, param_args ) -> None: """Make sure a parametrized argument can override a parametrized fixture. This was a regression introduced in the fix for #736. """ - pytester.makepyfile( - f""" - import pytest - @pytest.fixture(params=[1, 2]) - def fixt(request): - return request.param + @pytest.fixture(params=[1, 2]) + def fixt(request): + return request.param - @pytest.mark.parametrize({param_args}, [(3, 'x'), (4, 'x')]) - def test_foo(fixt, val): - pass - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + @pytest.mark.parametrize(param_args, [(3, "x"), (4, "x")]) + def test_foo(fixt, val): + pass - def test_override_parametrized_fixture_with_indirect( - self, pytester: Pytester - ) -> None: + run_tests(fixt, test_foo, rootpath=tmp_path).assert_outcomes(passed=2) + + def test_override_parametrized_fixture_with_indirect(self, tmp_path: Path) -> None: """Make sure a parametrized argument can override a parametrized fixture. This was a regression introduced in the fix for #736. """ - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=["a"]) - def fixt(request): - return request.param * 2 + @pytest.fixture(params=["a"]) + def fixt(request): + return request.param * 2 - def test_fixt(fixt): - assert fixt == "aa" + def test_fixt(fixt): + assert fixt == "aa" - @pytest.mark.parametrize("fixt", ['b'], indirect=True) - def test_indirect(fixt): - assert fixt == "bb" - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + @pytest.mark.parametrize("fixt", ["b"], indirect=True) + def test_indirect(fixt): + assert fixt == "bb" - def test_scope_session(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [] - @pytest.fixture(scope="module") - def arg(): - values.append(1) - return 1 + record = run_tests(fixt, test_fixt, test_indirect, rootpath=tmp_path) + record.assert_outcomes(passed=2) - def test_1(arg): - assert arg == 1 - def test_2(arg): + def test_scope_session(self, tmp_path: Path) -> None: + values: list[int] = [] + + @pytest.fixture(scope="module") + def arg(): + values.append(1) + return 1 + + def test_1(arg): + assert arg == 1 + + def test_2(arg): + assert arg == 1 + assert len(values) == 1 + + class TestClass: + def test3(self, arg): assert arg == 1 assert len(values) == 1 - class TestClass(object): - def test3(self, arg): - assert arg == 1 - assert len(values) == 1 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=3) - def test_scope_session_exc(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [] - @pytest.fixture(scope="session") - def fix(): - values.append(1) - pytest.skip('skipping') + record = run_tests(arg, test_1, test_2, TestClass, rootpath=tmp_path) + record.assert_outcomes(passed=3) - def test_1(fix): - pass - def test_2(fix): - pass - def test_last(): - assert values == [1] - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(skipped=2, passed=1) + def test_scope_session_exc(self, tmp_path: Path) -> None: + values: list[int] = [] - def test_scope_session_exc_two_fix(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [] - m = [] - @pytest.fixture(scope="session") - def a(): - values.append(1) - pytest.skip('skipping') + @pytest.fixture(scope="session") + def fix(): + values.append(1) + pytest.skip("skipping") + + def test_1(fix): + pass + + def test_2(fix): + pass + + def test_last(): + assert values == [1] + + record = run_tests(fix, test_1, test_2, test_last, rootpath=tmp_path) + record.assert_outcomes(skipped=2, passed=1) + + def test_scope_session_exc_two_fix(self, tmp_path: Path) -> None: + values: list[int] = [] + m: list[int] = [] + + @pytest.fixture(scope="session") + def a(): + values.append(1) + pytest.skip("skipping") + + @pytest.fixture(scope="session") + def b(a): + m.append(1) + + def test_1(b): + pass + + def test_2(b): + pass + + def test_last(): + assert values == [1] + assert m == [] + + record = run_tests(a, b, test_1, test_2, test_last, rootpath=tmp_path) + record.assert_outcomes(skipped=2, passed=1) + + def test_scope_exc(self, tmp_path: Path) -> None: + reqs: list[int] = [] + + class ConftestPlugin: @pytest.fixture(scope="session") - def b(a): - m.append(1) + def fix(self, request): + reqs.append(1) + pytest.skip() - def test_1(b): - pass - def test_2(b): - pass - def test_last(): - assert values == [1] - assert m == [] - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(skipped=2, passed=1) + @pytest.fixture + def req_list(self): + return reqs - def test_scope_exc(self, pytester: Pytester) -> None: - pytester.makepyfile( - test_foo=""" - def test_foo(fix): - pass - """, - test_bar=""" - def test_bar(fix): - pass - """, - conftest=""" - import pytest - reqs = [] - @pytest.fixture(scope="session") - def fix(request): - reqs.append(1) - pytest.skip() - @pytest.fixture - def req_list(): - return reqs - """, - test_real=""" - def test_last(req_list): - assert req_list == [1] - """, + def test_foo(fix): + pass + + def test_bar(fix): + pass + + def test_last(req_list): + assert req_list == [1] + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_foo", test_foo), + build_module("test_bar", test_bar), + build_module("test_real", test_last), + spec=spec, ) - reprec = pytester.inline_run() - reprec.assertoutcome(skipped=2, passed=1) + record.assert_outcomes(skipped=2, passed=1) - def test_scope_module_uses_session(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [] - @pytest.fixture(scope="module") - def arg(): - values.append(1) - return 1 + def test_scope_module_uses_session(self, tmp_path: Path) -> None: + values: list[int] = [] - def test_1(arg): - assert arg == 1 - def test_2(arg): + @pytest.fixture(scope="module") + def arg(): + values.append(1) + return 1 + + def test_1(arg): + assert arg == 1 + + def test_2(arg): + assert arg == 1 + assert len(values) == 1 + + class TestClass: + def test3(self, arg): assert arg == 1 assert len(values) == 1 - class TestClass(object): - def test3(self, arg): - assert arg == 1 - assert len(values) == 1 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=3) - def test_scope_module_and_finalizer(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - finalized_list = [] - created_list = [] + record = run_tests(arg, test_1, test_2, TestClass, rootpath=tmp_path) + record.assert_outcomes(passed=3) + + def test_scope_module_and_finalizer(self, tmp_path: Path) -> None: + finalized_list: list[int] = [] + created_list: list[int] = [] + + class ConftestPlugin: @pytest.fixture(scope="module") - def arg(request): + def arg(self, request): created_list.append(1) assert request.scope == "module" request.addfinalizer(lambda: finalized_list.append(1)) + @pytest.fixture - def created(request): + def created(self, request): return len(created_list) + @pytest.fixture - def finalized(request): + def finalized(self, request): return len(finalized_list) - """ - ) - pytester.makepyfile( - test_mod1=""" - def test_1(arg, created, finalized): - assert created == 1 - assert finalized == 0 - def test_2(arg, created, finalized): - assert created == 1 - assert finalized == 0""", - test_mod2=""" - def test_3(arg, created, finalized): - assert created == 2 - assert finalized == 1""", - test_mode3=""" - def test_4(arg, created, finalized): - assert created == 3 - assert finalized == 2 - """, + + def test_1(arg, created, finalized): + assert created == 1 + assert finalized == 0 + + def test_2(arg, created, finalized): + assert created == 1 + assert finalized == 0 + + def test_3(arg, created, finalized): + assert created == 2 + assert finalized == 1 + + def test_4(arg, created, finalized): + assert created == 3 + assert finalized == 2 + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_mod1", test_1, test_2), + build_module("test_mod2", test_3), + build_module("test_mode3", test_4), + spec=spec, ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=4) + record.assert_outcomes(passed=4) - def test_scope_mismatch_various(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - finalized = [] - created = [] + def test_scope_mismatch_various(self, tmp_path: Path) -> None: + class ConftestPlugin: @pytest.fixture(scope="function") - def arg(request): + def arg(self, request): pass - """ - ) - pytester.makepyfile( - test_mod1=""" - import pytest - @pytest.fixture(scope="session") - def arg(request): - request.getfixturevalue("arg") - def test_1(arg): - pass - """ + + @pytest.fixture(scope="session") + def arg(request): + request.getfixturevalue("arg") + + def test_1(arg): + pass + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_mod1", arg=arg, test_1=test_1), + spec=spec, + capture_output=True, ) - result = pytester.runpytest() - assert result.ret != 0 - result.stdout.fnmatch_lines( + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( ["*ScopeMismatch*You tried*function*session*request*"] ) + # ensemble: asserts the fixtures' file:line, host-anchored for in-memory + # sources. def test_scope_mismatch_already_computed_dynamic(self, pytester: Pytester) -> None: pytester.makepyfile( test_it=""" @@ -2872,192 +2890,171 @@ def test_it(request, fixfunc): ] ) - def test_dynamic_scope(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest + def test_dynamic_scope(self, tmp_path: Path) -> None: + calls: list[str] = [] + def dynamic_scope(fixture_name, config): + if config.getoption("--extend-scope"): + return "session" + return "function" - def pytest_addoption(parser): + class ConftestPlugin: + def pytest_addoption(self, parser): parser.addoption("--extend-scope", action="store_true", default=False) - - def dynamic_scope(fixture_name, config): - if config.getoption("--extend-scope"): - return "session" - return "function" - - @pytest.fixture(scope=dynamic_scope) - def dynamic_fixture(calls=[]): + def dynamic_fixture(self): calls.append("call") return len(calls) - """ - ) + def test_first(dynamic_fixture): + assert dynamic_fixture == 1 - pytester.makepyfile( - """ - def test_first(dynamic_fixture): - assert dynamic_fixture == 1 + def test_second(dynamic_fixture): + assert dynamic_fixture == 2 + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(test_first, test_second, spec=spec) + record.assert_outcomes(passed=2) + calls.clear() + record = run_tests( + test_first, test_second, spec=spec.replace(args=("--extend-scope",)) + ) + record.assert_outcomes(passed=1, failed=1) - def test_second(dynamic_fixture): - assert dynamic_fixture == 2 + def test_dynamic_scope_bad_return(self, tmp_path: Path) -> None: + def dynamic_scope(**_): + return "wrong-scope" - """ + @pytest.fixture(scope=dynamic_scope) # type: ignore[arg-type] + def fixture(): + pass + + record = run_tests( + build_module("test_dynamic_scope_bad_return", fixture=fixture), + rootpath=tmp_path, + capture_output=True, + ) + record.stdout.fnmatch_lines( + "*Fixture 'fixture' from test_dynamic_scope_bad_return.py " + "got an unexpected scope value 'wrong-scope'*" ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + def test_register_only_with_mark(self, tmp_path: Path) -> None: + class ConftestPlugin: + @pytest.fixture + def arg(self): + return 1 - reprec = pytester.inline_run("--extend-scope") - reprec.assertoutcome(passed=1, failed=1) + @pytest.fixture + def arg(arg): + return arg + 1 - def test_dynamic_scope_bad_return(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_1(arg): + assert arg == 2 - def dynamic_scope(**_): - return "wrong-scope" + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(build_module("test_mod1", arg=arg, test_1=test_1), spec=spec) + record.assert_outcomes(passed=1) - @pytest.fixture(scope=dynamic_scope) - def fixture(): - pass + def test_parametrize_and_scope(self, tmp_path: Path) -> None: + values: list[str] = [] - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - "Fixture 'fixture' from test_dynamic_scope_bad_return.py " - "got an unexpected scope value 'wrong-scope'" - ) + @pytest.fixture(scope="module", params=["a", "b", "c"]) + def arg(request): + return request.param - def test_register_only_with_mark(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - @pytest.fixture() - def arg(): - return 1 - """ - ) - pytester.makepyfile( - test_mod1=""" - import pytest - @pytest.fixture() - def arg(arg): - return arg + 1 - def test_1(arg): - assert arg == 2 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + def test_param(arg): + values.append(arg) - def test_parametrize_and_scope(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope="module", params=["a", "b", "c"]) - def arg(request): - return request.param - values = [] - def test_param(arg): - values.append(arg) - """ - ) - reprec = pytester.inline_run("-v") - reprec.assertoutcome(passed=3) - values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + record = run_tests(arg, test_param, rootpath=tmp_path) + record.assert_outcomes(passed=3) assert len(values) == 3 assert "a" in values - assert "b" in values - assert "c" in values - - def test_scope_mismatch(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - @pytest.fixture(scope="function") - def arg(request): - pass - """ - ) - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope="session") - def arg(arg): - pass - def test_mismatch(arg): + assert "b" in values + assert "c" in values + + def test_scope_mismatch(self, tmp_path: Path) -> None: + class ConftestPlugin: + @pytest.fixture(scope="function") + def arg(self, request): pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*ScopeMismatch*", "*1 error*"]) - def test_parametrize_separated_order(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + @pytest.fixture(scope="session") + def arg(arg): + pass - @pytest.fixture(scope="module", params=[1, 2]) - def arg(request): - return request.param + def test_mismatch(arg): + pass - values = [] - def test_1(arg): - values.append(arg) - def test_2(arg): - values.append(arg) - """ + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_mismatch", arg=arg, test_mismatch=test_mismatch), + spec=spec, + capture_output=True, ) - reprec = pytester.inline_run("-v") - reprec.assertoutcome(passed=4) - values = reprec.getcalls("pytest_runtest_call")[0].item.module.values - assert values == [1, 1, 2, 2] + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines(["*ScopeMismatch*", "*1 error*"]) - def test_module_parametrized_ordering(self, pytester: Pytester) -> None: - pytester.makeini( - """ - [pytest] - console_output_style=classic - """ - ) - pytester.makeconftest( - """ - import pytest + def test_parametrize_separated_order(self, tmp_path: Path) -> None: + values: list[int] = [] + + @pytest.fixture(scope="module", params=[1, 2]) + def arg(request): + return request.param + + def test_1(arg): + values.append(arg) + + def test_2(arg): + values.append(arg) + + record = run_tests(arg, test_1, test_2, rootpath=tmp_path) + record.assert_outcomes(passed=4) + assert values == [1, 1, 2, 2] + def test_module_parametrized_ordering(self, tmp_path: Path) -> None: + class ConftestPlugin: @pytest.fixture(scope="session", params="s1 s2".split()) - def sarg(): + def sarg(self): pass + @pytest.fixture(scope="module", params="m1 m2".split()) - def marg(): + def marg(self): pass - """ + + def test_func(sarg): + pass + + def test_func1(marg): + pass + + def test_func2(sarg): + pass + + def test_func3(sarg, marg): + pass + + def test_func3b(sarg, marg): + pass + + def test_func4(marg): + pass + + spec = ConfigSpec( + rootpath=tmp_path, + args=("-v",), + inicfg={"console_output_style": "classic"}, + extra_plugins=(ConftestPlugin(),), ) - pytester.makepyfile( - test_mod1=""" - def test_func(sarg): - pass - def test_func1(marg): - pass - """, - test_mod2=""" - def test_func2(sarg): - pass - def test_func3(sarg, marg): - pass - def test_func3b(sarg, marg): - pass - def test_func4(marg): - pass - """, + record = run_tests( + build_module("test_mod1", test_func, test_func1), + build_module("test_mod2", test_func2, test_func3, test_func3b, test_func4), + spec=spec, + capture_output=True, ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( """ test_mod1.py::test_func[s1] PASSED test_mod2.py::test_func2[s1] PASSED @@ -3078,43 +3075,42 @@ def test_func4(marg): """ ) - def test_dynamic_parametrized_ordering(self, pytester: Pytester) -> None: - pytester.makeini( - """ - [pytest] - console_output_style=classic - """ - ) - pytester.makeconftest( - """ - import pytest - - def pytest_configure(config): - class DynamicFixturePlugin(object): - @pytest.fixture(scope='session', params=['flavor1', 'flavor2']) + def test_dynamic_parametrized_ordering(self, tmp_path: Path) -> None: + class ConftestPlugin: + def pytest_configure(self, config): + class DynamicFixturePlugin: + @pytest.fixture(scope="session", params=["flavor1", "flavor2"]) def flavor(self, request): return request.param - config.pluginmanager.register(DynamicFixturePlugin(), 'flavor-fixture') - @pytest.fixture(scope='session', params=['vxlan', 'vlan']) - def encap(request): + config.pluginmanager.register(DynamicFixturePlugin(), "flavor-fixture") + + @pytest.fixture(scope="session", params=["vxlan", "vlan"]) + def encap(self, request): return request.param - @pytest.fixture(scope='session', autouse='True') - def reprovision(request, flavor, encap): + @pytest.fixture(scope="session", autouse="True") # type: ignore[call-overload] + def reprovision(self, request, flavor, encap): pass - """ + + def test(reprovision): + pass + + def test2(reprovision): + pass + + spec = ConfigSpec( + rootpath=tmp_path, + args=("-v",), + inicfg={"console_output_style": "classic"}, + extra_plugins=(ConftestPlugin(),), ) - pytester.makepyfile( - """ - def test(reprovision): - pass - def test2(reprovision): - pass - """ + record = run_tests( + build_module("test_dynamic_parametrized_ordering", test, test2), + spec=spec, + capture_output=True, ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( """ test_dynamic_parametrized_ordering.py::test[flavor1-vxlan] PASSED test_dynamic_parametrized_ordering.py::test2[flavor1-vxlan] PASSED @@ -3127,50 +3123,48 @@ def test2(reprovision): """ ) - def test_class_ordering(self, pytester: Pytester) -> None: - pytester.makeini( - """ - [pytest] - console_output_style=classic - """ - ) - pytester.makeconftest( - """ - import pytest - - values = [] + def test_class_ordering(self, tmp_path: Path) -> None: + values: list[str] = [] - @pytest.fixture(scope="function", params=[1,2]) - def farg(request): + class ConftestPlugin: + @pytest.fixture(scope="function", params=[1, 2]) + def farg(self, request): return request.param @pytest.fixture(scope="class", params=list("ab")) - def carg(request): + def carg(self, request): return request.param @pytest.fixture(scope="function", autouse=True) - def append(request, farg, carg): + def append(self, request, farg, carg): def fin(): - values.append("fin_%s%s" % (carg, farg)) + values.append(f"fin_{carg}{farg}") + request.addfinalizer(fin) - """ - ) - pytester.makepyfile( - """ - import pytest - class TestClass2(object): - def test_1(self): - pass - def test_2(self): - pass - class TestClass(object): - def test_3(self): - pass - """ + class TestClass2: + def test_1(self): + pass + + def test_2(self): + pass + + class TestClass: + def test_3(self): + pass + + spec = ConfigSpec( + rootpath=tmp_path, + args=("-v",), + inicfg={"console_output_style": "classic"}, + extra_plugins=(ConftestPlugin(),), ) - result = pytester.runpytest("-vs") - result.stdout.re_match_lines( + record = run_tests( + build_module("test_class_ordering", TestClass2, TestClass), + spec=spec, + capture_output=True, + ) + record.stdout.re_match_lines( r""" test_class_ordering.py::TestClass2::test_1\[a-1\] PASSED test_class_ordering.py::TestClass2::test_1\[a-2\] PASSED @@ -3188,40 +3182,40 @@ def test_3(self): ) def test_parametrize_separated_order_higher_scope_first( - self, pytester: Pytester + self, tmp_path: Path ) -> None: - pytester.makepyfile( - """ - import pytest + values: list[str] = [] - @pytest.fixture(scope="function", params=[1, 2]) - def arg(request): - param = request.param - request.addfinalizer(lambda: values.append("fin:%s" % param)) - values.append("create:%s" % param) - return request.param + @pytest.fixture(scope="function", params=[1, 2]) + def arg(request): + param = request.param + request.addfinalizer(lambda: values.append(f"fin:{param}")) + values.append(f"create:{param}") + return request.param - @pytest.fixture(scope="module", params=["mod1", "mod2"]) - def modarg(request): - param = request.param - request.addfinalizer(lambda: values.append("fin:%s" % param)) - values.append("create:%s" % param) - return request.param + @pytest.fixture(scope="module", params=["mod1", "mod2"]) + def modarg(request): + param = request.param + request.addfinalizer(lambda: values.append(f"fin:{param}")) + values.append(f"create:{param}") + return request.param - values = [] - def test_1(arg): - values.append("test1") - def test_2(modarg): - values.append("test2") - def test_3(arg, modarg): - values.append("test3") - def test_4(modarg, arg): - values.append("test4") - """ + def test_1(arg): + values.append("test1") + + def test_2(modarg): + values.append("test2") + + def test_3(arg, modarg): + values.append("test3") + + def test_4(modarg, arg): + values.append("test4") + + record = run_tests( + arg, modarg, test_1, test_2, test_3, test_4, rootpath=tmp_path ) - reprec = pytester.inline_run("-v") - reprec.assertoutcome(passed=12) - values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + record.assert_outcomes(passed=12) expected = [ "create:1", "test1", @@ -3265,52 +3259,45 @@ def test_4(modarg, arg): pprint.pprint(list(zip_longest(values, expected))) assert values == expected - def test_parametrized_fixture_teardown_order(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=[1,2], scope="class") - def param1(request): - return request.param + def test_parametrized_fixture_teardown_order(self, tmp_path: Path) -> None: + values: list[int] = [] - values = [] + @pytest.fixture(params=[1, 2], scope="class") + def param1(request): + return request.param - class TestClass(object): - @pytest.fixture(scope="class", autouse=True) - @classmethod - def setup1(cls, request, param1): - values.append(1) - request.addfinalizer(cls.teardown1) + class TestClass: + @pytest.fixture(scope="class", autouse=True) + @classmethod + def setup1(cls, request, param1): + values.append(1) + request.addfinalizer(cls.teardown1) - @classmethod - def teardown1(self): - assert values.pop() == 1 + @classmethod + def teardown1(self): + assert values.pop() == 1 - @pytest.fixture(scope="class", autouse=True) - @classmethod - def setup2(cls, request, param1): - values.append(2) - request.addfinalizer(cls.teardown2) + @pytest.fixture(scope="class", autouse=True) + @classmethod + def setup2(cls, request, param1): + values.append(2) + request.addfinalizer(cls.teardown2) - @classmethod - def teardown2(cls): - assert values.pop() == 2 + @classmethod + def teardown2(cls): + assert values.pop() == 2 - def test(self): - pass + def test(self): + pass - def test_finish(): - assert not values - """ - ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( - """ - *3 passed* - """ - ) - assert result.ret == 0 + def test_finish(): + assert not values + record = run_tests(param1, TestClass, test_finish, rootpath=tmp_path) + record.assert_outcomes(passed=3) + + # ensemble: the subject is the finalizer of a subdirectory module's + # override reaching the rootdir conftest fixture, plus -s stdout. def test_fixture_finalizer(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -3346,70 +3333,60 @@ def test_browser(browser): for test in ["test_browser"]: reprec.stdout.fnmatch_lines(["*Finalized*"]) - def test_class_scope_with_normal_tests(self, pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import pytest + def test_class_scope_with_normal_tests(self, tmp_path: Path) -> None: + class Box: + value = 0 - class Box(object): - value = 0 + @pytest.fixture(scope="class") + def a(request): + Box.value += 1 + return Box.value - @pytest.fixture(scope='class') - def a(request): - Box.value += 1 - return Box.value + def test_a(a): + assert a == 1 - def test_a(a): - assert a == 1 + class Test1: + def test_b(self, a): + assert a == 2 - class Test1(object): - def test_b(self, a): - assert a == 2 + class Test2: + def test_c(self, a): + assert a == 3 - class Test2(object): - def test_c(self, a): - assert a == 3""" - ) - reprec = pytester.inline_run(testpath) + record = run_tests(a, test_a, Test1, Test2, rootpath=tmp_path) for test in ["test_a", "test_b", "test_c"]: - assert reprec.matchreport(test).passed + assert record[test].passed - def test_request_is_clean(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - values = [] - @pytest.fixture(params=[1, 2]) - def fix(request): - request.addfinalizer(lambda: values.append(request.param)) - def test_fix(fix): - pass - """ - ) - reprec = pytester.inline_run("-s") - values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + def test_request_is_clean(self, tmp_path: Path) -> None: + values: list[int] = [] + + @pytest.fixture(params=[1, 2]) + def fix(request): + request.addfinalizer(lambda: values.append(request.param)) + + def test_fix(fix): + pass + + run_tests(fix, test_fix, rootpath=tmp_path) assert values == [1, 2] - def test_parametrize_separated_lifecycle(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_separated_lifecycle(self, tmp_path: Path) -> None: + values: list[object] = [] - values = [] - @pytest.fixture(scope="module", params=[1, 2]) - def arg(request): - x = request.param - request.addfinalizer(lambda: values.append("fin%s" % x)) - return request.param - def test_1(arg): - values.append(arg) - def test_2(arg): - values.append(arg) - """ - ) - reprec = pytester.inline_run("-vs") - reprec.assertoutcome(passed=4) - values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + @pytest.fixture(scope="module", params=[1, 2]) + def arg(request): + x = request.param + request.addfinalizer(lambda: values.append(f"fin{x}")) + return request.param + + def test_1(arg): + values.append(arg) + + def test_2(arg): + values.append(arg) + + record = run_tests(arg, test_1, test_2, rootpath=tmp_path) + record.assert_outcomes(passed=4) import pprint pprint.pprint(values) @@ -3420,95 +3397,89 @@ def test_2(arg): assert values[5] == "fin2" def test_parametrize_function_scoped_finalizers_called( - self, pytester: Pytester + self, tmp_path: Path ) -> None: - pytester.makepyfile( - """ - import pytest + values: list[object] = [] - @pytest.fixture(scope="function", params=[1, 2]) - def arg(request): - x = request.param - request.addfinalizer(lambda: values.append("fin%s" % x)) - return request.param + @pytest.fixture(scope="function", params=[1, 2]) + def arg(request): + x = request.param + request.addfinalizer(lambda: values.append(f"fin{x}")) + return request.param - values = [] - def test_1(arg): - values.append(arg) - def test_2(arg): - values.append(arg) - def test_3(): - assert len(values) == 8 - assert values == [1, "fin1", 2, "fin2", 1, "fin1", 2, "fin2"] - """ - ) - reprec = pytester.inline_run("-v") - reprec.assertoutcome(passed=5) + def test_1(arg): + values.append(arg) + + def test_2(arg): + values.append(arg) + + def test_3(): + assert len(values) == 8 + assert values == [1, "fin1", 2, "fin2", 1, "fin1", 2, "fin2"] + + record = run_tests(arg, test_1, test_2, test_3, rootpath=tmp_path) + record.assert_outcomes(passed=5) @pytest.mark.parametrize("scope", ["session", "function", "module"]) - def test_finalizer_order_on_parametrization( - self, scope, pytester: Pytester - ) -> None: + def test_finalizer_order_on_parametrization(self, scope, tmp_path: Path) -> None: """#246""" - pytester.makepyfile( - f""" - import pytest - values = [] + values: list[str] = [] - @pytest.fixture(scope={scope!r}, params=["1"]) - def fix1(request): - return request.param + @pytest.fixture(scope=scope, params=["1"]) + def fix1(request): + return request.param - @pytest.fixture(scope={scope!r}) - def fix2(request, base): - def cleanup_fix2(): - assert not values, "base should not have been finalized" - request.addfinalizer(cleanup_fix2) + @pytest.fixture(scope=scope) + def fix2(request, base): + def cleanup_fix2(): + assert not values, "base should not have been finalized" - @pytest.fixture(scope={scope!r}) - def base(request, fix1): - def cleanup_base(): - values.append("fin_base") - print("finalizing base") - request.addfinalizer(cleanup_base) + request.addfinalizer(cleanup_fix2) - def test_begin(): - pass - def test_baz(base, fix2): - pass - def test_other(): - pass - """ + @pytest.fixture(scope=scope) + def base(request, fix1): + def cleanup_base(): + values.append("fin_base") + print("finalizing base") + + request.addfinalizer(cleanup_base) + + def test_begin(): + pass + + def test_baz(base, fix2): + pass + + def test_other(): + pass + + record = run_tests( + fix1, fix2, base, test_begin, test_baz, test_other, rootpath=tmp_path ) - reprec = pytester.inline_run("-lvs") - reprec.assertoutcome(passed=3) + record.assert_outcomes(passed=3) - def test_class_scope_parametrization_ordering(self, pytester: Pytester) -> None: + def test_class_scope_parametrization_ordering(self, tmp_path: Path) -> None: """#396""" - pytester.makepyfile( - """ - import pytest - values = [] - @pytest.fixture(params=["John", "Doe"], scope="class") - def human(request): - request.addfinalizer(lambda: values.append("fin %s" % request.param)) - return request.param + values: list[str] = [] - class TestGreetings(object): - def test_hello(self, human): - values.append("test_hello") + @pytest.fixture(params=["John", "Doe"], scope="class") + def human(request): + request.addfinalizer(lambda: values.append(f"fin {request.param}")) + return request.param - class TestMetrics(object): - def test_name(self, human): - values.append("test_name") + class TestGreetings: + def test_hello(self, human): + values.append("test_hello") - def test_population(self, human): - values.append("test_population") - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=6) - values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + class TestMetrics: + def test_name(self, human): + values.append("test_name") + + def test_population(self, human): + values.append("test_population") + + record = run_tests(human, TestGreetings, TestMetrics, rootpath=tmp_path) + record.assert_outcomes(passed=6) assert values == [ "test_hello", "fin John", @@ -3522,89 +3493,75 @@ def test_population(self, human): "fin Doe", ] - def test_parametrize_setup_function(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_setup_function(self, tmp_path: Path) -> None: + values: list[object] = [] - @pytest.fixture(scope="module", params=[1, 2]) - def arg(request): - return request.param + @pytest.fixture(scope="module", params=[1, 2]) + def arg(request): + return request.param - @pytest.fixture(scope="module", autouse=True) - def mysetup(request, arg): - request.addfinalizer(lambda: values.append("fin%s" % arg)) - values.append("setup%s" % arg) + @pytest.fixture(scope="module", autouse=True) + def mysetup(request, arg): + request.addfinalizer(lambda: values.append(f"fin{arg}")) + values.append(f"setup{arg}") - values = [] - def test_1(arg): - values.append(arg) - def test_2(arg): - values.append(arg) - def test_3(): - import pprint - pprint.pprint(values) - if arg == 1: - assert values == ["setup1", 1, 1, ] - elif arg == 2: - assert values == ["setup1", 1, 1, "fin1", - "setup2", 2, 2, ] + def test_1(arg): + values.append(arg) - """ - ) - reprec = pytester.inline_run("-v") - reprec.assertoutcome(passed=6) + def test_2(arg): + values.append(arg) + + def test_3(): + import pprint + + pprint.pprint(values) + # ``arg`` is the fixture object here, exactly as in the original + # module-level source: neither branch is ever taken. + arg_value: object = arg + if arg_value == 1: + assert values == ["setup1", 1, 1] + elif arg_value == 2: + assert values == ["setup1", 1, 1, "fin1", "setup2", 2, 2] + + record = run_tests(arg, mysetup, test_1, test_2, test_3, rootpath=tmp_path) + record.assert_outcomes(passed=6) def test_fixture_marked_function_not_collected_as_test( - self, pytester: Pytester + self, tmp_path: Path ) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def test_app(): - return 1 - - def test_something(test_app): - assert test_app == 1 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + @pytest.fixture + def test_app(): + return 1 - def test_params_and_ids(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_something(test_app): + assert test_app == 1 - @pytest.fixture(params=[object(), object()], - ids=['alpha', 'beta']) - def fix(request): - return request.param + record = run_tests(test_app, test_something, rootpath=tmp_path) + record.assert_outcomes(passed=1) - def test_foo(fix): - assert 1 - """ - ) - res = pytester.runpytest("-v") - res.stdout.fnmatch_lines(["*test_foo*alpha*", "*test_foo*beta*"]) + def test_params_and_ids(self, tmp_path: Path) -> None: + @pytest.fixture(params=[object(), object()], ids=["alpha", "beta"]) + def fix(request): + return request.param - def test_params_and_ids_yieldfixture(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_foo(fix): + assert 1 - @pytest.fixture(params=[object(), object()], ids=['alpha', 'beta']) - def fix(request): - yield request.param + items = collect_tests(fix, test_foo, rootpath=tmp_path) + assert [item.name for item in items] == ["test_foo[alpha]", "test_foo[beta]"] - def test_foo(fix): - assert 1 - """ - ) - res = pytester.runpytest("-v") - res.stdout.fnmatch_lines(["*test_foo*alpha*", "*test_foo*beta*"]) + def test_params_and_ids_yieldfixture(self, tmp_path: Path) -> None: + @pytest.fixture(params=[object(), object()], ids=["alpha", "beta"]) + def fix(request): + yield request.param + + def test_foo(fix): + assert 1 + + items = collect_tests(fix, test_foo, rootpath=tmp_path) + assert [item.name for item in items] == ["test_foo[alpha]", "test_foo[beta]"] + # ensemble: needs two subprocess runs with different PYTHONHASHSEED. def test_deterministic_fixture_collection( self, pytester: Pytester, monkeypatch ) -> None: @@ -3662,87 +3619,79 @@ class TestRequestScopeAccess: ], ) - def test_setup(self, pytester: Pytester, scope, ok, error) -> None: - pytester.makepyfile( - f""" - import pytest - @pytest.fixture(scope={scope!r}, autouse=True) - def myscoped(request): - for x in {ok.split()}: - assert hasattr(request, x) - for x in {error.split()}: - with pytest.raises(AttributeError): - getattr(request, x) - assert request.session - assert request.config - def test_func(): - pass - """ - ) - reprec = pytester.inline_run("-l") - reprec.assertoutcome(passed=1) + def test_setup(self, tmp_path: Path, scope, ok, error) -> None: + @pytest.fixture(scope=scope, autouse=True) + def myscoped(request): + for x in ok.split(): + assert hasattr(request, x) + for x in error.split(): + with pytest.raises(AttributeError): + getattr(request, x) + assert request.session + assert request.config - def test_funcarg(self, pytester: Pytester, scope, ok, error) -> None: - pytester.makepyfile( - f""" - import pytest - @pytest.fixture(scope={scope!r}) - def arg(request): - for x in {ok.split()!r}: - assert hasattr(request, x) - for x in {error.split()!r}: - with pytest.raises(AttributeError): - getattr(request, x) - assert request.session - assert request.config - def test_func(arg): - pass - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + def test_func(): + pass + + run_tests(myscoped, test_func, rootpath=tmp_path).assert_outcomes(passed=1) + + def test_funcarg(self, tmp_path: Path, scope, ok, error) -> None: + @pytest.fixture(scope=scope) + def arg(request): + for x in ok.split(): + assert hasattr(request, x) + for x in error.split(): + with pytest.raises(AttributeError): + getattr(request, x) + assert request.session + assert request.config + + def test_func(arg): + pass + + run_tests(arg, test_func, rootpath=tmp_path).assert_outcomes(passed=1) class TestErrors: - def test_subfactory_missing_funcarg(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture() - def gen(qwe123): - return 1 - def test_something(gen): - pass - """ - ) - result = pytester.runpytest() - assert result.ret != 0 - result.stdout.fnmatch_lines( + def test_subfactory_missing_funcarg(self, tmp_path: Path) -> None: + @pytest.fixture + def gen(qwe123): + return 1 + + def test_something(gen): + pass + + record = run_tests(gen, test_something, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( ["*def gen(qwe123):*", "*fixture*qwe123*not found*", "*1 error*"] ) - def test_issue498_fixture_finalizer_failing(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def fix1(request): - def f(): - raise KeyError - request.addfinalizer(f) - return object() + def test_issue498_fixture_finalizer_failing(self, tmp_path: Path) -> None: + values: list[object] = [] - values = [] - def test_1(fix1): - values.append(fix1) - def test_2(fix1): - values.append(fix1) - def test_3(): - assert values[0] != values[1] - """ + @pytest.fixture + def fix1(request): + def f(): + raise KeyError + + request.addfinalizer(f) + return object() + + def test_1(fix1): + values.append(fix1) + + def test_2(fix1): + values.append(fix1) + + def test_3(): + assert values[0] != values[1] + + record = run_tests( + fix1, test_1, test_2, test_3, rootpath=tmp_path, capture_output=True ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record.assert_outcomes(passed=3, errors=2) + record.stdout.fnmatch_lines( """ *ERROR*teardown*test_1* *KeyError* @@ -3752,50 +3701,55 @@ def test_3(): """ ) - def test_setupfunc_missing_funcarg(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(autouse=True) - def gen(qwe123): - return 1 - def test_something(): - pass - """ - ) - result = pytester.runpytest() - assert result.ret != 0 - result.stdout.fnmatch_lines( + def test_setupfunc_missing_funcarg(self, tmp_path: Path) -> None: + @pytest.fixture(autouse=True) + def gen(qwe123): + return 1 + + def test_something(): + pass + + record = run_tests(gen, test_something, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( ["*def gen(qwe123):*", "*fixture*qwe123*not found*", "*1 error*"] ) - def test_cached_exception_doesnt_get_longer(self, pytester: Pytester) -> None: + def test_cached_exception_doesnt_get_longer(self, tmp_path: Path) -> None: """Regression test for #12204.""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope="session") - def bad(): 1 / 0 - def test_1(bad): pass - def test_2(bad): pass - def test_3(bad): pass - """ - ) + @pytest.fixture(scope="session") + def bad(): + 1 / 0 # noqa: B018 - result = pytester.runpytest_inprocess("--tb=native") - assert result.ret == ExitCode.TESTS_FAILED - failures = result.reprec.getfailures() # type: ignore[attr-defined] + def test_1(bad): ... + + def test_2(bad): ... + + def test_3(bad): ... + + # --tb is registered by the terminal plugin, so it has to be loaded; + # capture_output keeps what it renders away from the outer stdout. + spec = ConfigSpec(rootpath=tmp_path, args=("--tb=native",)).with_plugins( + "terminal" + ) + record = run_tests(bad, test_1, test_2, test_3, spec=spec, capture_output=True) + record.assert_outcomes(errors=3) + failures = [report for report in record.reports if report.failed] assert len(failures) == 3 - lines1 = failures[1].longrepr.reprtraceback.reprentries[0].lines - lines2 = failures[2].longrepr.reprtraceback.reprentries[0].lines + lines1 = failures[1].longrepr.reprtraceback.reprentries[0].lines # type: ignore[union-attr] + lines2 = failures[2].longrepr.reprtraceback.reprentries[0].lines # type: ignore[union-attr] assert len(lines1) == len(lines2) +# ensemble: every test here drives ``--fixtures``, which is implemented as a +# ``pytest_cmdline_main`` hook and renders fixture *definition* locations; an +# ensemble neither reaches cmdline_main nor has non-host source locations. class TestShowFixtures: - def test_funcarg_compat(self, pytester: Pytester) -> None: - config = pytester.parseconfigure("--funcargs") - assert config.option.showfixtures + def test_funcarg_compat(self, tmp_path: Path) -> None: + spec = ConfigSpec(rootpath=tmp_path, args=("--funcargs",)) + with Ensemble(spec=spec) as ensemble: + assert ensemble.config.option.showfixtures def test_show_help(self, pytester: Pytester) -> None: result = pytester.runpytest("--fixtures", "--help") @@ -4085,99 +4039,87 @@ def foo(): class TestContextManagerFixtureFuncs: - def test_simple(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def arg1(): - print("setup") - yield 1 - print("teardown") - def test_1(arg1): - print("test1", arg1) - def test_2(arg1): - print("test2", arg1) - assert 0 - """ - ) - result = pytester.runpytest("-s") - result.stdout.fnmatch_lines( - """ - *setup* - *test1 1* - *teardown* - *setup* - *test2 1* - *teardown* - """ - ) + def test_simple(self, tmp_path: Path) -> None: + # The original watched the ordering through printed output under -s; + # recording the events directly asserts the same ordering without + # depending on capture, which an ensemble does not provide. + events: list[str] = [] - def test_scoped(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope="module") - def arg1(): - print("setup") - yield 1 - print("teardown") - def test_1(arg1): - print("test1", arg1) - def test_2(arg1): - print("test2", arg1) - """ - ) - result = pytester.runpytest("-s") - result.stdout.fnmatch_lines( - """ - *setup* - *test1 1* - *test2 1* - *teardown* - """ - ) + @pytest.fixture + def arg1(): + events.append("setup") + yield 1 + events.append("teardown") + + def test_1(arg1): + events.append(f"test1 {arg1}") + + def test_2(arg1): + events.append(f"test2 {arg1}") + assert 0 + + record = run_tests(arg1, test_1, test_2, rootpath=tmp_path) + record.assert_outcomes(passed=1, failed=1) + assert events == [ + "setup", + "test1 1", + "teardown", + "setup", + "test2 1", + "teardown", + ] - def test_setup_exception(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope="module") - def arg1(): - pytest.fail("setup") - yield 1 - def test_1(arg1): - pass - """ - ) - result = pytester.runpytest("-s") - result.stdout.fnmatch_lines( - """ - *pytest.fail*setup* - *1 error* - """ - ) + def test_scoped(self, tmp_path: Path) -> None: + events: list[str] = [] - def test_teardown_exception(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope="module") - def arg1(): - yield 1 - pytest.fail("teardown") - def test_1(arg1): - pass - """ - ) - result = pytester.runpytest("-s") - result.stdout.fnmatch_lines( - """ - *pytest.fail*teardown* - *1 passed*1 error* - """ - ) + @pytest.fixture(scope="module") + def arg1(): + events.append("setup") + yield 1 + events.append("teardown") + + def test_1(arg1): + events.append(f"test1 {arg1}") + + def test_2(arg1): + events.append(f"test2 {arg1}") + + record = run_tests(arg1, test_1, test_2, rootpath=tmp_path) + record.assert_outcomes(passed=2) + assert events == ["setup", "test1 1", "test2 1", "teardown"] + + def test_setup_exception(self, tmp_path: Path) -> None: + @pytest.fixture(scope="module") + def arg1(): + pytest.fail("setup") + yield 1 # type: ignore[unreachable] + + def test_1(arg1): + pass + + record = run_tests(arg1, test_1, rootpath=tmp_path) + record.assert_outcomes(errors=1) + setup = record["test_1"].setup + assert setup is not None + assert "Failed: setup" in setup.longreprtext + + def test_teardown_exception(self, tmp_path: Path) -> None: + @pytest.fixture(scope="module") + def arg1(): + yield 1 + pytest.fail("teardown") + + def test_1(arg1): + pass + record = run_tests(arg1, test_1, rootpath=tmp_path) + record.assert_outcomes(passed=1, errors=1) + teardown = record["test_1"].teardown + assert teardown is not None + assert "Failed: teardown" in teardown.longreprtext + + # ensemble: asserts the offending fixture's file:line, host-anchored for + # in-memory sources. def test_yields_more_than_one(self, pytester: Pytester) -> None: pytester.makepyfile( """ @@ -4198,21 +4140,24 @@ def test_1(arg1): """ ) - def test_custom_name(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(name='meow') - def arg1(): - return 'mew' - def test_1(meow): - print(meow) - """ - ) - result = pytester.runpytest("-s") - result.stdout.fnmatch_lines(["*mew*"]) + def test_custom_name(self, tmp_path: Path) -> None: + seen: list[str] = [] + @pytest.fixture(name="meow") + def arg1(): + return "mew" + def test_1(meow): + seen.append(meow) + + record = run_tests(arg1, test_1, rootpath=tmp_path) + record.assert_outcomes(passed=1) + assert seen == ["mew"] + + +# ensemble: every test here asserts where the requested fixture is *defined* +# and where it was requested from, as file:line; for in-memory sources both +# resolve into this host file. class TestParameterizedSubRequest: def test_call_from_fixture(self, pytester: Pytester) -> None: pytester.makepyfile( @@ -4365,6 +4310,8 @@ def test_foo(request): ) +# ensemble: the subject is a rootdir conftest and a subdirectory conftest +# both implementing the hook, and the order they run in. def test_pytest_fixture_setup_and_post_finalizer_hook(pytester: Pytester) -> None: pytester.makeconftest( """ @@ -4408,24 +4355,21 @@ def test_func(my_fixture): ) -def test_fixture_post_finalizer_called_once(pytester: Pytester) -> None: +def test_fixture_post_finalizer_called_once(tmp_path: Path) -> None: """Test that pytest_fixture_post_finalizer is called only once per fixture teardown. When a fixture depends on multiple parametrized fixtures and all their parameters change at the same time, the dependent fixture should be torn down only once, and pytest_fixture_post_finalizer should be called only once for it. """ - pytester.makeconftest( - """ - import pytest + finalizer_calls: list[str] = [] - finalizer_calls = [] - - def pytest_fixture_post_finalizer(fixturedef, request): + class ConftestPlugin: + def pytest_fixture_post_finalizer(self, fixturedef, request): finalizer_calls.append(fixturedef.argname) @pytest.fixture(autouse=True) - def check_finalizer_calls(request): + def check_finalizer_calls(self, request): yield # After each test, verify no duplicate finalizer calls. if finalizer_calls: @@ -4433,75 +4377,74 @@ def check_finalizer_calls(request): f"Duplicate finalizer calls detected: {finalizer_calls}" ) finalizer_calls.clear() - """ - ) - pytester.makepyfile( - test_fixtures=""" - import pytest - @pytest.fixture(scope="session") - def foo(request): - return request.param + @pytest.fixture(scope="session") + def foo(request): + return request.param - @pytest.fixture(scope="session") - def bar(request): - return request.param + @pytest.fixture(scope="session") + def bar(request): + return request.param - @pytest.fixture(scope="session") - def baz(foo, bar): - return f"{foo}-{bar}" - - @pytest.mark.parametrize("foo,bar", [(1, 1)], indirect=True) - def test_first(foo, bar, baz): - assert foo == 1 - assert bar == 1 - assert baz == "1-1" - - @pytest.mark.parametrize("foo,bar", [(2, 2)], indirect=True) - def test_second(foo, bar, baz): - assert foo == 2 - assert bar == 2 - assert baz == "2-2" - """ - ) - result = pytester.runpytest("-v") + @pytest.fixture(scope="session") + def baz(foo, bar): + return f"{foo}-{bar}" + + @pytest.mark.parametrize("foo,bar", [(1, 1)], indirect=True) + def test_first(foo, bar, baz): + assert foo == 1 + assert bar == 1 + assert baz == "1-1" + + @pytest.mark.parametrize("foo,bar", [(2, 2)], indirect=True) + def test_second(foo, bar, baz): + assert foo == 2 + assert bar == 2 + assert baz == "2-2" + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + module = build_module("test_fixtures", foo, bar, baz, test_first, test_second) + record = run_tests(module, spec=spec) # The test passes, which means no duplicate finalizer calls were detected # by the check_finalizer_calls autouse fixture. - result.assert_outcomes(passed=2) + record.assert_outcomes(passed=2) -def test_fixture_post_finalizer_hook_exception(pytester: Pytester) -> None: +def test_fixture_post_finalizer_hook_exception(tmp_path: Path) -> None: """Test that exceptions in pytest_fixture_post_finalizer hook are caught. Also verifies that the fixture cache is properly reset even when the post_finalizer hook raises an exception, so the fixture can be rebuilt in subsequent tests. """ - pytester.makeconftest( - """ - import pytest - def pytest_fixture_post_finalizer(fixturedef, request): + class ConftestPlugin: + def pytest_fixture_post_finalizer(self, fixturedef, request): if "test_first" in request.node.nodeid: raise RuntimeError("Error in post finalizer hook") @pytest.fixture - def my_fixture(request): + def my_fixture(self, request): yield request.node.nodeid - """ - ) - pytester.makepyfile( - test_fixtures=""" - def test_first(my_fixture): - assert "test_first" in my_fixture - def test_second(my_fixture): - assert "test_second" in my_fixture - """ + def test_first(my_fixture): + assert "test_first" in my_fixture + + def test_second(my_fixture): + assert "test_second" in my_fixture + + spec = ConfigSpec( + rootpath=tmp_path, + args=("-v", "--setup-show"), + extra_plugins=(ConftestPlugin(),), + ).with_plugins("setuponly") + record = run_tests( + build_module("test_fixtures", test_first, test_second), + spec=spec, + capture_output=True, ) - result = pytester.runpytest("-v", "--setup-show") - result.assert_outcomes(passed=2, errors=1) - result.stdout.fnmatch_lines( + record.assert_outcomes(passed=2, errors=1) + record.stdout.fnmatch_lines( [ "*test_first*PASSED", "*test_first*ERROR", @@ -4509,7 +4452,7 @@ def test_second(my_fixture): ] ) # Verify fixture is setup twice (rebuilt for test_second despite error). - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "test_fixtures.py::test_first ", " SETUP F my_fixture", @@ -4583,42 +4526,35 @@ class TestScopeOrdering: """Class of tests that ensure fixtures are ordered based on their scopes (#2405)""" @pytest.mark.parametrize("variant", ["mark", "autouse"]) - def test_func_closure_module_auto( - self, pytester: Pytester, variant, monkeypatch - ) -> None: + def test_func_closure_module_auto(self, tmp_path: Path, variant) -> None: """Semantically identical to the example posted in #2405 when ``use_mark=True``""" - monkeypatch.setenv("FIXTURE_ACTIVATION_VARIANT", variant) - pytester.makepyfile( - """ - import warnings - import os - import pytest - VAR = 'FIXTURE_ACTIVATION_VARIANT' - VALID_VARS = ('autouse', 'mark') - - VARIANT = os.environ.get(VAR) - if VARIANT is None or VARIANT not in VALID_VARS: - warnings.warn("{!r} is not in {}, assuming autouse".format(VARIANT, VALID_VARS) ) - variant = 'mark' - @pytest.fixture(scope='module', autouse=VARIANT == 'autouse') - def m1(): pass - - if VARIANT=='mark': - pytestmark = pytest.mark.usefixtures('m1') + @pytest.fixture(scope="module", autouse=variant == "autouse") + def m1(): + pass - @pytest.fixture(scope='function', autouse=True) - def f1(): pass + @pytest.fixture(scope="function", autouse=True) + def f1(): + pass - def test_func(m1): - pass - """ - ) - items, _ = pytester.inline_genitems() - assert isinstance(items[0], Function) - request = TopRequest(items[0], _ispytest=True) - assert request.fixturenames == "m1 f1".split() + def test_func(m1): + pass + module = build_module( + "test_func_closure_module_auto", + m1, + f1, + test_func, + pytestmark=pytest.mark.usefixtures("m1") if variant == "mark" else [], + ) + with Ensemble(module, rootpath=tmp_path) as ensemble: + items = ensemble.collect() + assert isinstance(items[0], Function) + request = TopRequest(items[0], _ispytest=True) + assert request.fixturenames == "m1 f1".split() + + # ensemble: the closure under test contains a package-scoped fixture, and + # an ensemble has no Package node for it to bind to. def test_func_closure_with_native_fixtures(self, pytester: Pytester) -> None: """Sanity check that verifies the order returned by the closures and the actual fixture execution order: the execution order may differ because @@ -4688,60 +4624,62 @@ def test_foo(f1, p1, m1, f2, s1): result = pytester.runpytest("-vv") result.assert_outcomes(passed=1) - def test_func_closure_module(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_func_closure_module(self, tmp_path: Path) -> None: + @pytest.fixture(scope="module") + def m1(): + pass - @pytest.fixture(scope='module') - def m1(): pass + @pytest.fixture(scope="function") + def f1(): + pass - @pytest.fixture(scope='function') - def f1(): pass + def test_func(f1, m1): + pass - def test_func(f1, m1): - pass - """ - ) - items, _ = pytester.inline_genitems() - assert isinstance(items[0], Function) - request = TopRequest(items[0], _ispytest=True) - assert request.fixturenames == "m1 f1".split() + with Ensemble(m1, f1, test_func, rootpath=tmp_path) as ensemble: + items = ensemble.collect() + assert isinstance(items[0], Function) + request = TopRequest(items[0], _ispytest=True) + assert request.fixturenames == "m1 f1".split() - def test_func_closure_scopes_reordered(self, pytester: Pytester) -> None: + def test_func_closure_scopes_reordered(self, tmp_path: Path) -> None: """Test ensures that fixtures are ordered by scope regardless of the order of the parameters, although fixtures of same scope keep the declared order """ - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope='session') - def s1(): pass + @pytest.fixture(scope="session") + def s1(): + pass - @pytest.fixture(scope='module') - def m1(): pass + @pytest.fixture(scope="module") + def m1(): + pass - @pytest.fixture(scope='function') - def f1(): pass + @pytest.fixture(scope="function") + def f1(): + pass - @pytest.fixture(scope='function') - def f2(): pass + @pytest.fixture(scope="function") + def f2(): + pass - class Test: + class Test: + @pytest.fixture(scope="class") + def c1(cls): + pass - @pytest.fixture(scope='class') - def c1(cls): pass + def test_func(self, f2, f1, c1, m1, s1): + pass - def test_func(self, f2, f1, c1, m1, s1): - pass - """ - ) - items, _ = pytester.inline_genitems() - assert isinstance(items[0], Function) - request = TopRequest(items[0], _ispytest=True) - assert request.fixturenames == "s1 m1 c1 f2 f1".split() + # Fixture *definition* order matters here, and in an ensemble that is + # the order the members are passed in. + with Ensemble(s1, m1, f1, f2, Test, rootpath=tmp_path) as ensemble: + items = ensemble.collect() + assert isinstance(items[0], Function) + request = TopRequest(items[0], _ispytest=True) + assert request.fixturenames == "s1 m1 c1 f2 f1".split() + # ensemble: conftests in nested directories, one of them package-scoped. def test_func_closure_same_scope_closer_root_first( self, pytester: Pytester ) -> None: @@ -4785,6 +4723,7 @@ def test_func(m_test, f1): request = TopRequest(items[0], _ispytest=True) assert request.fixturenames == "p_sub m_conf m_sub m_test f1".split() + # ensemble: the closure under test contains a package-scoped fixture. def test_func_closure_all_scopes_complex(self, pytester: Pytester) -> None: """Complex test involving all scopes and mixing autouse with normal fixtures""" pytester.makeconftest( @@ -4830,6 +4769,7 @@ def test_func(self, f2, f1, m2): request = TopRequest(items[0], _ispytest=True) assert request.fixturenames == "s1 p1 m1 m2 c1 f2 f1".split() + # ensemble: package-scoped fixture, and a package layout. def test_parametrized_package_scope_reordering(self, pytester: Pytester) -> None: """A parameterized package-scoped fixture correctly reorders items to minimize setups & teardowns. @@ -4864,7 +4804,7 @@ def fix(request): ) def test_reorder_by_param_value_across_parametrize_calls( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Items parametrized by separate parametrize() calls are grouped by the *value* of higher-scoped parameters, so that equal values share a @@ -4872,27 +4812,29 @@ def test_reorder_by_param_value_across_parametrize_calls( Regression test for #8914. """ - pytester.makepyfile( - test_8914=""" - import pytest - @pytest.fixture(scope="session") - def prepare(request): - return request.param + @pytest.fixture(scope="session") + def prepare(request): + return request.param - @pytest.mark.parametrize("prepare", ["dina"], indirect=True, scope="session") - def test_1(prepare): pass + @pytest.mark.parametrize("prepare", ["dina"], indirect=True, scope="session") + def test_1(prepare): ... - @pytest.mark.parametrize("prepare", ["more"], indirect=True, scope="session") - def test_2(prepare): pass + @pytest.mark.parametrize("prepare", ["more"], indirect=True, scope="session") + def test_2(prepare): ... - @pytest.mark.parametrize("prepare", ["dina"], indirect=True, scope="session") - def test_3(prepare): pass - """ + @pytest.mark.parametrize("prepare", ["dina"], indirect=True, scope="session") + def test_3(prepare): ... + + spec = ConfigSpec(rootpath=tmp_path, args=("--setup-plan",)).with_plugins( + "setupplan", "setuponly" ) - result = pytester.runpytest("--setup-plan") - assert result.ret == ExitCode.OK - result.stdout.fnmatch_lines( + record = run_tests( + build_module("test_8914", prepare, test_1, test_2, test_3), + spec=spec, + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "SETUP S prepare['dina']", " test_8914.py::test_1[dina] (fixtures used: prepare, request)", @@ -4904,30 +4846,34 @@ def test_3(prepare): pass ], ) - def test_reorder_unhashable_params_fall_back_to_index( - self, pytester: Pytester - ) -> None: + def test_reorder_unhashable_params_fall_back_to_index(self, tmp_path: Path) -> None: """Unhashable parameter values are grouped by their index within their parametrize() call, as they were before #8914 was fixed. """ - pytester.makepyfile( - test_unhashable=""" - import pytest - @pytest.fixture(scope="module") - def fix(request): - return request.param + @pytest.fixture(scope="module") + def fix(request): + return request.param + + @pytest.mark.parametrize( + "fix", [{"a": 1}, {"b": 2}], indirect=True, scope="module" + ) + def test_1(fix): ... - @pytest.mark.parametrize("fix", [{"a": 1}, {"b": 2}], indirect=True, scope="module") - def test_1(fix): pass + @pytest.mark.parametrize( + "fix", [{"a": 1}, {"b": 2}], indirect=True, scope="module" + ) + def test_2(fix): ... - @pytest.mark.parametrize("fix", [{"a": 1}, {"b": 2}], indirect=True, scope="module") - def test_2(fix): pass - """ + spec = ConfigSpec(rootpath=tmp_path, args=("--setup-plan",)).with_plugins( + "setupplan", "setuponly" ) - result = pytester.runpytest("--setup-plan") - assert result.ret == ExitCode.OK - result.stdout.fnmatch_lines( + record = run_tests( + build_module("test_unhashable", fix, test_1, test_2), + spec=spec, + capture_output=True, + ) + record.stdout.fnmatch_lines( [ " SETUP M fix[{'a': 1}]", " test_unhashable.py::test_1[fix0] (fixtures used: fix, request)", @@ -4940,31 +4886,33 @@ def test_2(fix): pass ], ) - def test_reorder_mixed_hashable_unhashable_params(self, pytester: Pytester) -> None: + def test_reorder_mixed_hashable_unhashable_params(self, tmp_path: Path) -> None: """Hashable and unhashable values parametrizing the same fixture only group with their own kind: values with values, unhashables by index. """ - pytester.makepyfile( - test_mixed=""" - import pytest - @pytest.fixture(scope="module") - def fix(request): - return request.param + @pytest.fixture(scope="module") + def fix(request): + return request.param - @pytest.mark.parametrize("fix", [{"a": 1}], indirect=True, scope="module") - def test_1(fix): pass + @pytest.mark.parametrize("fix", [{"a": 1}], indirect=True, scope="module") + def test_1(fix): ... - @pytest.mark.parametrize("fix", ["x"], indirect=True, scope="module") - def test_2(fix): pass + @pytest.mark.parametrize("fix", ["x"], indirect=True, scope="module") + def test_2(fix): ... - @pytest.mark.parametrize("fix", ["x"], indirect=True, scope="module") - def test_3(fix): pass - """ + @pytest.mark.parametrize("fix", ["x"], indirect=True, scope="module") + def test_3(fix): ... + + spec = ConfigSpec(rootpath=tmp_path, args=("--setup-plan",)).with_plugins( + "setupplan", "setuponly" ) - result = pytester.runpytest("--setup-plan") - assert result.ret == ExitCode.OK - result.stdout.fnmatch_lines( + record = run_tests( + build_module("test_mixed", fix, test_1, test_2, test_3), + spec=spec, + capture_output=True, + ) + record.stdout.fnmatch_lines( [ " SETUP M fix[{'a': 1}]", " test_mixed.py::test_1[fix0] (fixtures used: fix, request)", @@ -4976,37 +4924,35 @@ def test_3(fix): pass ], ) - def test_reorder_params_with_exotic_eq(self, pytester: Pytester) -> None: + def test_reorder_params_with_exotic_eq(self, tmp_path: Path) -> None: """Parameter values whose ``__eq__`` raises or returns non-booleans (e.g. numpy arrays) do not break collection or reordering (#6497). """ - pytester.makepyfile( - """ - import pytest - class Exotic: - def __init__(self, value): - self.value = value - def __eq__(self, other): - raise ValueError("cannot compare") - def __hash__(self): - return 0 + class Exotic: + def __init__(self, value): + self.value = value + + def __eq__(self, other): + raise ValueError("cannot compare") - @pytest.fixture(scope="module") - def fix(request): - return request.param + def __hash__(self): + return 0 - @pytest.mark.parametrize("fix", [Exotic(1)], indirect=True, scope="module") - def test_1(fix): pass + @pytest.fixture(scope="module") + def fix(request): + return request.param - @pytest.mark.parametrize("fix", [Exotic(2)], indirect=True, scope="module") - def test_2(fix): pass - """ - ) - result = pytester.runpytest() - assert result.ret == ExitCode.OK - result.assert_outcomes(passed=2) + @pytest.mark.parametrize("fix", [Exotic(1)], indirect=True, scope="module") + def test_1(fix): ... + @pytest.mark.parametrize("fix", [Exotic(2)], indirect=True, scope="module") + def test_2(fix): ... + + record = run_tests(fix, test_1, test_2, rootpath=tmp_path) + record.assert_outcomes(passed=2) + + # ensemble: package layout with package-scoped fixtures in two packages. def test_multiple_packages(self, pytester: Pytester) -> None: """Complex test involving multiple package fixtures. Make sure teardowns are executed in order. @@ -5081,37 +5027,26 @@ def test_2(fix): reprec = pytester.inline_run() reprec.assertoutcome(passed=2) - def test_class_fixture_self_instance(self, pytester: Pytester) -> None: + def test_class_fixture_self_instance(self, tmp_path: Path) -> None: """Check that plugin classes which implement fixtures receive the plugin instance as self (see #2270). """ - pytester.makeconftest( - """ - import pytest - def pytest_configure(config): - config.pluginmanager.register(MyPlugin()) + class MyPlugin: + def __init__(self): + self.arg = 1 - class MyPlugin(): - def __init__(self): - self.arg = 1 + @pytest.fixture(scope="function") + def myfix(self): + assert isinstance(self, MyPlugin) + return self.arg - @pytest.fixture(scope='function') - def myfix(self): - assert isinstance(self, MyPlugin) - return self.arg - """ - ) + class TestClass: + def test_1(self, myfix): + assert myfix == 1 - pytester.makepyfile( - """ - class TestClass(object): - def test_1(self, myfix): - assert myfix == 1 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(MyPlugin(),)) + run_tests(TestClass, spec=spec).assert_outcomes(passed=1) def test_call_fixture_function_error(): @@ -5125,6 +5060,9 @@ def fix(): assert fix() == 1 +# ensemble: the double decoration raises at decoration time, so it can only +# be observed as a *module import* failure (see test_fixture_disallow_twice +# for the direct form). def test_fixture_double_decorator(pytester: Pytester) -> None: """Check if an error is raised when using @pytest.fixture twice.""" pytester.makepyfile( @@ -5146,6 +5084,8 @@ def fixt(): ) +# ensemble: `@pytest.fixture` on a class raises at decoration time, so it can +# only be observed as a module import failure. def test_fixture_class(pytester: Pytester) -> None: """Check if an error is raised when using @pytest.fixture on a class.""" pytester.makepyfile( @@ -5161,49 +5101,57 @@ class A: result.assert_outcomes(errors=1) -def test_fixture_param_shadowing(pytester: Pytester) -> None: +def test_fixture_param_shadowing(tmp_path: Path) -> None: """Parametrized arguments would be shadowed if a fixture with the same name also exists (#5036)""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=['a', 'b']) - def argroot(request): - return request.param - - @pytest.fixture - def arg(argroot): - return argroot + @pytest.fixture(params=["a", "b"]) + def argroot(request): + return request.param - # This should only be parametrized directly - @pytest.mark.parametrize("arg", [1]) - def test_direct(arg): - assert arg == 1 + @pytest.fixture + def arg(argroot): + return argroot - # This should be parametrized based on the fixtures - def test_normal_fixture(arg): - assert isinstance(arg, str) + # This should only be parametrized directly + @pytest.mark.parametrize("arg", [1]) + def test_direct(arg): + assert arg == 1 - # Indirect should still work: + # This should be parametrized based on the fixtures + def test_normal_fixture(arg): + assert isinstance(arg, str) - @pytest.fixture - def arg2(request): - return 2*request.param + # Indirect should still work: - @pytest.mark.parametrize("arg2", [1], indirect=True) - def test_indirect(arg2): - assert arg2 == 2 - """ + @pytest.fixture + def arg2(request): + return 2 * request.param + + @pytest.mark.parametrize("arg2", [1], indirect=True) + def test_indirect(arg2): + assert arg2 == 2 + + module = build_module( + "test_fixture_param_shadowing", + argroot, + test_direct, + test_normal_fixture, + test_indirect, + arg=arg, + arg2=arg2, ) + record = run_tests(module, rootpath=tmp_path) # Only one test should have run - result = pytester.runpytest("-v") - result.assert_outcomes(passed=4) - result.stdout.fnmatch_lines(["*::test_direct[[]1[]]*"]) - result.stdout.fnmatch_lines(["*::test_normal_fixture[[]a[]]*"]) - result.stdout.fnmatch_lines(["*::test_normal_fixture[[]b[]]*"]) - result.stdout.fnmatch_lines(["*::test_indirect[[]1[]]*"]) + record.assert_outcomes(passed=4) + assert sorted(record.by_test) == [ + "test_fixture_param_shadowing.py::test_direct[1]", + "test_fixture_param_shadowing.py::test_indirect[1]", + "test_fixture_param_shadowing.py::test_normal_fixture[a]", + "test_fixture_param_shadowing.py::test_normal_fixture[b]", + ] +# ensemble: asserts the file:line the reserved fixture name was used at. def test_fixture_named_request(pytester: Pytester) -> None: pytester.copy_example("fixtures/test_fixture_named_request.py") result = pytester.runpytest() @@ -5215,130 +5163,125 @@ def test_fixture_named_request(pytester: Pytester) -> None: ) -def test_indirect_fixture_does_not_break_scope(pytester: Pytester) -> None: +def test_indirect_fixture_does_not_break_scope(tmp_path: Path) -> None: """Ensure that fixture scope is respected when using indirect fixtures (#570)""" - pytester.makepyfile( - """ - import pytest - instantiated = [] - - @pytest.fixture(scope="session") - def fixture_1(request): - instantiated.append(("fixture_1", request.param)) + instantiated: list[tuple[str, str]] = [] + @pytest.fixture(scope="session") + def fixture_1(request): + instantiated.append(("fixture_1", request.param)) - @pytest.fixture(scope="session") - def fixture_2(request): - instantiated.append(("fixture_2", request.param)) + @pytest.fixture(scope="session") + def fixture_2(request): + instantiated.append(("fixture_2", request.param)) + + scenarios = [ + ("A", "a1"), + ("A", "a2"), + ("B", "b1"), + ("B", "b2"), + ("C", "c1"), + ("C", "c2"), + ] + @pytest.mark.parametrize( + "fixture_1,fixture_2", scenarios, indirect=["fixture_1", "fixture_2"] + ) + def test_create_fixtures(fixture_1, fixture_2): + pass - scenarios = [ - ("A", "a1"), - ("A", "a2"), - ("B", "b1"), - ("B", "b2"), - ("C", "c1"), - ("C", "c2"), + def test_check_fixture_instantiations(): + assert instantiated == [ + ("fixture_1", "A"), + ("fixture_2", "a1"), + ("fixture_2", "a2"), + ("fixture_1", "B"), + ("fixture_2", "b1"), + ("fixture_2", "b2"), + ("fixture_1", "C"), + ("fixture_2", "c1"), + ("fixture_2", "c2"), ] - @pytest.mark.parametrize( - "fixture_1,fixture_2", scenarios, indirect=["fixture_1", "fixture_2"] - ) - def test_create_fixtures(fixture_1, fixture_2): - pass - - - def test_check_fixture_instantiations(): - assert instantiated == [ - ('fixture_1', 'A'), - ('fixture_2', 'a1'), - ('fixture_2', 'a2'), - ('fixture_1', 'B'), - ('fixture_2', 'b1'), - ('fixture_2', 'b2'), - ('fixture_1', 'C'), - ('fixture_2', 'c1'), - ('fixture_2', 'c2'), - ] - """ + record = run_tests( + fixture_1, + fixture_2, + test_create_fixtures, + test_check_fixture_instantiations, + rootpath=tmp_path, ) - result = pytester.runpytest() - result.assert_outcomes(passed=7) + record.assert_outcomes(passed=7) -def test_fixture_parametrization_nparray(pytester: Pytester) -> None: - pytest.importorskip("numpy") +def test_fixture_parametrization_nparray(tmp_path: Path) -> None: + numpy = pytest.importorskip("numpy") - pytester.makepyfile( - """ - from numpy import linspace - from pytest import fixture + @pytest.fixture(params=numpy.linspace(1, 10, 10)) + def value(request): + return request.param - @fixture(params=linspace(1, 10, 10)) - def value(request): - return request.param + def test_bug(value): + assert value == value - def test_bug(value): - assert value == value - """ - ) - result = pytester.runpytest() - result.assert_outcomes(passed=10) + run_tests(value, test_bug, rootpath=tmp_path).assert_outcomes(passed=10) -def test_fixture_arg_ordering(pytester: Pytester) -> None: +def test_fixture_arg_ordering(tmp_path: Path) -> None: """ This test describes how fixtures in the same scope but without explicit dependencies between them are created. While users should make dependencies explicit, often they rely on this order, so this test exists to catch regressions in this regard. See #6540 and #6492. """ - p1 = pytester.makepyfile( - """ - import pytest + suffixes: list[str] = [] - suffixes = [] + @pytest.fixture + def fix_1(): + suffixes.append("fix_1") - @pytest.fixture - def fix_1(): suffixes.append("fix_1") - @pytest.fixture - def fix_2(): suffixes.append("fix_2") - @pytest.fixture - def fix_3(): suffixes.append("fix_3") - @pytest.fixture - def fix_4(): suffixes.append("fix_4") - @pytest.fixture - def fix_5(): suffixes.append("fix_5") + @pytest.fixture + def fix_2(): + suffixes.append("fix_2") - @pytest.fixture - def fix_combined(fix_1, fix_2, fix_3, fix_4, fix_5): pass + @pytest.fixture + def fix_3(): + suffixes.append("fix_3") - def test_suffix(fix_combined): - assert suffixes == ["fix_1", "fix_2", "fix_3", "fix_4", "fix_5"] - """ - ) - result = pytester.runpytest("-vv", str(p1)) - assert result.ret == 0 + @pytest.fixture + def fix_4(): + suffixes.append("fix_4") + @pytest.fixture + def fix_5(): + suffixes.append("fix_5") -def test_yield_fixture_with_no_value(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(name='custom') - def empty_yield(): - if False: - yield + @pytest.fixture + def fix_combined(fix_1, fix_2, fix_3, fix_4, fix_5): + pass - def test_fixt(custom): - pass - """ + def test_suffix(fix_combined): + assert suffixes == ["fix_1", "fix_2", "fix_3", "fix_4", "fix_5"] + + record = run_tests( + fix_1, fix_2, fix_3, fix_4, fix_5, fix_combined, test_suffix, rootpath=tmp_path ) - expected = "E ValueError: custom did not yield a value" - result = pytester.runpytest() - result.assert_outcomes(errors=1) - result.stdout.fnmatch_lines([expected]) - assert result.ret == ExitCode.TESTS_FAILED + record.assert_outcomes(passed=1) + + +def test_yield_fixture_with_no_value(tmp_path: Path) -> None: + @pytest.fixture(name="custom") + def empty_yield(): + if False: + yield # type: ignore[unreachable] + + def test_fixt(custom): + pass + + record = run_tests(empty_yield, test_fixt, rootpath=tmp_path) + record.assert_outcomes(errors=1) + setup = record["test_fixt"].setup + assert setup is not None + assert "ValueError: custom did not yield a value" in setup.longreprtext def test_deduplicate_names() -> None: @@ -5348,7 +5291,7 @@ def test_deduplicate_names() -> None: assert items == ("a", "b", "c", "d", "g", "f", "e") -def test_staticmethod_classmethod_fixture_instance(pytester: Pytester) -> None: +def test_staticmethod_classmethod_fixture_instance(tmp_path: Path) -> None: """Ensure that static and class methods get and have access to a fresh instance. @@ -5356,857 +5299,721 @@ def test_staticmethod_classmethod_fixture_instance(pytester: Pytester) -> None: Regression test for #12065. """ - pytester.makepyfile( - """ - import pytest - class Test: - ran_setup_method = False - ran_fixture = False + class Test: + ran_setup_method = False + ran_fixture = False - def setup_method(self): - assert not self.ran_setup_method - self.ran_setup_method = True + def setup_method(self): + assert not self.ran_setup_method + self.ran_setup_method = True - @pytest.fixture(autouse=True) - def fixture(self): - assert not self.ran_fixture - self.ran_fixture = True + @pytest.fixture(autouse=True) + def fixture(self): + assert not self.ran_fixture + self.ran_fixture = True - def test_method(self): - assert self.ran_setup_method - assert self.ran_fixture + def test_method(self): + assert self.ran_setup_method + assert self.ran_fixture + + @staticmethod + def test_1(request): + assert request.instance.ran_setup_method + assert request.instance.ran_fixture - @staticmethod - def test_1(request): - assert request.instance.ran_setup_method - assert request.instance.ran_fixture + @classmethod + def test_2(cls, request): + assert request.instance.ran_setup_method + assert request.instance.ran_fixture - @classmethod - def test_2(cls, request): - assert request.instance.ran_setup_method - assert request.instance.ran_fixture - """ - ) - result = pytester.runpytest() - assert result.ret == ExitCode.OK - result.assert_outcomes(passed=3) + run_tests(Test, rootpath=tmp_path).assert_outcomes(passed=3) -def test_scoped_fixture_caching(pytester: Pytester) -> None: +def test_scoped_fixture_caching(tmp_path: Path) -> None: """Make sure setup and finalization is only run once when using scoped fixture multiple times.""" - pytester.makepyfile( - """ - from __future__ import annotations - - from typing import Generator - - import pytest - executed: list[str] = [] - @pytest.fixture(scope="class") - def fixture_1() -> Generator[None, None, None]: - executed.append("fix setup") - yield - executed.append("fix teardown") + executed: list[str] = [] + @pytest.fixture(scope="class") + def fixture_1(): + executed.append("fix setup") + yield + executed.append("fix teardown") - class TestFixtureCaching: - def test_1(self, fixture_1: None) -> None: - assert executed == ["fix setup"] + class TestFixtureCaching: + def test_1(self, fixture_1: None) -> None: + assert executed == ["fix setup"] - def test_2(self, fixture_1: None) -> None: - assert executed == ["fix setup"] + def test_2(self, fixture_1: None) -> None: + assert executed == ["fix setup"] + def test_expected_setup_and_teardown() -> None: + assert executed == ["fix setup", "fix teardown"] - def test_expected_setup_and_teardown() -> None: - assert executed == ["fix setup", "fix teardown"] - """ + record = run_tests( + fixture_1, + TestFixtureCaching, + test_expected_setup_and_teardown, + rootpath=tmp_path, ) - result = pytester.runpytest() - assert result.ret == 0 + record.assert_outcomes(passed=3) -def test_scoped_fixture_caching_exception(pytester: Pytester) -> None: +def test_scoped_fixture_caching_exception(tmp_path: Path) -> None: """Make sure setup & finalization is only run once for scoped fixture, with a cached exception.""" - pytester.makepyfile( - """ - from __future__ import annotations - - import pytest - executed_crash: list[str] = [] - + executed_crash: list[str] = [] - @pytest.fixture(scope="class") - def fixture_crash(request: pytest.FixtureRequest) -> None: - executed_crash.append("fix_crash setup") - - def my_finalizer() -> None: - executed_crash.append("fix_crash teardown") + @pytest.fixture(scope="class") + def fixture_crash(request: pytest.FixtureRequest) -> None: + executed_crash.append("fix_crash setup") - request.addfinalizer(my_finalizer) + def my_finalizer() -> None: + executed_crash.append("fix_crash teardown") - raise Exception("foo") + request.addfinalizer(my_finalizer) + raise Exception("foo") - class TestFixtureCachingException: - @pytest.mark.xfail - def test_crash_1(self, fixture_crash: None) -> None: - ... + class TestFixtureCachingException: + @pytest.mark.xfail + def test_crash_1(self, fixture_crash: None) -> None: ... - @pytest.mark.xfail - def test_crash_2(self, fixture_crash: None) -> None: - ... + @pytest.mark.xfail + def test_crash_2(self, fixture_crash: None) -> None: ... + def test_crash_expected_setup_and_teardown() -> None: + assert executed_crash == ["fix_crash setup", "fix_crash teardown"] - def test_crash_expected_setup_and_teardown() -> None: - assert executed_crash == ["fix_crash setup", "fix_crash teardown"] - """ + record = run_tests( + fixture_crash, + TestFixtureCachingException, + test_crash_expected_setup_and_teardown, + rootpath=tmp_path, ) - result = pytester.runpytest() - assert result.ret == 0 + # The original only asserted a zero exit status; the two crashing tests + # are xfail, so they count as xfailed rather than errors. + record.assert_outcomes(passed=1, xfailed=2) -def test_scoped_fixture_teardown_order(pytester: Pytester) -> None: +def test_scoped_fixture_teardown_order(tmp_path: Path) -> None: """ Make sure teardowns happen in reverse order of setup with scoped fixtures, when a later test only depends on a subset of scoped fixtures. Regression test for https://github.com/pytest-dev/pytest/issues/1489 """ - pytester.makepyfile( - """ - from typing import Generator - - import pytest - - - last_executed = "" - - - @pytest.fixture(scope="module") - def fixture_1() -> Generator[None, None, None]: - global last_executed - assert last_executed == "" - last_executed = "fixture_1_setup" - yield - assert last_executed == "fixture_2_teardown" - last_executed = "fixture_1_teardown" - - - @pytest.fixture(scope="module") - def fixture_2() -> Generator[None, None, None]: - global last_executed - assert last_executed == "fixture_1_setup" - last_executed = "fixture_2_setup" - yield - assert last_executed == "run_test" - last_executed = "fixture_2_teardown" - - - def test_fixture_teardown_order(fixture_1: None, fixture_2: None) -> None: - global last_executed - assert last_executed == "fixture_2_setup" - last_executed = "run_test" - + # The original's module global becomes a one-element list. + last_executed = [""] + + @pytest.fixture(scope="module") + def fixture_1(): + assert last_executed[0] == "" + last_executed[0] = "fixture_1_setup" + yield + assert last_executed[0] == "fixture_2_teardown" + last_executed[0] = "fixture_1_teardown" + + @pytest.fixture(scope="module") + def fixture_2(): + assert last_executed[0] == "fixture_1_setup" + last_executed[0] = "fixture_2_setup" + yield + assert last_executed[0] == "run_test" + last_executed[0] = "fixture_2_teardown" + + def test_fixture_teardown_order(fixture_1: None, fixture_2: None) -> None: + assert last_executed[0] == "fixture_2_setup" + last_executed[0] = "run_test" + + def test_2(fixture_1: None) -> None: + # This would previously queue an additional teardown of fixture_1, + # despite fixture_1's value being cached, which caused fixture_1 to be + # torn down before fixture_2 - violating the rule that teardowns should + # happen in reverse order of setup. + pass - def test_2(fixture_1: None) -> None: - # This would previously queue an additional teardown of fixture_1, - # despite fixture_1's value being cached, which caused fixture_1 to be - # torn down before fixture_2 - violating the rule that teardowns should - # happen in reverse order of setup. - pass - """ + record = run_tests( + fixture_1, fixture_2, test_fixture_teardown_order, test_2, rootpath=tmp_path ) - result = pytester.runpytest() - assert result.ret == 0 + record.assert_outcomes(passed=2) + assert last_executed[0] == "fixture_1_teardown" -def test_subfixture_teardown_order(pytester: Pytester) -> None: +def test_subfixture_teardown_order(tmp_path: Path) -> None: """ Make sure fixtures don't re-register their finalization in parent fixtures multiple times, causing ordering failure in their teardowns. Regression test for #12135 """ - pytester.makepyfile( - """ - import pytest + execution_order: list[str] = [] - execution_order = [] + @pytest.fixture(scope="class") + def fixture_1(): ... - @pytest.fixture(scope="class") - def fixture_1(): - ... + @pytest.fixture(scope="class") + def fixture_2(fixture_1): + execution_order.append("setup 2") + yield + execution_order.append("teardown 2") - @pytest.fixture(scope="class") - def fixture_2(fixture_1): - execution_order.append("setup 2") - yield - execution_order.append("teardown 2") + @pytest.fixture(scope="class") + def fixture_3(fixture_1): + execution_order.append("setup 3") + yield + execution_order.append("teardown 3") - @pytest.fixture(scope="class") - def fixture_3(fixture_1): - execution_order.append("setup 3") - yield - execution_order.append("teardown 3") + class TestFoo: + def test_initialize_fixtures(self, fixture_2, fixture_3): ... - class TestFoo: - def test_initialize_fixtures(self, fixture_2, fixture_3): - ... + # This would previously reschedule fixture_2's finalizer in the parent fixture, + # causing it to be torn down before fixture 3. + def test_reschedule_fixture_2(self, fixture_2): ... - # This would previously reschedule fixture_2's finalizer in the parent fixture, - # causing it to be torn down before fixture 3. - def test_reschedule_fixture_2(self, fixture_2): - ... + # Force finalization directly on fixture_1 + # Otherwise the cleanup would sequence 3&2 before 1 as normal. + @pytest.mark.parametrize("fixture_1", [None], indirect=["fixture_1"]) + def test_finalize_fixture_1(self, fixture_1): ... - # Force finalization directly on fixture_1 - # Otherwise the cleanup would sequence 3&2 before 1 as normal. - @pytest.mark.parametrize("fixture_1", [None], indirect=["fixture_1"]) - def test_finalize_fixture_1(self, fixture_1): - ... + def test_result(): + assert execution_order == ["setup 2", "setup 3", "teardown 3", "teardown 2"] - def test_result(): - assert execution_order == ["setup 2", "setup 3", "teardown 3", "teardown 2"] - """ + record = run_tests( + fixture_1, fixture_2, fixture_3, TestFoo, test_result, rootpath=tmp_path ) - result = pytester.runpytest() - assert result.ret == 0 + record.assert_outcomes(passed=4) -def test_parametrized_fixture_scope_allowed(pytester: Pytester) -> None: +def test_parametrized_fixture_scope_allowed(tmp_path: Path) -> None: """ Make sure scope from parametrize does not affect fixture's ability to be depended upon. Regression test for #13248 """ - pytester.makepyfile( - """ - import pytest - - @pytest.fixture(scope="session") - def my_fixture(request): - return getattr(request, "param", None) - - @pytest.fixture(scope="session") - def another_fixture(my_fixture): - return my_fixture - - @pytest.mark.parametrize("my_fixture", ["a value"], indirect=True, scope="function") - def test_foo(another_fixture): - assert another_fixture == "a value" - """ - ) - result = pytester.runpytest() - result.assert_outcomes(passed=1) - -def test_collect_positional_only(pytester: Pytester) -> None: - """Support the collection of tests with positional-only arguments (#13376).""" - pytester.makepyfile( - """ - import pytest - - class Test: - @pytest.fixture - def fix(self): - return 1 + @pytest.fixture(scope="session") + def my_fixture(request): + return getattr(request, "param", None) - def test_method(self, /, fix): - assert fix == 1 - """ - ) - result = pytester.runpytest() - result.assert_outcomes(passed=1) + @pytest.fixture(scope="session") + def another_fixture(my_fixture): + return my_fixture + @pytest.mark.parametrize("my_fixture", ["a value"], indirect=True, scope="function") + def test_foo(another_fixture): + assert another_fixture == "a value" -def test_parametrization_dependency_pruning(pytester: Pytester) -> None: - """Test that when a fixture is dynamically shadowed by parameterization, it - is properly pruned and not executed.""" - pytester.makepyfile( - """ - import pytest + record = run_tests(my_fixture, another_fixture, test_foo, rootpath=tmp_path) + record.assert_outcomes(passed=1) - # This fixture should never run because shadowed_fixture is parametrized. +def test_collect_positional_only(tmp_path: Path) -> None: + """Support the collection of tests with positional-only arguments (#13376).""" + + class Test: @pytest.fixture - def boom(): - raise RuntimeError("BOOM!") + def fix(self): + return 1 + def test_method(self, /, fix): + assert fix == 1 - # This fixture is shadowed by metafunc.parametrize in pytest_generate_tests. - @pytest.fixture - def shadowed_fixture(boom): - return "fixture_value" + run_tests(Test, rootpath=tmp_path).assert_outcomes(passed=1) - # Dynamically parametrize shadowed_fixture, replacing the fixture with direct values. - def pytest_generate_tests(metafunc): - if "shadowed_fixture" in metafunc.fixturenames: - metafunc.parametrize("shadowed_fixture", ["param1", "param2"]) +def test_parametrization_dependency_pruning(tmp_path: Path) -> None: + """Test that when a fixture is dynamically shadowed by parameterization, it + is properly pruned and not executed.""" + # This fixture should never run because shadowed_fixture is parametrized. + @pytest.fixture + def boom(): + raise RuntimeError("BOOM!") - # This test should receive shadowed_fixture as a parametrized value, and - # boom should not explode. - def test_shadowed(shadowed_fixture): - assert shadowed_fixture in ["param1", "param2"] - """ + # This fixture is shadowed by metafunc.parametrize in pytest_generate_tests. + @pytest.fixture + def shadowed_fixture(boom): + return "fixture_value" + + # Dynamically parametrize shadowed_fixture, replacing the fixture with direct values. + def pytest_generate_tests(metafunc): + if "shadowed_fixture" in metafunc.fixturenames: + metafunc.parametrize("shadowed_fixture", ["param1", "param2"]) + + # This test should receive shadowed_fixture as a parametrized value, and + # boom should not explode. + def test_shadowed(shadowed_fixture): + assert shadowed_fixture in ["param1", "param2"] + + module = build_module( + "test_parametrization_dependency_pruning", + boom, + shadowed_fixture, + test_shadowed, + pytest_generate_tests=pytest_generate_tests, ) - result = pytester.runpytest() - result.assert_outcomes(passed=2) + run_tests(module, rootpath=tmp_path).assert_outcomes(passed=2) -def test_fixture_closure_with_overrides(pytester: Pytester) -> None: +def test_fixture_closure_with_overrides(tmp_path: Path) -> None: """Test that an item's static fixture closure properly includes transitive dependencies through overridden fixtures (#13773).""" - pytester.makeconftest( - """ - import pytest + class ConftestPlugin: @pytest.fixture - def db(): pass + def db(self): ... @pytest.fixture - def app(db): pass - """ - ) - pytester.makepyfile( - """ - import pytest + def app(self, db): ... + + # Overrides conftest-level `app` and requests it. + @pytest.fixture + def app(app): ... - # Overrides conftest-level `app` and requests it. + class TestClass: + # Overrides module-level `app` and requests it. @pytest.fixture - def app(app): pass + def app(self, app): ... - class TestClass: - # Overrides module-level `app` and requests it. - @pytest.fixture - def app(self, app): pass - - def test_something(self, request, app): - # Both dynamic and static fixture closures should include 'db'. - assert 'db' in request.fixturenames - assert 'db' in request.node.fixturenames - # No dynamic dependencies, should be equal. - assert set(request.fixturenames) == set(request.node.fixturenames) - """ - ) - result = pytester.runpytest("-v") - result.assert_outcomes(passed=1) + def test_something(self, request, app): + # Both dynamic and static fixture closures should include 'db'. + assert "db" in request.fixturenames + assert "db" in request.node.fixturenames + # No dynamic dependencies, should be equal. + assert set(request.fixturenames) == set(request.node.fixturenames) + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(build_module("test_overrides", TestClass, app=app), spec=spec) + record.assert_outcomes(passed=1) -def test_fixture_closure_with_overrides_and_intermediary(pytester: Pytester) -> None: +def test_fixture_closure_with_overrides_and_intermediary(tmp_path: Path) -> None: """Test that an item's static fixture closure properly includes transitive dependencies through overridden fixtures (#13773). A more complicated case than test_fixture_closure_with_overrides, adds an intermediary so the override chain is not direct. """ - pytester.makeconftest( - """ - import pytest + class ConftestPlugin: @pytest.fixture - def db(): pass + def db(self): ... @pytest.fixture - def app(db): pass + def app(self, db): ... @pytest.fixture - def intermediate(app): pass - """ - ) - pytester.makepyfile( - """ - import pytest + def intermediate(self, app): ... - # Overrides conftest-level `app` and requests it. + # Overrides conftest-level `app` and requests it. + @pytest.fixture + def app(intermediate): ... + + class TestClass: + # Overrides module-level `app` and requests it. @pytest.fixture - def app(intermediate): pass + def app(self, app): ... - class TestClass: - # Overrides module-level `app` and requests it. - @pytest.fixture - def app(self, app): pass - - def test_something(self, request, app): - # Both dynamic and static fixture closures should include 'db'. - assert 'db' in request.fixturenames - assert 'db' in request.node.fixturenames - # No dynamic dependencies, should be equal. - assert set(request.fixturenames) == set(request.node.fixturenames) - """ - ) - result = pytester.runpytest("-v") - result.assert_outcomes(passed=1) + def test_something(self, request, app): + # Both dynamic and static fixture closures should include 'db'. + assert "db" in request.fixturenames + assert "db" in request.node.fixturenames + # No dynamic dependencies, should be equal. + assert set(request.fixturenames) == set(request.node.fixturenames) + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(build_module("test_intermediary", TestClass, app=app), spec=spec) + record.assert_outcomes(passed=1) -def test_fixture_closure_with_overrides_and_parametrization(pytester: Pytester) -> None: + +def test_fixture_closure_with_overrides_and_parametrization(tmp_path: Path) -> None: """Test that an item's static fixture closure properly includes transitive dependencies through overridden fixtures (#13773) when also including parametrization (#14248).""" - pytester.makeconftest( - """ - import pytest + class ConftestPlugin: @pytest.fixture - def db(): pass + def db(self): ... @pytest.fixture - def app(db): pass - """ - ) - pytester.makepyfile( - """ - import pytest + def app(self, db): ... - # Overrides conftest-level `app` and requests it. - @pytest.fixture - def app(app): pass + # Overrides conftest-level `app` and requests it. + @pytest.fixture + def app(app): ... - class TestClass: - # Overrides module-level `app` and requests it. - @pytest.fixture - def app(self, app): pass - - @pytest.mark.parametrize("a", [1]) - def test_something(self, request, app, a): - # Both dynamic and static fixture closures should include 'db'. - assert 'db' in request.fixturenames - assert 'db' in request.node.fixturenames - # No dynamic dependencies, should be equal. - assert set(request.fixturenames) == set(request.node.fixturenames) - """ + class TestClass: + # Overrides module-level `app` and requests it. + @pytest.fixture + def app(self, app): ... + + @pytest.mark.parametrize("a", [1]) + def test_something(self, request, app, a): + # Both dynamic and static fixture closures should include 'db'. + assert "db" in request.fixturenames + assert "db" in request.node.fixturenames + # No dynamic dependencies, should be equal. + assert set(request.fixturenames) == set(request.node.fixturenames) + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests( + build_module("test_parametrized_override", TestClass, app=app), spec=spec ) - result = pytester.runpytest("-v") - result.assert_outcomes(passed=1) + record.assert_outcomes(passed=1) -def test_fixture_closure_with_broken_override_chain(pytester: Pytester) -> None: +def test_fixture_closure_with_broken_override_chain(tmp_path: Path) -> None: """Test that an item's static fixture closure properly includes transitive dependencies through overridden fixtures (#13773). A more complicated case than test_fixture_closure_with_overrides, one of the fixtures in the chain doesn't call its super, so it shouldn't be included. """ - pytester.makeconftest( - """ - import pytest + class ConftestPlugin: @pytest.fixture - def db(): pass + def db(self): ... @pytest.fixture - def app(db): pass - """ - ) - pytester.makepyfile( - """ - import pytest + def app(self, db): ... + + # Overrides conftest-level `app` and *doesn't* request it. + @pytest.fixture + def app(): ... - # Overrides conftest-level `app` and *doesn't* request it. + class TestClass: + # Overrides module-level `app` and requests it. @pytest.fixture - def app(): pass + def app(self, app): ... - class TestClass: - # Overrides module-level `app` and requests it. - @pytest.fixture - def app(self, app): pass - - def test_something(self, request, app): - # Both dynamic and static fixture closures should include 'db'. - assert 'db' not in request.fixturenames - assert 'db' not in request.node.fixturenames - # No dynamic dependencies, should be equal. - assert set(request.fixturenames) == set(request.node.fixturenames) - """ - ) - result = pytester.runpytest("-v") - result.assert_outcomes(passed=1) + def test_something(self, request, app): + # Both dynamic and static fixture closures should include 'db'. + assert "db" not in request.fixturenames + assert "db" not in request.node.fixturenames + # No dynamic dependencies, should be equal. + assert set(request.fixturenames) == set(request.node.fixturenames) + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(build_module("test_broken_chain", TestClass, app=app), spec=spec) + record.assert_outcomes(passed=1) -def test_fixture_closure_handles_circular_dependencies(pytester: Pytester) -> None: +def test_fixture_closure_handles_circular_dependencies(tmp_path: Path) -> None: """Test that getfixtureclosure properly handles circular dependencies. The test will error in the runtest phase due to the fixture loop, but the closure computation still completes. """ - pytester.makepyfile( - """ - import pytest - # Direct circular dependency. - @pytest.fixture - def fix_a(fix_b): pass + # Direct circular dependency. + @pytest.fixture + def fix_a(fix_b): ... - @pytest.fixture - def fix_b(fix_a): pass + @pytest.fixture + def fix_b(fix_a): ... - # Indirect circular dependency through multiple fixtures. - @pytest.fixture - def fix_x(fix_y): pass + # Indirect circular dependency through multiple fixtures. + @pytest.fixture + def fix_x(fix_y): ... - @pytest.fixture - def fix_y(fix_z): pass + @pytest.fixture + def fix_y(fix_z): ... - @pytest.fixture - def fix_z(fix_x): pass + @pytest.fixture + def fix_z(fix_x): ... - def test_circular_deps(fix_a, fix_x): - pass - """ + def test_circular_deps(fix_a, fix_x): + pass + + items = collect_tests( + fix_a, fix_b, fix_x, fix_y, fix_z, test_circular_deps, rootpath=tmp_path ) - items, _hookrec = pytester.inline_genitems() assert isinstance(items[0], Function) assert items[0].fixturenames == ["fix_a", "fix_b", "fix_x", "fix_y", "fix_z"] -def test_fixture_closure_handles_diamond_dependencies(pytester: Pytester) -> None: +def test_fixture_closure_handles_diamond_dependencies(tmp_path: Path) -> None: """Test that getfixtureclosure properly handles diamond dependencies.""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def db(): pass + @pytest.fixture + def db(): ... - @pytest.fixture - def user(db): pass + @pytest.fixture + def user(db): ... - @pytest.fixture - def session(db): pass + @pytest.fixture + def session(db): ... - @pytest.fixture - def app(user, session): pass + @pytest.fixture + def app(user, session): ... + + def test_diamond_deps(request, app): + assert request.node.fixturenames == [ + "request", + "app", + "user", + "db", + "session", + ] + assert request.fixturenames == ["request", "app", "user", "db", "session"] - def test_diamond_deps(request, app): - assert request.node.fixturenames == ["request", "app", "user", "db", "session"] - assert request.fixturenames == ["request", "app", "user", "db", "session"] - """ - ) - result = pytester.runpytest("-v") - result.assert_outcomes(passed=1) + record = run_tests(db, user, session, app, test_diamond_deps, rootpath=tmp_path) + record.assert_outcomes(passed=1) def test_fixture_closure_with_complex_override_and_shared_deps( - pytester: Pytester, + tmp_path: Path, ) -> None: """Test that shared dependencies in override chains are processed only once.""" - pytester.makeconftest( - """ - import pytest + class ConftestPlugin: @pytest.fixture - def db(): pass + def db(self): ... @pytest.fixture - def cache(): pass + def cache(self): ... @pytest.fixture - def settings(): pass + def settings(self): ... @pytest.fixture - def app(db, cache, settings): pass - """ - ) - pytester.makepyfile( - """ - import pytest + def app(self, db, cache, settings): ... - # Override app, but also directly use cache and settings. - # This creates multiple paths to the same fixtures. - @pytest.fixture - def app(app, cache, settings): pass + # Override app, but also directly use cache and settings. + # This creates multiple paths to the same fixtures. + @pytest.fixture + def app(app, cache, settings): ... - class TestClass: - # Another override that uses both app and cache. - @pytest.fixture - def app(self, app, cache): pass + class TestClass: + # Another override that uses both app and cache. + @pytest.fixture + def app(self, app, cache): ... + + def test_shared_deps(self, request, app): + assert request.node.fixturenames == [ + "request", + "app", + "db", + "cache", + "settings", + ] - def test_shared_deps(self, request, app): - assert request.node.fixturenames == ["request", "app", "db", "cache", "settings"] - """ - ) - result = pytester.runpytest("-v") - result.assert_outcomes(passed=1) + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(build_module("test_shared_deps", TestClass, app=app), spec=spec) + record.assert_outcomes(passed=1) -def test_fixture_closure_with_parametrize_ignore(pytester: Pytester) -> None: +def test_fixture_closure_with_parametrize_ignore(tmp_path: Path) -> None: """Test that getfixtureclosure properly handles parametrization argnames which override a fixture.""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def fix1(fix2): pass + @pytest.fixture + def fix1(fix2): ... - @pytest.fixture - def fix2(fix3): pass + @pytest.fixture + def fix2(fix3): ... - @pytest.fixture - def fix3(): pass + @pytest.fixture + def fix3(): ... - @pytest.mark.parametrize('fix2', ['2']) - def test_it(request, fix1): - assert request.node.fixturenames == ["request", "fix1", "fix2"] - assert request.fixturenames == ["request", "fix1", "fix2"] - """ - ) - result = pytester.runpytest("-v") - result.assert_outcomes(passed=1) + @pytest.mark.parametrize("fix2", ["2"]) + def test_it(request, fix1): + assert request.node.fixturenames == ["request", "fix1", "fix2"] + assert request.fixturenames == ["request", "fix1", "fix2"] + record = run_tests(fix1, fix2, fix3, test_it, rootpath=tmp_path) + record.assert_outcomes(passed=1) -def test_overridden_fixture_depends_on_parametrized(pytester: Pytester) -> None: + +def test_overridden_fixture_depends_on_parametrized(tmp_path: Path) -> None: """#11075""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=["foo"]) - def fixture_foo(request): - yield request.param + @pytest.fixture(params=["foo"]) + def fixture_foo(request): + yield request.param + + @pytest.fixture + def fixture_bar(fixture_foo): + yield fixture_foo + class TestFoobar: @pytest.fixture - def fixture_bar(fixture_foo): - yield fixture_foo + def fixture_bar(self, fixture_bar): + yield fixture_bar - class TestFoobar: - @pytest.fixture - def fixture_bar(self, fixture_bar): - yield fixture_bar + def test_foobar(self, fixture_bar): + assert fixture_bar == "foo" - def test_foobar(self, fixture_bar): - assert fixture_bar == "foo" - """ - ) - result = pytester.runpytest("-v") - result.assert_outcomes(passed=1) + record = run_tests(fixture_foo, fixture_bar, TestFoobar, rootpath=tmp_path) + record.assert_outcomes(passed=1) -@pytest.mark.filterwarnings("default:cannot discover fixture *:pytest.PytestWarning") -def test_custom_decorated_fixture_warning(pytester: Pytester) -> None: - """Fixtures wrapped by custom decorators using functools.wraps warn.""" - pytester.makepyfile( - """ - import pytest - import functools +def _custom_deco(func): + import functools - def custom_deco(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - return wrapper + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) - class TestClass: - @custom_deco - @pytest.fixture - def my_fixture(self): - return "fixture_value" + return wrapper - def test_fixture_usage(self, my_fixture): - assert my_fixture == "fixture_value" - """ - ) - result = pytester.runpytest_inprocess( - "-v", "-rw", "-W", "default::pytest.PytestWarning" - ) - result.stdout.fnmatch_lines( - [ - "*test_custom_decorated_fixture_warning.py:*: " - "PytestWarning: cannot discover fixture 'my_fixture' " - "due to being wrapped in decorators*" - ] - ) +#: An ensemble that must see fixture-discovery warnings: the ensemble's own +#: filters take precedence over the host suite's ``filterwarnings = error``. +_WARN_INICFG = {"filterwarnings": ["default::pytest.PytestWarning"]} - result.stdout.fnmatch_lines(["*fixture 'my_fixture' not found*"]) - result.assert_outcomes(errors=1) +def test_custom_decorated_fixture_warning(tmp_path: Path) -> None: + """Fixtures wrapped by custom decorators using functools.wraps warn.""" + + class TestClass: + @_custom_deco + @pytest.fixture + def my_fixture(self): + return "fixture_value" -@pytest.mark.filterwarnings("default:cannot discover fixture *:pytest.PytestWarning") -def test_custom_decorated_fixture_above_classmethod_warning( - pytester: Pytester, -) -> None: + def test_fixture_usage(self, my_fixture): + assert my_fixture == "fixture_value" + + spec = ConfigSpec(rootpath=tmp_path, inicfg=_WARN_INICFG) + record = run_tests(TestClass, spec=spec, capture_output=True) + # The original matched the warning's file:line too, which is host-anchored + # for an in-memory source. + assert [str(w.message) for w in record.warnings] == [ + "cannot discover fixture 'my_fixture' due to being wrapped in decorators" + ] + record.stdout.fnmatch_lines(["*fixture 'my_fixture' not found*"]) + record.assert_outcomes(errors=1, warnings=1) + + +def test_custom_decorated_fixture_above_classmethod_warning(tmp_path: Path) -> None: """Warn when wraps hides a fixture that itself wraps @classmethod. The fixture definition stores the classmethod descriptor; warning emission peels it to reach the underlying function for warn_explicit_for. """ - pytester.makepyfile( - """ - import pytest - import functools - def custom_deco(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - return wrapper - - class TestClass: - @custom_deco - @pytest.fixture(scope="class") - @classmethod - def my_fixture(cls): - return "fixture_value" + class TestClass: + @_custom_deco + @pytest.fixture(scope="class") + @classmethod + def my_fixture(cls): + return "fixture_value" - def test_fixture_usage(self, my_fixture): - assert my_fixture == "fixture_value" - """ - ) - result = pytester.runpytest_inprocess( - "-v", "-rw", "-W", "default::pytest.PytestWarning" - ) + def test_fixture_usage(self, my_fixture): + assert my_fixture == "fixture_value" - result.stdout.fnmatch_lines( - [ - "*PytestWarning: cannot discover fixture 'my_fixture' " - "due to being wrapped in decorators*" - ] - ) - result.stdout.fnmatch_lines(["*fixture 'my_fixture' not found*"]) - result.assert_outcomes(errors=1) + spec = ConfigSpec(rootpath=tmp_path, inicfg=_WARN_INICFG) + record = run_tests(TestClass, spec=spec, capture_output=True) + assert [str(w.message) for w in record.warnings] == [ + "cannot discover fixture 'my_fixture' due to being wrapped in decorators" + ] + record.stdout.fnmatch_lines(["*fixture 'my_fixture' not found*"]) + record.assert_outcomes(errors=1, warnings=1) -@pytest.mark.filterwarnings("default:cannot discover fixture *:pytest.PytestWarning") -def test_classmethod_above_fixture_warning(pytester: Pytester) -> None: +def test_classmethod_above_fixture_warning(tmp_path: Path) -> None: """@classmethod above @pytest.fixture hides the fixture (#13507).""" - pytester.makepyfile( - """ - import pytest - class TestFixture: - @classmethod - @pytest.fixture(scope="class") - def fixt(cls): - return 1 + class TestFixture: + @classmethod + @pytest.fixture(scope="class") + def fixt(cls): + return 1 - def test_fixt(self, fixt): - assert fixt == 1 - """ - ) - result = pytester.runpytest_inprocess( - "-v", "-rw", "-W", "default::pytest.PytestWarning" - ) + def test_fixt(self, fixt): + assert fixt == 1 - result.stdout.fnmatch_lines( - [ - "*test_classmethod_above_fixture_warning.py:*: " - "PytestWarning: cannot discover fixture 'fixt' because it is " - "wrapped by @classmethod; place @pytest.fixture above @classmethod*" - ] - ) - result.stdout.fnmatch_lines(["*fixture 'fixt' not found*"]) - result.assert_outcomes(errors=1) + spec = ConfigSpec(rootpath=tmp_path, inicfg=_WARN_INICFG) + record = run_tests(TestFixture, spec=spec, capture_output=True) + assert [str(w.message) for w in record.warnings] == [ + "cannot discover fixture 'fixt' because it is wrapped by @classmethod; " + "place @pytest.fixture above @classmethod" + ] + record.stdout.fnmatch_lines(["*fixture 'fixt' not found*"]) + record.assert_outcomes(errors=1, warnings=1) -@pytest.mark.filterwarnings( - "default:fixture * is wrapped by @staticmethod*:pytest.PytestWarning" -) -def test_staticmethod_above_fixture_warning(pytester: Pytester) -> None: +def test_staticmethod_above_fixture_warning(tmp_path: Path) -> None: """@staticmethod above @pytest.fixture always warns. Unlike ``classmethod``, discovery still finds the fixture via ``staticmethod.__get__``, so the test can pass; a leading ``self``/``cls`` already fails as a missing fixture without special-casing here. """ - pytester.makepyfile( - """ - import pytest - class TestFixture: - @staticmethod - @pytest.fixture - def fixt(): - return 1 + class TestFixture: + @staticmethod + @pytest.fixture + def fixt(): + return 1 - def test_fixt(self, fixt): - assert fixt == 1 - """ - ) - result = pytester.runpytest_inprocess( - "-v", "-rw", "-W", "default::pytest.PytestWarning" - ) + def test_fixt(self, fixt): + assert fixt == 1 - result.stdout.fnmatch_lines( - [ - "*test_staticmethod_above_fixture_warning.py:*: " - "PytestWarning: fixture 'fixt' is wrapped by @staticmethod above " - "@pytest.fixture; place @pytest.fixture above @staticmethod*" - ] - ) - result.assert_outcomes(passed=1) + spec = ConfigSpec(rootpath=tmp_path, inicfg=_WARN_INICFG) + record = run_tests(TestFixture, spec=spec) + assert [str(w.message) for w in record.warnings] == [ + "fixture 'fixt' is wrapped by @staticmethod above @pytest.fixture; " + "place @pytest.fixture above @staticmethod" + ] + record.assert_outcomes(passed=1, warnings=1) -def test_fixture_above_classmethod_still_works(pytester: Pytester) -> None: +def test_fixture_above_classmethod_still_works(tmp_path: Path) -> None: """Documented order @pytest.fixture above @classmethod remains discoverable.""" - pytester.makepyfile( - """ - import pytest - class TestFixture: - @pytest.fixture(scope="class") - @classmethod - def fixt(cls): - return 1 + class TestFixture: + @pytest.fixture(scope="class") + @classmethod + def fixt(cls): + return 1 - def test_fixt(self, fixt): - assert fixt == 1 - """ - ) - result = pytester.runpytest("-v") - result.assert_outcomes(passed=1) + def test_fixt(self, fixt): + assert fixt == 1 + run_tests(TestFixture, rootpath=tmp_path).assert_outcomes(passed=1) -def test_fixture_above_staticmethod_still_works(pytester: Pytester) -> None: + +def test_fixture_above_staticmethod_still_works(tmp_path: Path) -> None: """@pytest.fixture above @staticmethod remains discoverable without warning.""" - pytester.makepyfile( - """ - import pytest - class TestFixture: - @pytest.fixture - @staticmethod - def fixt(): - return 1 + class TestFixture: + @pytest.fixture + @staticmethod + def fixt(): + return 1 - def test_fixt(self, fixt): - assert fixt == 1 - """ + def test_fixt(self, fixt): + assert fixt == 1 + + # -W error::pytest.PytestWarning of the original. + spec = ConfigSpec( + rootpath=tmp_path, inicfg={"filterwarnings": ["error::pytest.PytestWarning"]} ) - result = pytester.runpytest("-W", "error::pytest.PytestWarning", "-v") - result.assert_outcomes(passed=1) + run_tests(TestFixture, spec=spec).assert_outcomes(passed=1, warnings=0) -@pytest.mark.filterwarnings("default:cannot discover fixture *:pytest.PytestWarning") -def test_classmethod_above_fixture_warning_inherited(pytester: Pytester) -> None: +def test_classmethod_above_fixture_warning_inherited(tmp_path: Path) -> None: """MRO ``__dict__`` lookup finds @classmethod wrappers on a base class.""" - pytester.makepyfile( - """ - import pytest - - class Base: - @classmethod - @pytest.fixture(scope="class") - def fixt(cls): - return 1 - - class TestFixture(Base): - def test_fixt(self, fixt): - assert fixt == 1 - """ - ) - result = pytester.runpytest_inprocess( - "-v", "-rw", "-W", "default::pytest.PytestWarning" - ) - result.stdout.fnmatch_lines( - [ - "*PytestWarning: cannot discover fixture 'fixt' because it is " - "wrapped by @classmethod; place @pytest.fixture above @classmethod*" - ] - ) - result.stdout.fnmatch_lines(["*fixture 'fixt' not found*"]) - result.assert_outcomes(errors=1) + class Base: + @classmethod + @pytest.fixture(scope="class") + def fixt(cls): + return 1 + + class TestFixture(Base): + def test_fixt(self, fixt): + assert fixt == 1 + + spec = ConfigSpec(rootpath=tmp_path, inicfg=_WARN_INICFG) + record = run_tests(TestFixture, spec=spec, capture_output=True) + assert [str(w.message) for w in record.warnings] == [ + "cannot discover fixture 'fixt' because it is wrapped by @classmethod; " + "place @pytest.fixture above @classmethod" + ] + record.stdout.fnmatch_lines(["*fixture 'fixt' not found*"]) + record.assert_outcomes(errors=1, warnings=1) From cd816c6cff321523cdbfbef5a718d8b45bd339ab Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 19:38:35 +0200 Subject: [PATCH 02/30] testing: port metafunc.py to _pytest.ensemble 62 of 67 pytester-driven tests ported; the ~54 that already drive Metafunc and IdMaker directly are untouched. Module- and class-level pytest_generate_tests work natively because the synthesized module is a real module object, and conftest-level hooks come over as plugin objects in extra_plugins. Most -v/fnmatch_lines assertions here only checked parametrization ids and pass/fail, so they became exact collect_tests() name lists and record[...] lookups - stronger than the globs they replace. Four rendering-centric tests keep their fnmatch_lines verbatim via capture_output=True. The xfail tests asserted `skipped` because HookRecorder lumps xfails in with skips; they now assert `xfailed`, with a note at each site so the change does not read as a mistake. Four stay on pytester: conftest directory scoping, a misspelled marker that raises against the host config at decoration time, a package-scope reordering test that would go green while exercising a different path, and a subprocess test. Also fixes a trap this port exposed: Pytester.genitems() does not reorder, but the full collection protocol does, so a test unpacking items positionally silently got the wrong ones. It now looks them up by name. `hellow` joins the codespell ignore list: it is a deliberate parametrize id that predates this change, and codespell only scans touched files. --- pyproject.toml | 2 +- testing/python/metafunc.py | 2133 ++++++++++++++++++------------------ 2 files changed, 1094 insertions(+), 1041 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b467ed0fba0..ac4992ace7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -369,7 +369,7 @@ disable = [ ] [tool.codespell] -ignore-words-list = "afile,asend,asser,assertio,feld,hove,ned,noes,notin,paramete,parth,tesults,varius,wil" +ignore-words-list = "afile,asend,asser,assertio,feld,hellow,hove,ned,noes,notin,paramete,parth,tesults,varius,wil" skip = "AUTHORS,*/plugin_list.rst" write-changes = true diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 89577901141..d8dab8486f4 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -21,6 +21,8 @@ from _pytest.compat import getfuncargnames from _pytest.compat import NOTSET from _pytest.ensemble import build_module +from _pytest.ensemble import collect_tests +from _pytest.ensemble import ConfigSpec from _pytest.ensemble import run_tests from _pytest.outcomes import fail from _pytest.outcomes import Failed @@ -211,7 +213,7 @@ def func(x): ): metafunc.parametrize("x", [1], scope="doggy") # type: ignore[arg-type] - def test_parametrize_request_name(self, pytester: Pytester) -> None: + def test_parametrize_request_name(self) -> None: """Show proper error when 'request' is used as a parameter name in parametrize (#6183)""" def func(request): @@ -788,58 +790,49 @@ def test_idmaker_duplicated_empty_str(self) -> None: ).make_unique_parameterset_ids() assert result == ["0", "1"] - def test_parametrize_ids_exception(self, pytester: Pytester) -> None: - """ - :param pytester: the instance of Pytester class, a temporary - test directory. - """ - pytester.makepyfile( - """ - import pytest + def test_parametrize_ids_exception(self, tmp_path: Path) -> None: + """An ids callable that raises reports which parameter it choked on.""" - def ids(arg): - raise Exception("bad ids") + def ids(arg): + raise Exception("bad ids") - @pytest.mark.parametrize("arg", ["a", "b"], ids=ids) - def test_foo(arg): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + @pytest.mark.parametrize("arg", ["a", "b"], ids=ids) + def test_foo(arg): + pass + + # ensemble: the module name is part of the reported nodeid. + module = build_module("test_parametrize_ids_exception", test_foo) + record = run_tests(module, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( [ "*Exception: bad ids", "*test_foo: error raised while trying to determine id of parameter 'arg' at position 0", ] ) - def test_parametrize_ids_returns_non_string(self, pytester: Pytester) -> None: - pytester.makepyfile( - """\ - import pytest + def test_parametrize_ids_returns_non_string(self, tmp_path: Path) -> None: + def ids(d): + return d - def ids(d): - return d + @pytest.mark.parametrize("arg", ({1: 2}, {3, 4}), ids=ids) + def test(arg): + assert arg - @pytest.mark.parametrize("arg", ({1: 2}, {3, 4}), ids=ids) - def test(arg): - assert arg + @pytest.mark.parametrize("arg", (1, 2.0, True), ids=ids) + def test_int(arg): + assert arg - @pytest.mark.parametrize("arg", (1, 2.0, True), ids=ids) - def test_int(arg): - assert arg - """ - ) - result = pytester.runpytest("-vv", "-s") - result.stdout.fnmatch_lines( - [ - "test_parametrize_ids_returns_non_string.py::test[arg0] PASSED", - "test_parametrize_ids_returns_non_string.py::test[arg1] PASSED", - "test_parametrize_ids_returns_non_string.py::test_int[1] PASSED", - "test_parametrize_ids_returns_non_string.py::test_int[2.0] PASSED", - "test_parametrize_ids_returns_non_string.py::test_int[True] PASSED", - ] - ) + module = build_module("test_parametrize_ids_returns_non_string", test, test_int) + record = run_tests(module, rootpath=tmp_path) + assert list(record.by_test) == [ + "test_parametrize_ids_returns_non_string.py::test[arg0]", + "test_parametrize_ids_returns_non_string.py::test[arg1]", + "test_parametrize_ids_returns_non_string.py::test_int[1]", + "test_parametrize_ids_returns_non_string.py::test_int[2.0]", + "test_parametrize_ids_returns_non_string.py::test_int[True]", + ] + record.assert_outcomes(passed=5) def test_idmaker_with_ids(self) -> None: result = IdMaker( @@ -945,33 +938,37 @@ def func(x, y): with pytest.raises(TypeError, match="positional arguments"): metafunc.parametrize("x, y", [("a", "b")], ["x"]) # type: ignore[call-arg] - def test_parametrize_indirect_list_functional(self, pytester: Pytester) -> None: + def test_parametrize_indirect_list_functional(self, tmp_path: Path) -> None: """ #714 Test parametrization with 'indirect' parameter applied on particular arguments. As y is direct, its value should be used directly rather than being passed to the fixture y. - - :param pytester: the instance of Pytester class, a temporary - test directory. - """ - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope='function') - def x(request): - return request.param * 3 - @pytest.fixture(scope='function') - def y(request): - return request.param * 2 - @pytest.mark.parametrize('x, y', [('a', 'b')], indirect=['x']) - def test_simple(x,y): - assert len(x) == 3 - assert len(y) == 1 """ + + @pytest.fixture(scope="function") + def x(request): + return request.param * 3 + + @pytest.fixture(scope="function") + def y(request): + return request.param * 2 + + @pytest.mark.parametrize("x, y", [("a", "b")], indirect=["x"]) + def test_simple(x, y): + assert len(x) == 3 + assert len(y) == 1 + + record = run_tests( + build_module( + "test_parametrize_indirect_list_functional", x, y, test_simple + ), + rootpath=tmp_path, ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines(["*test_simple*a-b*", "*1 passed*"]) + assert list(record.by_test) == [ + "test_parametrize_indirect_list_functional.py::test_simple[a-b]" + ] + record.assert_outcomes(passed=1) def test_parametrize_indirect_list_error(self) -> None: """#714""" @@ -984,7 +981,7 @@ def func(x, y): metafunc.parametrize("x, y", [("a", "b")], indirect=["x", "z"]) def test_parametrize_uses_no_fixture_error_indirect_false( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """The 'uses no fixture' error tells the user at collection time that the parametrize data they've set up doesn't correspond to the @@ -993,134 +990,117 @@ def test_parametrize_uses_no_fixture_error_indirect_false( #714 """ - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize('x, y', [('a', 'b')], indirect=False) - def test_simple(x): - assert len(x) == 3 - """ - ) - result = pytester.runpytest("--collect-only") - result.stdout.fnmatch_lines(["*uses no argument 'y'*"]) + @pytest.mark.parametrize("x, y", [("a", "b")], indirect=False) + def test_simple(x): + assert len(x) == 3 + + with pytest.raises(pytest.Collector.CollectError, match="uses no argument 'y'"): + collect_tests(test_simple, rootpath=tmp_path) def test_parametrize_uses_no_fixture_error_indirect_true( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """#714""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope='function') - def x(request): - return request.param * 3 - @pytest.fixture(scope='function') - def y(request): - return request.param * 2 - - @pytest.mark.parametrize('x, y', [('a', 'b')], indirect=True) - def test_simple(x): - assert len(x) == 3 - """ - ) - result = pytester.runpytest("--collect-only") - result.stdout.fnmatch_lines(["*uses no fixture 'y'*"]) + + @pytest.fixture(scope="function") + def x(request): + return request.param * 3 + + @pytest.fixture(scope="function") + def y(request): + return request.param * 2 + + @pytest.mark.parametrize("x, y", [("a", "b")], indirect=True) + def test_simple(x): + assert len(x) == 3 + + with pytest.raises(pytest.Collector.CollectError, match="uses no fixture 'y'"): + collect_tests(x, y, test_simple, rootpath=tmp_path) def test_parametrize_indirect_uses_no_fixture_error_indirect_string( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """#714""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope='function') - def x(request): - return request.param * 3 - @pytest.mark.parametrize('x, y', [('a', 'b')], indirect='y') - def test_simple(x): - assert len(x) == 3 - """ - ) - result = pytester.runpytest("--collect-only") - result.stdout.fnmatch_lines(["*uses no fixture 'y'*"]) + @pytest.fixture(scope="function") + def x(request): + return request.param * 3 + + @pytest.mark.parametrize("x, y", [("a", "b")], indirect="y") + def test_simple(x): + assert len(x) == 3 + + with pytest.raises(pytest.Collector.CollectError, match="uses no fixture 'y'"): + collect_tests(x, test_simple, rootpath=tmp_path) def test_parametrize_indirect_uses_no_fixture_error_indirect_list( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """#714""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope='function') - def x(request): - return request.param * 3 - @pytest.mark.parametrize('x, y', [('a', 'b')], indirect=['y']) - def test_simple(x): - assert len(x) == 3 - """ - ) - result = pytester.runpytest("--collect-only") - result.stdout.fnmatch_lines(["*uses no fixture 'y'*"]) + @pytest.fixture(scope="function") + def x(request): + return request.param * 3 - def test_parametrize_argument_not_in_indirect_list( - self, pytester: Pytester - ) -> None: + @pytest.mark.parametrize("x, y", [("a", "b")], indirect=["y"]) + def test_simple(x): + assert len(x) == 3 + + with pytest.raises(pytest.Collector.CollectError, match="uses no fixture 'y'"): + collect_tests(x, test_simple, rootpath=tmp_path) + + def test_parametrize_argument_not_in_indirect_list(self, tmp_path: Path) -> None: """#714""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope='function') - def x(request): - return request.param * 3 - @pytest.mark.parametrize('x, y', [('a', 'b')], indirect=['x']) - def test_simple(x): - assert len(x) == 3 - """ - ) - result = pytester.runpytest("--collect-only") - result.stdout.fnmatch_lines(["*uses no argument 'y'*"]) + @pytest.fixture(scope="function") + def x(request): + return request.param * 3 + + @pytest.mark.parametrize("x, y", [("a", "b")], indirect=["x"]) + def test_simple(x): + assert len(x) == 3 + + with pytest.raises(pytest.Collector.CollectError, match="uses no argument 'y'"): + collect_tests(x, test_simple, rootpath=tmp_path) def test_parametrize_gives_indicative_error_on_function_with_default_argument( - self, pytester: Pytester + self, tmp_path: Path ) -> None: - pytester.makepyfile( - """ - import pytest + @pytest.mark.parametrize("x, y", [("a", "b")]) + def test_simple(x, y=1): + assert len(x) == 1 - @pytest.mark.parametrize('x, y', [('a', 'b')]) - def test_simple(x, y=1): - assert len(x) == 1 - """ - ) - result = pytester.runpytest("--collect-only") - result.stdout.fnmatch_lines( - ["*already takes an argument 'y' with a default value"] - ) + with pytest.raises( + pytest.Collector.CollectError, + match="already takes an argument 'y' with a default value", + ): + collect_tests(test_simple, rootpath=tmp_path) - def test_parametrize_functional(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def pytest_generate_tests(metafunc): - metafunc.parametrize('x', [1,2], indirect=True) - metafunc.parametrize('y', [2]) - @pytest.fixture - def x(request): - return request.param * 10 - - def test_simple(x,y): - assert x in (10,20) - assert y == 2 - """ - ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( - ["*test_simple*1-2*", "*test_simple*2-2*", "*2 passed*"] + def test_parametrize_functional(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + metafunc.parametrize("x", [1, 2], indirect=True) + metafunc.parametrize("y", [2]) + + @pytest.fixture + def x(request): + return request.param * 10 + + def test_simple(x, y): + assert x in (10, 20) + assert y == 2 + + record = run_tests( + build_module( + "test_parametrize_functional", pytest_generate_tests, x, test_simple + ), + rootpath=tmp_path, ) + assert list(record.by_test) == [ + "test_parametrize_functional.py::test_simple[1-2]", + "test_parametrize_functional.py::test_simple[2-2]", + ] + record.assert_outcomes(passed=2) def test_parametrize_onearg(self) -> None: metafunc = self.Metafunc(lambda x: None) @@ -1148,74 +1128,71 @@ def test_parametrize_twoargs(self) -> None: assert metafunc._calls[1].params == dict(x=3, y=4) assert metafunc._calls[1].id == "3-4" - def test_high_scoped_parametrize_reordering(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_high_scoped_parametrize_reordering(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("arg2", [3, 4]) + @pytest.mark.parametrize("arg1", [0, 1, 2], scope="module") + def test1(arg1, arg2): + pass - @pytest.mark.parametrize("arg2", [3, 4]) - @pytest.mark.parametrize("arg1", [0, 1, 2], scope='module') - def test1(arg1, arg2): - pass + def test2(): + pass - def test2(): - pass + @pytest.mark.parametrize("arg1", [0, 1, 2], scope="module") + def test3(arg1): + pass - @pytest.mark.parametrize("arg1", [0, 1, 2], scope='module') - def test3(arg1): - pass - """ + # ensemble: collection order is the order the members are listed in, + # so this mirrors the original file's definition order. + items = collect_tests( + build_module( + "test_high_scoped_parametrize_reordering", test1, test2, test3 + ), + rootpath=tmp_path, ) - result = pytester.runpytest("--collect-only") # Items are grouped by the *value* of the module-scoped arg1 (#8914), # so arg1 is set up only once per distinct value: 0, 1, 2. - result.stdout.re_match_lines( - [ - r" ", - r" ", - r" ", - r" ", - r" ", - r" ", - r" ", - r" ", - r" ", - r" ", - ] - ) + assert [item.name for item in items] == [ + "test1[0-3]", + "test1[0-4]", + "test3[0]", + "test1[1-3]", + "test1[1-4]", + "test3[1]", + "test1[2-3]", + "test1[2-4]", + "test3[2]", + "test2", + ] - def test_parametrize_multiple_times(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - pytestmark = pytest.mark.parametrize("x", [1,2]) - def test_func(x): + def test_parametrize_multiple_times(self, tmp_path: Path) -> None: + def test_func(x): + assert 0, x + + class TestClass: + pytestmark = pytest.mark.parametrize("y", [3, 4]) + + def test_meth(self, x, y): assert 0, x - class TestClass(object): - pytestmark = pytest.mark.parametrize("y", [3,4]) - def test_meth(self, x, y): - assert 0, x - """ - ) - result = pytester.runpytest() - assert result.ret == 1 - result.assert_outcomes(failed=6) - def test_parametrize_CSV(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize("x, y,", [(1,2), (2,3)]) - def test_func(x, y): - assert x+1 == y - """ + record = run_tests( + build_module( + "test_parametrize_multiple_times", + test_func, + TestClass, + pytestmark=pytest.mark.parametrize("x", [1, 2]), + ), + rootpath=tmp_path, ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + record.assert_outcomes(failed=6) - def test_parametrize_class_scenarios(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ + def test_parametrize_CSV(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("x, y,", [(1, 2), (2, 3)]) + def test_func(x, y): + assert x + 1 == y + + run_tests(test_func, rootpath=tmp_path).assert_outcomes(passed=2) + + def test_parametrize_class_scenarios(self, tmp_path: Path) -> None: # same as doc/en/example/parametrize scenario example def pytest_generate_tests(metafunc): idlist = [] @@ -1224,36 +1201,41 @@ def pytest_generate_tests(metafunc): idlist.append(scenario[0]) items = scenario[1].items() argnames = [x[0] for x in items] - argvalues.append(([x[1] for x in items])) + argvalues.append([x[1] for x in items]) metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class") - class Test(object): - scenarios = [['1', {'arg': {1: 2}, "arg2": "value2"}], - ['2', {'arg':'value2', "arg2": "value2"}]] + class Test: + scenarios = [ + ["1", {"arg": {1: 2}, "arg2": "value2"}], + ["2", {"arg": "value2", "arg2": "value2"}], + ] - def test_1(self, arg, arg2): - pass + def test_1(self, arg, arg2): + pass - def test_2(self, arg2, arg): - pass + def test_2(self, arg2, arg): + pass - def test_3(self, arg, arg2): - pass - """ - ) - result = pytester.runpytest("-v") - assert result.ret == 0 - result.stdout.fnmatch_lines( - """ - *test_1*1* - *test_2*1* - *test_3*1* - *test_1*2* - *test_2*2* - *test_3*2* - *6 passed* - """ - ) + def test_3(self, arg, arg2): + pass + + # ensemble: the collected order of the methods is their definition + # order in the class body, as in the original file. + record = run_tests( + build_module( + "test_parametrize_class_scenarios", pytest_generate_tests, Test + ), + rootpath=tmp_path, + ) + assert [nodeid.rpartition("::")[2] for nodeid in record.by_test] == [ + "test_1[1]", + "test_2[1]", + "test_3[1]", + "test_1[2]", + "test_2[2]", + "test_3[2]", + ] + record.assert_outcomes(passed=6) def test_parametrize_iterator_deprecation(self) -> None: """Test that using iterators for argvalues raises a deprecation warning.""" @@ -1275,35 +1257,40 @@ def data_generator() -> Iterator[int]: class TestMetafuncFunctional: - def test_attributes(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - # assumes that generate/provide runs in the same process - import sys, pytest - def pytest_generate_tests(metafunc): - metafunc.parametrize('metafunc', [metafunc]) + def test_attributes(self, tmp_path: Path) -> None: + # ensemble: the sources live in *this* file, so ``__name__`` inside + # them is this module's; the collected module's synthetic name is + # asserted against explicitly instead. + module_name = "test_attributes" - @pytest.fixture - def metafunc(request): - return request.param + def pytest_generate_tests(metafunc): + metafunc.parametrize("metafunc", [metafunc]) + + @pytest.fixture + def metafunc(request): + return request.param - def test_function(metafunc, pytestconfig): + def test_function(metafunc, pytestconfig): + assert metafunc.config == pytestconfig + assert metafunc.module.__name__ == module_name + assert metafunc.function == test_function + assert metafunc.cls is None + + class TestClass: + def test_method(self, metafunc, pytestconfig): assert metafunc.config == pytestconfig - assert metafunc.module.__name__ == __name__ - assert metafunc.function == test_function - assert metafunc.cls is None - - class TestClass(object): - def test_method(self, metafunc, pytestconfig): - assert metafunc.config == pytestconfig - assert metafunc.module.__name__ == __name__ - unbound = TestClass.test_method - assert metafunc.function == unbound - assert metafunc.cls == TestClass - """ + assert metafunc.module.__name__ == module_name + unbound = TestClass.test_method + assert metafunc.function == unbound + assert metafunc.cls == TestClass + + record = run_tests( + build_module( + module_name, pytest_generate_tests, metafunc, test_function, TestClass + ), + rootpath=tmp_path, ) - result = pytester.runpytest(p, "-v") - result.assert_outcomes(passed=2) + record.assert_outcomes(passed=2) def test_two_functions(self, tmp_path: Path) -> None: def pytest_generate_tests(metafunc): @@ -1323,274 +1310,300 @@ def test_func2(arg1): assert record["test_two_functions.py::test_func1[0]"].passed assert record["test_two_functions.py::test_func1[1]"].failed - def test_noself_in_method(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - def pytest_generate_tests(metafunc): - assert 'xyz' not in metafunc.fixturenames + def test_noself_in_method(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + assert "xyz" not in metafunc.fixturenames - class TestHello(object): - def test_hello(xyz): - pass - """ + class TestHello: + def test_hello(xyz): + pass + + record = run_tests( + build_module("test_noself_in_method", pytest_generate_tests, TestHello), + rootpath=tmp_path, ) - result = pytester.runpytest(p) - result.assert_outcomes(passed=1) + record.assert_outcomes(passed=1) - def test_generate_tests_in_class(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - class TestClass(object): - def pytest_generate_tests(self, metafunc): - metafunc.parametrize('hello', ['world'], ids=['hellow']) + def test_generate_tests_in_class(self, tmp_path: Path) -> None: + class TestClass: + def pytest_generate_tests(self, metafunc): + metafunc.parametrize("hello", ["world"], ids=["hellow"]) - def test_myfunc(self, hello): - assert hello == "world" - """ + def test_myfunc(self, hello): + assert hello == "world" + + record = run_tests( + build_module("test_generate_tests_in_class", TestClass), rootpath=tmp_path ) - result = pytester.runpytest("-v", p) - result.stdout.fnmatch_lines(["*test_myfunc*hello*PASS*", "*1 passed*"]) + assert list(record.by_test) == [ + "test_generate_tests_in_class.py::TestClass::test_myfunc[hellow]" + ] + record.assert_outcomes(passed=1) - def test_two_functions_not_same_instance(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - def pytest_generate_tests(metafunc): - metafunc.parametrize('arg1', [10, 20], ids=["0", "1"]) + def test_two_functions_not_same_instance(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + metafunc.parametrize("arg1", [10, 20], ids=["0", "1"]) - class TestClass(object): - def test_func(self, arg1): - assert not hasattr(self, 'x') - self.x = 1 - """ - ) - result = pytester.runpytest("-v", p) - result.stdout.fnmatch_lines( - ["*test_func*0*PASS*", "*test_func*1*PASS*", "*2 pass*"] + class TestClass: + def test_func(self, arg1): + assert not hasattr(self, "x") + self.x = 1 + + record = run_tests( + build_module( + "test_two_functions_not_same_instance", + pytest_generate_tests, + TestClass, + ), + rootpath=tmp_path, ) + assert list(record.by_test) == [ + "test_two_functions_not_same_instance.py::TestClass::test_func[0]", + "test_two_functions_not_same_instance.py::TestClass::test_func[1]", + ] + record.assert_outcomes(passed=2) - def test_issue28_setup_method_in_generate_tests(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - def pytest_generate_tests(metafunc): - metafunc.parametrize('arg1', [1]) - - class TestClass(object): - def test_method(self, arg1): - assert arg1 == self.val - def setup_method(self, func): - self.val = 1 - """ - ) - result = pytester.runpytest(p) - result.assert_outcomes(passed=1) + def test_issue28_setup_method_in_generate_tests(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + metafunc.parametrize("arg1", [1]) - def test_parametrize_functional2(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - def pytest_generate_tests(metafunc): - metafunc.parametrize("arg1", [1,2]) - metafunc.parametrize("arg2", [4,5]) - def test_hello(arg1, arg2): - assert 0, (arg1, arg2) - """ + class TestClass: + def test_method(self, arg1): + assert arg1 == self.val + + def setup_method(self, func): + self.val = 1 + + record = run_tests( + build_module( + "test_issue28_setup_method_in_generate_tests", + pytest_generate_tests, + TestClass, + ), + rootpath=tmp_path, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - ["*(1, 4)*", "*(1, 5)*", "*(2, 4)*", "*(2, 5)*", "*4 failed*"] + record.assert_outcomes(passed=1) + + def test_parametrize_functional2(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + metafunc.parametrize("arg1", [1, 2]) + metafunc.parametrize("arg2", [4, 5]) + + def test_hello(arg1, arg2): + assert 0, (arg1, arg2) + + record = run_tests( + build_module( + "test_parametrize_functional2", pytest_generate_tests, test_hello + ), + rootpath=tmp_path, ) + record.assert_outcomes(failed=4) + for args in [(1, 4), (1, 5), (2, 4), (2, 5)]: + item = record[f"test_hello[{args[0]}-{args[1]}]"] + assert item.call is not None + assert str(args) in str(item.call.longrepr) def test_parametrize_single_arg_trailing_comma_functional( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test that trailing comma in string argnames behaves like tuple argnames. Regression test for https://github.com/pytest-dev/pytest/issues/719 """ - pytester.makepyfile( - """ - import pytest + scenarios = [("a",), ("b",)] - scenarios = [('a',), ('b',)] - - @pytest.mark.parametrize(("arg",), scenarios) - def test_tuple_form(arg): - # Tuple argnames: values are unpacked from tuples - assert arg in ('a', 'b') - assert isinstance(arg, str) - - @pytest.mark.parametrize("arg,", scenarios) - def test_string_trailing_comma(arg): - # String with trailing comma: should behave like tuple form - assert arg in ('a', 'b') - assert isinstance(arg, str) - - @pytest.mark.parametrize("arg", scenarios) - def test_string_no_comma(arg): - # String without comma: tuples are passed as-is - assert arg in (('a',), ('b',)) - assert isinstance(arg, tuple) - """ + @pytest.mark.parametrize(("arg",), scenarios) + def test_tuple_form(arg): + # Tuple argnames: values are unpacked from tuples + assert arg in ("a", "b") + assert isinstance(arg, str) + + @pytest.mark.parametrize("arg,", scenarios) + def test_string_trailing_comma(arg): + # String with trailing comma: should behave like tuple form + assert arg in ("a", "b") + assert isinstance(arg, str) + + @pytest.mark.parametrize("arg", scenarios) + def test_string_no_comma(arg): + # String without comma: tuples are passed as-is + assert arg in (("a",), ("b",)) + assert isinstance(arg, tuple) + + record = run_tests( + test_tuple_form, + test_string_trailing_comma, + test_string_no_comma, + rootpath=tmp_path, + ) + record.assert_outcomes(passed=6) + + def test_parametrize_and_inner_getfixturevalue(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + metafunc.parametrize("arg1", [1], indirect=True) + metafunc.parametrize("arg2", [10], indirect=True) + + @pytest.fixture + def arg1(request): + x = request.getfixturevalue("arg2") + return x + request.param + + @pytest.fixture + def arg2(request): + return request.param + + def test_func1(arg1, arg2): + assert arg1 == 11 + + record = run_tests( + build_module( + "test_parametrize_and_inner_getfixturevalue", + pytest_generate_tests, + arg1, + arg2, + test_func1, + ), + rootpath=tmp_path, ) - result = pytester.runpytest("-v") - result.assert_outcomes(passed=6) + assert list(record.by_test) == [ + "test_parametrize_and_inner_getfixturevalue.py::test_func1[1-10]" + ] + record.assert_outcomes(passed=1) - def test_parametrize_and_inner_getfixturevalue(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - def pytest_generate_tests(metafunc): - metafunc.parametrize("arg1", [1], indirect=True) - metafunc.parametrize("arg2", [10], indirect=True) + def test_parametrize_on_setup_arg(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + assert "arg1" in metafunc.fixturenames + metafunc.parametrize("arg1", [1], indirect=True) + + @pytest.fixture + def arg1(request): + return request.param + + @pytest.fixture + def arg2(request, arg1): + return 10 * arg1 + + def test_func(arg2): + assert arg2 == 10 + + record = run_tests( + build_module( + "test_parametrize_on_setup_arg", + pytest_generate_tests, + arg1, + arg2, + test_func, + ), + rootpath=tmp_path, + ) + assert list(record.by_test) == [ + "test_parametrize_on_setup_arg.py::test_func[1]" + ] + record.assert_outcomes(passed=1) - import pytest - @pytest.fixture - def arg1(request): - x = request.getfixturevalue("arg2") - return x + request.param + def test_parametrize_with_ids(self, tmp_path: Path) -> None: + # ensemble: the original set console_output_style=classic purely to + # make the -v output greppable; nothing is read off the output now. + def pytest_generate_tests(metafunc): + metafunc.parametrize( + ("a", "b"), [(1, 1), (1, 2)], ids=["basic", "advanced"] + ) - @pytest.fixture - def arg2(request): - return request.param + def test_function(a, b): + assert a == b - def test_func1(arg1, arg2): - assert arg1 == 11 - """ + record = run_tests( + build_module( + "test_parametrize_with_ids", pytest_generate_tests, test_function + ), + rootpath=tmp_path, ) - result = pytester.runpytest("-v", p) - result.stdout.fnmatch_lines(["*test_func1*1*PASS*", "*1 passed*"]) + assert record["test_function[basic]"].passed + assert record["test_function[advanced]"].failed + record.assert_outcomes(passed=1, failed=1) - def test_parametrize_on_setup_arg(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - def pytest_generate_tests(metafunc): - assert "arg1" in metafunc.fixturenames - metafunc.parametrize("arg1", [1], indirect=True) - - import pytest - @pytest.fixture - def arg1(request): - return request.param + def test_parametrize_without_ids(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + metafunc.parametrize(("a", "b"), [(1, object()), (1.3, object())]) - @pytest.fixture - def arg2(request, arg1): - return 10 * arg1 + def test_function(a, b): + assert 1 - def test_func(arg2): - assert arg2 == 10 - """ + items = collect_tests( + build_module( + "test_parametrize_without_ids", pytest_generate_tests, test_function + ), + rootpath=tmp_path, ) - result = pytester.runpytest("-v", p) - result.stdout.fnmatch_lines(["*test_func*1*PASS*", "*1 passed*"]) + assert [item.name for item in items] == [ + "test_function[1-b0]", + "test_function[1.3-b1]", + ] - def test_parametrize_with_ids(self, pytester: Pytester) -> None: - pytester.makeini( - """ - [pytest] - console_output_style=classic - """ - ) - pytester.makepyfile( - """ - import pytest - def pytest_generate_tests(metafunc): - metafunc.parametrize(("a", "b"), [(1,1), (1,2)], - ids=["basic", "advanced"]) + def test_parametrize_with_None_in_ids(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + metafunc.parametrize( + ("a", "b"), [(1, 1), (1, 1), (1, 2)], ids=["basic", None, "advanced"] + ) - def test_function(a, b): - assert a == b - """ - ) - result = pytester.runpytest("-v") - assert result.ret == 1 - result.stdout.fnmatch_lines_random( - ["*test_function*basic*PASSED", "*test_function*advanced*FAILED"] + def test_function(a, b): + assert a == b + + record = run_tests( + build_module( + "test_parametrize_with_None_in_ids", + pytest_generate_tests, + test_function, + ), + rootpath=tmp_path, ) + assert record["test_function[basic]"].passed + assert record["test_function[1-1]"].passed + assert record["test_function[advanced]"].failed + record.assert_outcomes(passed=2, failed=1) - def test_parametrize_without_ids(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def pytest_generate_tests(metafunc): - metafunc.parametrize(("a", "b"), - [(1,object()), (1.3,object())]) + def test_fixture_parametrized_empty_ids(self, tmp_path: Path) -> None: + """Fixtures parametrized with empty ids cause an internal error (#1849).""" - def test_function(a, b): - assert 1 - """ - ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( - """ - *test_function*1-b0* - *test_function*1.3-b1* - """ - ) + @pytest.fixture(scope="module", ids=[], params=[]) + def temp(request): + return request.param - def test_parametrize_with_None_in_ids(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def pytest_generate_tests(metafunc): - metafunc.parametrize(("a", "b"), [(1,1), (1,1), (1,2)], - ids=["basic", None, "advanced"]) + def test_temp(temp): + pass - def test_function(a, b): - assert a == b - """ + record = run_tests( + build_module("test_fixture_parametrized_empty_ids", temp, test_temp), + rootpath=tmp_path, ) - result = pytester.runpytest("-v") - assert result.ret == 1 - result.stdout.fnmatch_lines_random( - [ - "*test_function*basic*PASSED*", - "*test_function*1-1*PASSED*", - "*test_function*advanced*FAILED*", - ] - ) - - def test_fixture_parametrized_empty_ids(self, pytester: Pytester) -> None: - """Fixtures parametrized with empty ids cause an internal error (#1849).""" - pytester.makepyfile( - """ - import pytest + record.assert_outcomes(skipped=1) - @pytest.fixture(scope="module", ids=[], params=[]) - def temp(request): - return request.param - - def test_temp(temp): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 1 skipped *"]) - - def test_parametrized_empty_ids(self, pytester: Pytester) -> None: + def test_parametrized_empty_ids(self, tmp_path: Path) -> None: """Tests parametrized with empty ids cause an internal error (#1849).""" - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize('temp', [], ids=list()) - def test_temp(temp): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 1 skipped *"]) + @pytest.mark.parametrize("temp", [], ids=list()) + def test_temp(temp): + pass - def test_parametrized_ids_invalid_type(self, pytester: Pytester) -> None: + run_tests(test_temp, rootpath=tmp_path).assert_outcomes(skipped=1) + + def test_parametrized_ids_invalid_type(self, tmp_path: Path) -> None: """Test error with non-strings/non-ints, without generator (#1857).""" - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize("x, expected", [(1, 2), (3, 4), (5, 6)], ids=(None, 2, OSError())) - def test_ids_numbers(x,expected): - assert x * 2 == expected - """ + @pytest.mark.parametrize( + "x, expected", + [(1, 2), (3, 4), (5, 6)], + ids=(None, 2, OSError()), # type: ignore[arg-type] ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + def test_ids_numbers(x, expected): + assert x * 2 == expected + + # ensemble: the module name is part of the reported nodeid. + module = build_module("test_parametrized_ids_invalid_type", test_ids_numbers) + record = run_tests(module, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( [ "In test_parametrized_ids_invalid_type.py::test_ids_numbers: ids contains unsupported value " "OSError() (type: ) at index 2. " @@ -1599,86 +1612,104 @@ def test_ids_numbers(x,expected): ) def test_parametrize_with_identical_ids_get_unique_names( - self, pytester: Pytester + self, tmp_path: Path ) -> None: - pytester.makepyfile( - """ - import pytest - def pytest_generate_tests(metafunc): - metafunc.parametrize(("a", "b"), [(1,1), (1,2)], - ids=["a", "a"]) + def pytest_generate_tests(metafunc): + metafunc.parametrize(("a", "b"), [(1, 1), (1, 2)], ids=["a", "a"]) - def test_function(a, b): - assert a == b - """ - ) - result = pytester.runpytest("-v") - assert result.ret == 1 - result.stdout.fnmatch_lines_random( - ["*test_function*a0*PASSED*", "*test_function*a1*FAILED*"] + def test_function(a, b): + assert a == b + + record = run_tests( + build_module( + "test_parametrize_with_identical_ids_get_unique_names", + pytest_generate_tests, + test_function, + ), + rootpath=tmp_path, ) + assert record["test_function[a0]"].passed + assert record["test_function[a1]"].failed + record.assert_outcomes(passed=1, failed=1) @pytest.mark.parametrize(("scope", "length"), [("module", 2), ("function", 4)]) def test_parametrize_scope_overrides( - self, pytester: Pytester, scope: str, length: int + self, tmp_path: Path, scope: str, length: int ) -> None: - pytester.makepyfile( - f""" - import pytest - values = [] - def pytest_generate_tests(metafunc): - if "arg" in metafunc.fixturenames: - metafunc.parametrize("arg", [1,2], indirect=True, - scope={scope!r}) - @pytest.fixture - def arg(request): - values.append(request.param) - return request.param - def test_hello(arg): - assert arg in (1,2) - def test_world(arg): - assert arg in (1,2) - def test_checklength(): - assert len(values) == {length} - """ + values: list[object] = [] + + def pytest_generate_tests(metafunc): + if "arg" in metafunc.fixturenames: + metafunc.parametrize("arg", [1, 2], indirect=True, scope=scope) + + @pytest.fixture + def arg(request): + values.append(request.param) + return request.param + + def test_hello(arg): + assert arg in (1, 2) + + def test_world(arg): + assert arg in (1, 2) + + def test_checklength(): + assert len(values) == length + + record = run_tests( + build_module( + "test_parametrize_scope_overrides", + pytest_generate_tests, + arg, + test_hello, + test_world, + test_checklength, + ), + rootpath=tmp_path, ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=5) + record.assert_outcomes(passed=5) - def test_parametrize_issue323(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_issue323(self, tmp_path: Path) -> None: + @pytest.fixture(scope="module", params=range(966)) + def foo(request): + return request.param - @pytest.fixture(scope='module', params=range(966)) - def foo(request): - return request.param + def test_it(foo): + pass - def test_it(foo): - pass - def test_it2(foo): - pass - """ + def test_it2(foo): + pass + + # ensemble: collect_tests raises on a failed collection, so this can + # no longer collect nothing and pass by accident. + items = collect_tests( + build_module("test_parametrize_issue323", foo, test_it, test_it2), + rootpath=tmp_path, ) - reprec = pytester.inline_run("--collect-only") - assert not reprec.getcalls("pytest_internalerror") + assert len(items) == 2 * 966 - def test_usefixtures_seen_in_generate_tests(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def pytest_generate_tests(metafunc): - assert "abc" in metafunc.fixturenames - metafunc.parametrize("abc", [1]) + def test_usefixtures_seen_in_generate_tests(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + assert "abc" in metafunc.fixturenames + metafunc.parametrize("abc", [1]) - @pytest.mark.usefixtures("abc") - def test_function(): - pass - """ + @pytest.mark.usefixtures("abc") + def test_function(): + pass + + record = run_tests( + build_module( + "test_usefixtures_seen_in_generate_tests", + pytest_generate_tests, + test_function, + ), + rootpath=tmp_path, ) - reprec = pytester.runpytest() - reprec.assert_outcomes(passed=1) + record.assert_outcomes(passed=1) + # ensemble: conftest hooks become globally registered plugins, so there is + # no way to express a hook that only applies to one directory - which is + # exactly what this test is about. def test_generate_tests_only_done_in_subdir(self, pytester: Pytester) -> None: sub1 = pytester.mkpydir("sub1") sub2 = pytester.mkpydir("sub2") @@ -1709,24 +1740,28 @@ def pytest_generate_tests(metafunc): result = pytester.runpytest("--keep-duplicates", "-v", "-s", sub1, sub2, sub1) result.assert_outcomes(passed=3) - def test_generate_same_function_names_issue403(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_generate_same_function_names_issue403(self, tmp_path: Path) -> None: + def make_tests(): + @pytest.mark.parametrize("x", range(2)) + def test_foo(x): + pass - def make_tests(): - @pytest.mark.parametrize("x", range(2)) - def test_foo(x): - pass - return test_foo + return test_foo - test_x = make_tests() - test_y = make_tests() - """ + # ensemble: both functions are named ``test_foo``, so they have to be + # placed under explicit module attribute names. + record = run_tests( + build_module( + "test_generate_same_function_names_issue403", + test_x=make_tests(), + test_y=make_tests(), + ), + rootpath=tmp_path, ) - reprec = pytester.runpytest() - reprec.assert_outcomes(passed=4) + record.assert_outcomes(passed=4) + # ensemble: ``@pytest.mark.parametrise`` raises Failed against the *host* + # config while the decorator is applied, so the source cannot be built. def test_parametrize_misspelling(self, pytester: Pytester) -> None: """#463""" pytester.makepyfile( @@ -1755,53 +1790,58 @@ def test_foo(x): @pytest.mark.parametrize("scope", ["class", "package"]) def test_parametrize_missing_scope_doesnt_crash( - self, pytester: Pytester, scope: str + self, tmp_path: Path, scope: str ) -> None: """Doesn't crash when parametrize(scope=) is used without a corresponding node.""" - pytester.makepyfile( - f""" - import pytest - @pytest.mark.parametrize("x", [0], scope="{scope}") - def test_it(x): pass - """ - ) - result = pytester.runpytest() - assert result.ret == 0 + @pytest.mark.parametrize("x", [0], scope=scope) # type: ignore[arg-type] + def test_it(x): + pass + + run_tests(test_it, rootpath=tmp_path).assert_outcomes(passed=1) def test_parametrize_module_level_test_with_class_scope( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """ Test that a class-scoped parametrization without a corresponding `Class` gets module scope, i.e. we only create a single FixtureDef for it per module. """ - module = pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize("x", [0, 1], scope="class") - def test_1(x): - pass + @pytest.mark.parametrize("x", [0, 1], scope="class") + def test_1(x): + pass - @pytest.mark.parametrize("x", [1, 2], scope="module") - def test_2(x): - pass - """ + @pytest.mark.parametrize("x", [1, 2], scope="module") + def test_2(x): + pass + + items = collect_tests( + build_module( + "test_parametrize_module_level_test_with_class_scope", test_1, test_2 + ), + rootpath=tmp_path, ) - test_1_0, _, test_2_0, _ = pytester.genitems((pytester.getmodulecol(module),)) + # ensemble: unlike Pytester.genitems() this goes through the full + # collection protocol, which reorders high-scoped parametrizations, so + # the items are looked up by name rather than by position. + by_name = {item.name: item for item in items} + assert sorted(by_name) == ["test_1[0]", "test_1[1]", "test_2[1]", "test_2[2]"] + test_1_0 = by_name["test_1[0]"] assert isinstance(test_1_0, Function) - assert test_1_0.name == "test_1[0]" test_1_fixture_x = test_1_0._fixtureinfo.name2fixturedefs["x"][-1] + test_2_0 = by_name["test_2[1]"] assert isinstance(test_2_0, Function) - assert test_2_0.name == "test_2[1]" test_2_fixture_x = test_2_0._fixtureinfo.name2fixturedefs["x"][-1] assert test_1_fixture_x is test_2_fixture_x + # ensemble: this goes green under an ensemble, but for the wrong reason - + # the package-scoped conftest fixture degrades to a plugin fixture with no + # ``Package`` node, so a different reorder path is exercised. def test_reordering_with_scopeless_and_just_indirect_parametrization( self, pytester: Pytester ) -> None: @@ -1863,6 +1903,7 @@ def test_3(self, fixture): ] ) + # ensemble: needs a real subprocess running pytest.main() twice. def test_parametrize_generator_multiple_runs(self, pytester: Pytester) -> None: """Test that generators in parametrize work with multiple pytest.main() (deprecated).""" testfile = pytester.makepyfile( @@ -1892,34 +1933,35 @@ def test_foo(bar): ] ) - def test_parametrize_iterator_class_multiple_tests( - self, pytester: Pytester - ) -> None: + def test_parametrize_iterator_class_multiple_tests(self, tmp_path: Path) -> None: """Test that iterators in parametrize on a class get exhausted (deprecated).""" - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize("n", iter(range(2))) - class Test: - def test_1(self, n): - pass + @pytest.mark.parametrize("n", iter(range(2))) + class Test: + def test_1(self, n): + pass - def test_2(self, n): - pass - """ + def test_2(self, n): + pass + + # ensemble: the host suite's ``filterwarnings = error`` is inherited, + # which would turn the deprecation into a collection error. + spec = ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["always"]}) + record = run_tests( + build_module("test_parametrize_iterator_class_multiple_tests", Test), + spec=spec, ) - result = pytester.runpytest("-v", "-Wdefault") # Iterator gets exhausted after first test, second test gets no parameters. # This is deprecated. - result.assert_outcomes(passed=2, skipped=1) - result.stdout.fnmatch_lines( - [ - "*test_parametrize_iterator_class_multiple_tests.py::Test::test_1[[]0] PASSED*", - "*test_parametrize_iterator_class_multiple_tests.py::Test::test_1[[]1] PASSED*", - "*test_parametrize_iterator_class_multiple_tests.py::Test::test_2[[]NOTSET] SKIPPED*", - "*PytestRemovedIn10Warning: Passing a non-Collection iterable*", - ] + assert list(record.by_test) == [ + "test_parametrize_iterator_class_multiple_tests.py::Test::test_1[0]", + "test_parametrize_iterator_class_multiple_tests.py::Test::test_1[1]", + "test_parametrize_iterator_class_multiple_tests.py::Test::test_2[NOTSET]", + ] + record.assert_outcomes(passed=2, skipped=1) + assert any( + "Passing a non-Collection iterable" in str(warning.message) + for warning in record.warnings ) @@ -1927,182 +1969,190 @@ class TestMetafuncFunctionalAuto: """Tests related to automatically find out the correct scope for parametrized tests (#1832).""" - def test_parametrize_auto_scope(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_auto_scope(self, tmp_path: Path) -> None: + @pytest.fixture(scope="session", autouse=True) + def fixture(): + return 1 - @pytest.fixture(scope='session', autouse=True) - def fixture(): - return 1 - - @pytest.mark.parametrize('animal', ["dog", "cat"]) - def test_1(animal): - assert animal in ('dog', 'cat') + @pytest.mark.parametrize("animal", ["dog", "cat"]) + def test_1(animal): + assert animal in ("dog", "cat") - @pytest.mark.parametrize('animal', ['fish']) - def test_2(animal): - assert animal == 'fish' + @pytest.mark.parametrize("animal", ["fish"]) + def test_2(animal): + assert animal == "fish" - """ + record = run_tests( + build_module("test_parametrize_auto_scope", fixture, test_1, test_2), + rootpath=tmp_path, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 3 passed *"]) + record.assert_outcomes(passed=3) - def test_parametrize_auto_scope_indirect(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_auto_scope_indirect(self, tmp_path: Path) -> None: + @pytest.fixture(scope="session") + def echo(request): + return request.param - @pytest.fixture(scope='session') - def echo(request): - return request.param + @pytest.mark.parametrize( + "animal, echo", [("dog", 1), ("cat", 2)], indirect=["echo"] + ) + def test_1(animal, echo): + assert animal in ("dog", "cat") + assert echo in (1, 2, 3) - @pytest.mark.parametrize('animal, echo', [("dog", 1), ("cat", 2)], indirect=['echo']) - def test_1(animal, echo): - assert animal in ('dog', 'cat') - assert echo in (1, 2, 3) + @pytest.mark.parametrize("animal, echo", [("fish", 3)], indirect=["echo"]) + def test_2(animal, echo): + assert animal == "fish" + assert echo in (1, 2, 3) - @pytest.mark.parametrize('animal, echo', [('fish', 3)], indirect=['echo']) - def test_2(animal, echo): - assert animal == 'fish' - assert echo in (1, 2, 3) - """ + record = run_tests( + build_module("test_parametrize_auto_scope_indirect", echo, test_1, test_2), + rootpath=tmp_path, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 3 passed *"]) + record.assert_outcomes(passed=3) - def test_parametrize_auto_scope_override_fixture(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_auto_scope_override_fixture(self, tmp_path: Path) -> None: + @pytest.fixture(scope="session", autouse=True) + def animal(): + return "fox" - @pytest.fixture(scope='session', autouse=True) - def animal(): - return 'fox' + @pytest.mark.parametrize("animal", ["dog", "cat"]) + def test_1(animal): + assert animal in ("dog", "cat") - @pytest.mark.parametrize('animal', ["dog", "cat"]) - def test_1(animal): - assert animal in ('dog', 'cat') - """ + record = run_tests( + build_module( + "test_parametrize_auto_scope_override_fixture", animal, test_1 + ), + rootpath=tmp_path, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 2 passed *"]) + record.assert_outcomes(passed=2) - def test_parametrize_all_indirects(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_all_indirects(self, tmp_path: Path) -> None: + @pytest.fixture + def animal(request): + return request.param - @pytest.fixture() - def animal(request): - return request.param + @pytest.fixture(scope="session") + def echo(request): + return request.param - @pytest.fixture(scope='session') - def echo(request): - return request.param + @pytest.mark.parametrize( + "animal, echo", [("dog", 1), ("cat", 2)], indirect=True + ) + def test_1(animal, echo): + assert animal in ("dog", "cat") + assert echo in (1, 2, 3) - @pytest.mark.parametrize('animal, echo', [("dog", 1), ("cat", 2)], indirect=True) - def test_1(animal, echo): - assert animal in ('dog', 'cat') - assert echo in (1, 2, 3) + @pytest.mark.parametrize("animal, echo", [("fish", 3)], indirect=True) + def test_2(animal, echo): + assert animal == "fish" + assert echo in (1, 2, 3) - @pytest.mark.parametrize('animal, echo', [("fish", 3)], indirect=True) - def test_2(animal, echo): - assert animal == 'fish' - assert echo in (1, 2, 3) - """ + record = run_tests( + build_module( + "test_parametrize_all_indirects", animal, echo, test_1, test_2 + ), + rootpath=tmp_path, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 3 passed *"]) + record.assert_outcomes(passed=3) - def test_parametrize_some_arguments_auto_scope( - self, pytester: Pytester, monkeypatch - ) -> None: + def test_parametrize_some_arguments_auto_scope(self, tmp_path: Path) -> None: """Integration test for (#3941)""" + # ensemble: the sources are real objects, so the setup log is a plain + # closed-over list instead of an attribute smuggled onto ``sys``. class_fix_setup: list[object] = [] - monkeypatch.setattr(sys, "class_fix_setup", class_fix_setup, raising=False) func_fix_setup: list[object] = [] - monkeypatch.setattr(sys, "func_fix_setup", func_fix_setup, raising=False) - pytester.makepyfile( - """ - import pytest - import sys + @pytest.fixture(scope="class", autouse=True) + def class_fix(request): + class_fix_setup.append(request.param) - @pytest.fixture(scope='class', autouse=True) - def class_fix(request): - sys.class_fix_setup.append(request.param) + @pytest.fixture(autouse=True) + def func_fix(): + func_fix_setup.append(True) - @pytest.fixture(autouse=True) - def func_fix(): - sys.func_fix_setup.append(True) + @pytest.mark.parametrize("class_fix", [10, 20], indirect=True) + class Test: + def test_foo(self): + pass - @pytest.mark.parametrize('class_fix', [10, 20], indirect=True) - class Test: - def test_foo(self): - pass - def test_bar(self): - pass - """ + def test_bar(self): + pass + + record = run_tests( + build_module( + "test_parametrize_some_arguments_auto_scope", + class_fix, + func_fix, + Test, + ), + rootpath=tmp_path, ) - result = pytester.runpytest_inprocess() - result.stdout.fnmatch_lines(["* 4 passed in *"]) + record.assert_outcomes(passed=4) assert func_fix_setup == [True] * 4 assert class_fix_setup == [10, 20] - def test_parametrize_issue634(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_issue634(self, tmp_path: Path) -> None: + # ensemble: what the original grepped out of the captured stdout is + # recorded directly instead. + prepared: list[int] = [] - @pytest.fixture(scope='module') - def foo(request): - print('preparing foo-%d' % request.param) - return 'foo-%d' % request.param + @pytest.fixture(scope="module") + def foo(request): + prepared.append(request.param) + return f"foo-{request.param}" - def test_one(foo): - pass + def test_one(foo): + pass - def test_two(foo): - pass + def test_two(foo): + pass - test_two.test_with = (2, 3) + test_two.test_with = (2, 3) # type: ignore[attr-defined] - def pytest_generate_tests(metafunc): - params = (1, 2, 3, 4) - if not 'foo' in metafunc.fixturenames: - return + def pytest_generate_tests(metafunc): + params = (1, 2, 3, 4) + if "foo" not in metafunc.fixturenames: + return - test_with = getattr(metafunc.function, 'test_with', None) - if test_with: - params = test_with - metafunc.parametrize('foo', params, indirect=True) - """ + test_with = getattr(metafunc.function, "test_with", None) + if test_with: + params = test_with + metafunc.parametrize("foo", params, indirect=True) + + record = run_tests( + build_module( + "test_parametrize_issue634", + foo, + test_one, + test_two, + pytest_generate_tests, + ), + rootpath=tmp_path, ) - result = pytester.runpytest("-s") - output = result.stdout.str() - assert output.count("preparing foo-2") == 1 - assert output.count("preparing foo-3") == 1 + record.assert_outcomes(passed=6) + assert prepared.count(2) == 1 + assert prepared.count(3) == 1 class TestMarkersWithParametrization: """#308""" - def test_simple_mark(self, pytester: Pytester) -> None: - s = """ - import pytest - - @pytest.mark.foo - @pytest.mark.parametrize(("n", "expected"), [ + def test_simple_mark(self, tmp_path: Path) -> None: + @pytest.mark.foo + @pytest.mark.parametrize( + ("n", "expected"), + [ (1, 2), pytest.param(1, 3, marks=pytest.mark.bar), (2, 3), - ]) - def test_increment(n, expected): - assert n + 1 == expected - """ - items = pytester.getitems(s) + ], + ) + def test_increment(n, expected): + assert n + 1 == expected + + items = collect_tests(test_increment, rootpath=tmp_path) assert len(items) == 3 for item in items: assert "foo" in item.keywords @@ -2110,309 +2160,320 @@ def test_increment(n, expected): assert "bar" in items[1].keywords assert "bar" not in items[2].keywords - def test_select_based_on_mark(self, pytester: Pytester) -> None: - s = """ - import pytest - - @pytest.mark.parametrize(("n", "expected"), [ + def test_select_based_on_mark(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + ("n", "expected"), + [ (1, 2), pytest.param(2, 3, marks=pytest.mark.foo), (3, 4), - ]) - def test_increment(n, expected): - assert n + 1 == expected - """ - pytester.makepyfile(s) - rec = pytester.inline_run("-m", "foo") - passed, skipped, fail = rec.listoutcomes() - assert len(passed) == 1 - assert len(skipped) == 0 - assert len(fail) == 0 - - def test_simple_xfail(self, pytester: Pytester) -> None: - s = """ - import pytest + ], + ) + def test_increment(n, expected): + assert n + 1 == expected + + spec = ConfigSpec(rootpath=tmp_path, args=("-m", "foo")) + record = run_tests(test_increment, spec=spec) + record.assert_outcomes(passed=1, deselected=2) - @pytest.mark.parametrize(("n", "expected"), [ + def test_simple_xfail(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + ("n", "expected"), + [ (1, 2), pytest.param(1, 3, marks=pytest.mark.xfail), (2, 3), - ]) - def test_increment(n, expected): - assert n + 1 == expected - """ - pytester.makepyfile(s) - reprec = pytester.inline_run() - # xfail is skip?? - reprec.assertoutcome(passed=2, skipped=1) + ], + ) + def test_increment(n, expected): + assert n + 1 == expected - def test_simple_xfail_single_argname(self, pytester: Pytester) -> None: - s = """ - import pytest + # ensemble: HookRecorder.assertoutcome() lumped xfails in with the + # skips (hence the old "xfail is skip??"); RunRecord reports the real + # category, so this now asserts xfailed=1. + run_tests(test_increment, rootpath=tmp_path).assert_outcomes( + passed=2, xfailed=1 + ) - @pytest.mark.parametrize("n", [ + def test_simple_xfail_single_argname(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + "n", + [ 2, pytest.param(3, marks=pytest.mark.xfail), 4, - ]) - def test_isEven(n): - assert n % 2 == 0 - """ - pytester.makepyfile(s) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2, skipped=1) + ], + ) + def test_isEven(n): + assert n % 2 == 0 - def test_xfail_with_arg(self, pytester: Pytester) -> None: - s = """ - import pytest + # ensemble: xfailed, not skipped - see test_simple_xfail. + run_tests(test_isEven, rootpath=tmp_path).assert_outcomes(passed=2, xfailed=1) - @pytest.mark.parametrize(("n", "expected"), [ + def test_xfail_with_arg(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + ("n", "expected"), + [ (1, 2), pytest.param(1, 3, marks=pytest.mark.xfail("True")), (2, 3), - ]) - def test_increment(n, expected): - assert n + 1 == expected - """ - pytester.makepyfile(s) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2, skipped=1) + ], + ) + def test_increment(n, expected): + assert n + 1 == expected - def test_xfail_with_kwarg(self, pytester: Pytester) -> None: - s = """ - import pytest + # ensemble: xfailed, not skipped - see test_simple_xfail. + run_tests(test_increment, rootpath=tmp_path).assert_outcomes( + passed=2, xfailed=1 + ) - @pytest.mark.parametrize(("n", "expected"), [ + def test_xfail_with_kwarg(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + ("n", "expected"), + [ (1, 2), pytest.param(1, 3, marks=pytest.mark.xfail(reason="some bug")), (2, 3), - ]) - def test_increment(n, expected): - assert n + 1 == expected - """ - pytester.makepyfile(s) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2, skipped=1) + ], + ) + def test_increment(n, expected): + assert n + 1 == expected - def test_xfail_with_arg_and_kwarg(self, pytester: Pytester) -> None: - s = """ - import pytest + # ensemble: xfailed, not skipped - see test_simple_xfail. + run_tests(test_increment, rootpath=tmp_path).assert_outcomes( + passed=2, xfailed=1 + ) - @pytest.mark.parametrize(("n", "expected"), [ + def test_xfail_with_arg_and_kwarg(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + ("n", "expected"), + [ (1, 2), pytest.param(1, 3, marks=pytest.mark.xfail("True", reason="some bug")), (2, 3), - ]) - def test_increment(n, expected): - assert n + 1 == expected - """ - pytester.makepyfile(s) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2, skipped=1) + ], + ) + def test_increment(n, expected): + assert n + 1 == expected - @pytest.mark.parametrize("strict", [True, False]) - def test_xfail_passing_is_xpass(self, pytester: Pytester, strict: bool) -> None: - s = f""" - import pytest + # ensemble: xfailed, not skipped - see test_simple_xfail. + run_tests(test_increment, rootpath=tmp_path).assert_outcomes( + passed=2, xfailed=1 + ) - m = pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict}) + @pytest.mark.parametrize("strict", [True, False]) + def test_xfail_passing_is_xpass(self, tmp_path: Path, strict: bool) -> None: + m = pytest.mark.xfail( + "sys.version_info > (0, 0, 0)", reason="some bug", strict=strict + ) - @pytest.mark.parametrize(("n", "expected"), [ + @pytest.mark.parametrize( + ("n", "expected"), + [ (1, 2), pytest.param(2, 3, marks=m), (3, 4), - ]) - def test_increment(n, expected): - assert n + 1 == expected - """ - pytester.makepyfile(s) - reprec = pytester.inline_run() - passed, failed = (2, 1) if strict else (3, 0) - reprec.assertoutcome(passed=passed, failed=failed) - - def test_parametrize_called_in_generate_tests(self, pytester: Pytester) -> None: - s = """ - import pytest + ], + ) + def test_increment(n, expected): + assert n + 1 == expected + record = run_tests(test_increment, rootpath=tmp_path) + # ensemble: HookRecorder.assertoutcome() counted the non-strict xpass + # as a plain pass; RunRecord reports it as xpassed. + if strict: + record.assert_outcomes(passed=2, failed=1) + else: + record.assert_outcomes(passed=2, xpassed=1) - def pytest_generate_tests(metafunc): - passingTestData = [(1, 2), - (2, 3)] - failingTestData = [(1, 3), - (2, 2)] + def test_parametrize_called_in_generate_tests(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + passingTestData = [(1, 2), (2, 3)] + failingTestData = [(1, 3), (2, 2)] - testData = passingTestData + [pytest.param(*d, marks=pytest.mark.xfail) - for d in failingTestData] - metafunc.parametrize(("n", "expected"), testData) + testData = passingTestData + [ + pytest.param(*d, marks=pytest.mark.xfail) for d in failingTestData + ] + metafunc.parametrize(("n", "expected"), testData) + def test_increment(n, expected): + assert n + 1 == expected - def test_increment(n, expected): - assert n + 1 == expected - """ - pytester.makepyfile(s) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2, skipped=2) + record = run_tests( + build_module( + "test_parametrize_called_in_generate_tests", + pytest_generate_tests, + test_increment, + ), + rootpath=tmp_path, + ) + # ensemble: xfailed, not skipped - see test_simple_xfail. + record.assert_outcomes(passed=2, xfailed=2) - def test_parametrize_ID_generation_string_int_works( - self, pytester: Pytester - ) -> None: + def test_parametrize_ID_generation_string_int_works(self, tmp_path: Path) -> None: """#290""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def myfixture(): - return 'example' - @pytest.mark.parametrize( - 'limit', (0, '0')) - def test_limit(limit, myfixture): - return - """ + @pytest.fixture + def myfixture(): + return "example" + + @pytest.mark.parametrize("limit", (0, "0")) + def test_limit(limit, myfixture): + return + + record = run_tests( + build_module( + "test_parametrize_ID_generation_string_int_works", + myfixture, + test_limit, + ), + rootpath=tmp_path, ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=2) + record.assert_outcomes(passed=2) @pytest.mark.parametrize("strict", [True, False]) - def test_parametrize_marked_value(self, pytester: Pytester, strict: bool) -> None: - s = f""" - import pytest - - @pytest.mark.parametrize(("n", "expected"), [ + def test_parametrize_marked_value(self, tmp_path: Path, strict: bool) -> None: + @pytest.mark.parametrize( + ("n", "expected"), + [ pytest.param( - 2,3, - marks=pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict}), + 2, + 3, + marks=pytest.mark.xfail( + "sys.version_info > (0, 0, 0)", + reason="some bug", + strict=strict, + ), ), pytest.param( - 2,3, - marks=[pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict})], + 2, + 3, + marks=[ + pytest.mark.xfail( + "sys.version_info > (0, 0, 0)", + reason="some bug", + strict=strict, + ) + ], ), - ]) - def test_increment(n, expected): - assert n + 1 == expected - """ - pytester.makepyfile(s) - reprec = pytester.inline_run() - passed, failed = (0, 2) if strict else (2, 0) - reprec.assertoutcome(passed=passed, failed=failed) - - def test_pytest_make_parametrize_id(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - def pytest_make_parametrize_id(config, val): - return str(val * 2) - """ + ], ) - pytester.makepyfile( - """ - import pytest + def test_increment(n, expected): + assert n + 1 == expected + + record = run_tests(test_increment, rootpath=tmp_path) + # ensemble: HookRecorder.assertoutcome() counted the non-strict xpasses + # as plain passes; RunRecord reports them as xpassed. + if strict: + record.assert_outcomes(failed=2) + else: + record.assert_outcomes(xpassed=2) + + def test_pytest_make_parametrize_id(self, tmp_path: Path) -> None: + # ensemble: a conftest-level hook becomes a plugin object. + class ConftestPlugin: + def pytest_make_parametrize_id(self, config, val): + return str(val * 2) - @pytest.mark.parametrize("x", range(2)) - def test_func(x): - pass - """ - ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines(["*test_func*0*PASS*", "*test_func*2*PASS*"]) + @pytest.mark.parametrize("x", range(2)) + def test_func(x): + pass - def test_pytest_make_parametrize_id_with_argname(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - def pytest_make_parametrize_id(config, val, argname): - return str(val * 2 if argname == 'x' else val * 10) - """ - ) - pytester.makepyfile( - """ - import pytest + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(test_func, spec=spec) + assert [nodeid.rpartition("::")[2] for nodeid in record.by_test] == [ + "test_func[0]", + "test_func[2]", + ] + record.assert_outcomes(passed=2) - @pytest.mark.parametrize("x", range(2)) - def test_func_a(x): - pass + def test_pytest_make_parametrize_id_with_argname(self, tmp_path: Path) -> None: + # ensemble: a conftest-level hook becomes a plugin object. + class ConftestPlugin: + def pytest_make_parametrize_id(self, config, val, argname): + return str(val * 2 if argname == "x" else val * 10) - @pytest.mark.parametrize("y", [1]) - def test_func_b(y): - pass - """ - ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( - ["*test_func_a*0*PASS*", "*test_func_a*2*PASS*", "*test_func_b*10*PASS*"] - ) + @pytest.mark.parametrize("x", range(2)) + def test_func_a(x): + pass + + @pytest.mark.parametrize("y", [1]) + def test_func_b(y): + pass + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(test_func_a, test_func_b, spec=spec) + assert [nodeid.rpartition("::")[2] for nodeid in record.by_test] == [ + "test_func_a[0]", + "test_func_a[2]", + "test_func_b[10]", + ] + record.assert_outcomes(passed=3) - def test_parametrize_positional_args(self, pytester: Pytester) -> None: + def test_parametrize_positional_args(self, tmp_path: Path) -> None: """`indirect` and later arguments are keyword-only.""" - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize("a", [1], False) - def test_foo(a): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*TypeError*positional argument*"]) - result.assert_outcomes(errors=1) + @pytest.mark.parametrize("a", [1], False) # type: ignore[call-arg] + def test_foo(a): + pass - def test_parametrize_iterator(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import itertools - import pytest + record = run_tests(test_foo, rootpath=tmp_path, capture_output=True) + record.stdout.fnmatch_lines(["*TypeError*positional argument*"]) + record.assert_outcomes(errors=1) - id_parametrize = pytest.mark.parametrize( - ids=("param%d" % i for i in itertools.count()) - ) + def test_parametrize_iterator(self, tmp_path: Path) -> None: + id_parametrize = pytest.mark.parametrize( # type: ignore[call-arg] + ids=(f"param{i}" for i in itertools.count()) + ) - @id_parametrize('y', ['a', 'b']) - def test1(y): - pass + @id_parametrize("y", ["a", "b"]) + def test1(y): + pass - @id_parametrize('y', ['a', 'b']) - def test2(y): - pass + @id_parametrize("y", ["a", "b"]) + def test2(y): + pass - @pytest.mark.parametrize("a, b", [(1, 2), (3, 4)], ids=itertools.count()) - def test_converted_to_str(a, b): - pass - """ - ) - result = pytester.runpytest("-vv", "-s") - result.stdout.fnmatch_lines( - [ - "test_parametrize_iterator.py::test1[param0] PASSED", - "test_parametrize_iterator.py::test1[param1] PASSED", - "test_parametrize_iterator.py::test2[param0] PASSED", - "test_parametrize_iterator.py::test2[param1] PASSED", - "test_parametrize_iterator.py::test_converted_to_str[0] PASSED", - "test_parametrize_iterator.py::test_converted_to_str[1] PASSED", - "*= 6 passed in *", - ] - ) + @pytest.mark.parametrize("a, b", [(1, 2), (3, 4)], ids=itertools.count()) + def test_converted_to_str(a, b): + pass + + # ensemble: the collection order is the order the members are listed + # in, which the shared ids iterator depends on. + record = run_tests( + build_module( + "test_parametrize_iterator", test1, test2, test_converted_to_str + ), + rootpath=tmp_path, + ) + assert list(record.by_test) == [ + "test_parametrize_iterator.py::test1[param0]", + "test_parametrize_iterator.py::test1[param1]", + "test_parametrize_iterator.py::test2[param0]", + "test_parametrize_iterator.py::test2[param1]", + "test_parametrize_iterator.py::test_converted_to_str[0]", + "test_parametrize_iterator.py::test_converted_to_str[1]", + ] + record.assert_outcomes(passed=6) class TestHiddenParam: """Test that pytest.HIDDEN_PARAM works""" - def test_parametrize_ids(self, pytester: Pytester) -> None: - items = pytester.getitems( - """ - import pytest - - @pytest.mark.parametrize( - ("foo", "bar"), - [ - ("a", "x"), - ("b", "y"), - ("c", "z"), - ], - ids=["paramset1", pytest.HIDDEN_PARAM, "paramset3"], - ) - def test_func(foo, bar): - pass - """ + def test_parametrize_ids(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + ("foo", "bar"), + [ + ("a", "x"), + ("b", "y"), + ("c", "z"), + ], + ids=["paramset1", pytest.HIDDEN_PARAM, "paramset3"], ) + def test_func(foo, bar): + pass + + items = collect_tests(test_func, rootpath=tmp_path) names = [item.name for item in items] assert names == [ "test_func[paramset1]", @@ -2420,23 +2481,19 @@ def test_func(foo, bar): "test_func[paramset3]", ] - def test_param_id(self, pytester: Pytester) -> None: - items = pytester.getitems( - """ - import pytest - - @pytest.mark.parametrize( - ("foo", "bar"), - [ - pytest.param("a", "x", id="paramset1"), - pytest.param("b", "y", id=pytest.HIDDEN_PARAM), - ("c", "z"), - ], - ) - def test_func(foo, bar): - pass - """ + def test_param_id(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + ("foo", "bar"), + [ + pytest.param("a", "x", id="paramset1"), + pytest.param("b", "y", id=pytest.HIDDEN_PARAM), + ("c", "z"), + ], ) + def test_func(foo, bar): + pass + + items = collect_tests(test_func, rootpath=tmp_path) names = [item.name for item in items] assert names == [ "test_func[paramset1]", @@ -2444,25 +2501,27 @@ def test_func(foo, bar): "test_func[c-z]", ] - def test_multiple_hidden_param_is_forbidden(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - - @pytest.mark.parametrize( - ("foo", "bar"), - [ - ("a", "x"), - ("b", "y"), - ], - ids=[pytest.HIDDEN_PARAM, pytest.HIDDEN_PARAM], - ) - def test_func(foo, bar): - pass - """ + def test_multiple_hidden_param_is_forbidden(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + ("foo", "bar"), + [ + ("a", "x"), + ("b", "y"), + ], + ids=[pytest.HIDDEN_PARAM, pytest.HIDDEN_PARAM], ) - result = pytester.runpytest("--collect-only") - result.stdout.fnmatch_lines( + def test_func(foo, bar): + pass + + # ensemble: the module name is part of the reported nodeid. An + # ensemble never aborts the session, so the two lines the original + # matched about that ("! Interrupted: 1 error during collection !" and + # "no tests collected") have no equivalent; the structured error count + # is asserted instead. + module = build_module("test_multiple_hidden_param_is_forbidden", test_func) + record = run_tests(module, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( [ "collected 0 items / 1 error", "", @@ -2470,8 +2529,6 @@ def test_func(foo, bar): "*_ ERROR collecting test_multiple_hidden_param_is_forbidden.py _*", "E Failed: In test_multiple_hidden_param_is_forbidden.py::test_func: multiple instances of " "HIDDEN_PARAM cannot be used in the same parametrize call, because the tests names need to be unique.", - "*! Interrupted: 1 error during collection !*", - "*= no tests collected, 1 error in *", ] ) @@ -2493,24 +2550,20 @@ def test_idmaker_error_without_nodeid(self) -> None: with pytest.raises(Failed, match="ids contains unsupported value"): id_maker.make_unique_parameterset_ids() - def test_multiple_parametrize(self, pytester: Pytester) -> None: - items = pytester.getitems( - """ - import pytest - - @pytest.mark.parametrize( - "bar", - ["x", "y"], - ) - @pytest.mark.parametrize( - "foo", - ["a", "b"], - ids=["a", pytest.HIDDEN_PARAM], - ) - def test_func(foo, bar): - pass - """ + def test_multiple_parametrize(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + "bar", + ["x", "y"], + ) + @pytest.mark.parametrize( + "foo", + ["a", "b"], + ids=["a", pytest.HIDDEN_PARAM], ) + def test_func(foo, bar): + pass + + items = collect_tests(test_func, rootpath=tmp_path) names = [item.name for item in items] assert names == [ "test_func[a-x]", From df394605ae75d51b20090953b85f4efc6919e855 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 19:39:02 +0200 Subject: [PATCH 03/30] testing: port test_unittest.py to _pytest.ensemble 59 of 70 tests ported. Setup/teardown ordering tests record into closure lists asserted directly, which removes the smuggling of state through module attributes and through monkeypatched sys attributes. Outcome categories are corrected where HookRecorder's counting differed from the terminal's, each with a note: xunit-style setup failures are `errors`, trial todo and module-level pytestmark are `xfailed`, and one test was silently ignoring a teardown error. unittest's own setUp/tearDown failures stay call-phase `failed` - the two mappings genuinely differ. Moving sources out of makepyfile strings put them under ruff and mypy for the first time, which exposed two undefined names that had been sitting in the strings. The one in test_unittest_skip_issue148 is now a closure-list assertion that actually proves setUpClass/tearDownClass did not run. 11 stay on pytester: example-script driven (5), subprocess (3), pexpect (1), --fixtures output (1), and one deliberate file-based canary so real module import and path-based nodeids remain covered here. --- testing/test_unittest.py | 2343 ++++++++++++++++++-------------------- 1 file changed, 1125 insertions(+), 1218 deletions(-) diff --git a/testing/test_unittest.py b/testing/test_unittest.py index 20287d12cb3..12844c46631 100644 --- a/testing/test_unittest.py +++ b/testing/test_unittest.py @@ -1,195 +1,189 @@ # mypy: allow-untyped-defs from __future__ import annotations +import abc +import gc +from pathlib import Path import sys +import unittest +import _pytest._code from _pytest.config import ExitCode +from _pytest.ensemble import build_module +from _pytest.ensemble import collect_tests +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import run_tests from _pytest.monkeypatch import MonkeyPatch +from _pytest.outcomes import Exit from _pytest.pytester import Pytester import pytest -def test_simple_unittest(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - def testpassing(self): - self.assertEqual('foo', 'foo') - def test_failing(self): - self.assertEqual('foo', 'bar') - """ - ) - reprec = pytester.inline_run(testpath) - assert reprec.matchreport("testpassing").passed - assert reprec.matchreport("test_failing").failed +def test_simple_unittest(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + def testpassing(self): + self.assertEqual("foo", "foo") + def test_failing(self): + self.assertEqual("foo", "bar") -def test_runTest_method(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import unittest - class MyTestCaseWithRunTest(unittest.TestCase): - def runTest(self): - self.assertEqual('foo', 'foo') - class MyTestCaseWithoutRunTest(unittest.TestCase): - def runTest(self): - self.assertEqual('foo', 'foo') - def test_something(self): - pass - """ - ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( - """ - *MyTestCaseWithRunTest::runTest* - *MyTestCaseWithoutRunTest::test_something* - *2 passed* - """ - ) + record = run_tests(MyTestCase, rootpath=tmp_path) + assert record["testpassing"].passed + assert record["test_failing"].failed -def test_isclasscheck_issue53(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class _E(object): - def __getattr__(self, tag): - pass - E = _E() - """ - ) - result = pytester.runpytest(testpath) - assert result.ret == ExitCode.NO_TESTS_COLLECTED +def test_runTest_method(tmp_path: Path) -> None: + class MyTestCaseWithRunTest(unittest.TestCase): + def runTest(self): + self.assertEqual("foo", "foo") + class MyTestCaseWithoutRunTest(unittest.TestCase): + def runTest(self): + self.assertEqual("foo", "foo") -def test_setup(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - def setUp(self): - self.foo = 1 - def setup_method(self, method): - self.foo2 = 1 - def test_both(self): - self.assertEqual(1, self.foo) - assert self.foo2 == 1 - def teardown_method(self, method): - assert 0, "42" + def test_something(self): + pass - """ - ) - reprec = pytester.inline_run("-s", testpath) - assert reprec.matchreport("test_both", when="call").passed - rep = reprec.matchreport("test_both", when="teardown") - assert rep.failed and "42" in str(rep.longrepr) + sources = (MyTestCaseWithRunTest, MyTestCaseWithoutRunTest) + items = collect_tests(*sources, rootpath=tmp_path) + assert [item.nodeid.split("::", 1)[1] for item in items] == [ + "MyTestCaseWithRunTest::runTest", + "MyTestCaseWithoutRunTest::test_something", + ] + run_tests(*sources, rootpath=tmp_path).assert_outcomes(passed=2) -def test_setUpModule(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - values = [] +def test_isclasscheck_issue53(tmp_path: Path) -> None: + class _E: + def __getattr__(self, tag): + pass - def setUpModule(): - values.append(1) + module = build_module("test_isclasscheck_issue53", E=_E()) + assert collect_tests(module, rootpath=tmp_path) == [] - def tearDownModule(): - del values[0] - def test_hello(): - assert values == [1] +def test_setup(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + def setUp(self): + self.foo = 1 - def test_world(): - assert values == [1] - """ - ) - result = pytester.runpytest(testpath) - result.stdout.fnmatch_lines(["*2 passed*"]) + def setup_method(self, method): + self.foo2 = 1 + def test_both(self): + self.assertEqual(1, self.foo) + assert self.foo2 == 1 -def test_setUpModule_failing_no_teardown(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - values = [] + def teardown_method(self, method): + assert 0, "42" - def setUpModule(): - 0/0 + record = run_tests(MyTestCase, rootpath=tmp_path) + call = record["test_both"].call + assert call is not None and call.passed + teardown = record["test_both"].teardown + assert teardown is not None + assert teardown.failed and "42" in teardown.longreprtext - def tearDownModule(): - values.append(1) - def test_hello(): - pass - """ +def test_setUpModule(tmp_path: Path) -> None: + # the module level ``values`` of the original became a closure: a source + # collected in-memory keeps this module's globals, so a module level list + # would be *this* file's. + values = [] + + def setUpModule(): + values.append(1) + + def tearDownModule(): + del values[0] + + def test_hello(): + assert values == [1] + + def test_world(): + assert values == [1] + + module = build_module( + "test_setUpModule", setUpModule, tearDownModule, test_hello, test_world ) - reprec = pytester.inline_run(testpath) - reprec.assertoutcome(passed=0, failed=1) - call = reprec.getcalls("pytest_runtest_setup")[0] - assert not call.item.module.values + run_tests(module, rootpath=tmp_path).assert_outcomes(passed=2) + assert values == [] -def test_new_instances(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - def test_func1(self): - self.x = 2 - def test_func2(self): - assert not hasattr(self, 'x') - """ +def test_setUpModule_failing_no_teardown(tmp_path: Path) -> None: + values = [] + + def setUpModule(): + 0 / 0 # noqa: B018 + + def tearDownModule(): + values.append(1) + + def test_hello(): + pass + + module = build_module( + "test_setUpModule_failing_no_teardown", + setUpModule, + tearDownModule, + test_hello, ) - reprec = pytester.inline_run(testpath) - reprec.assertoutcome(passed=2) + record = run_tests(module, rootpath=tmp_path) + # setUpModule is an xunit *setup* fixture, so the terminal category is + # "error" where HookRecorder.assertoutcome() only counted a failed report. + record.assert_outcomes(passed=0, errors=1) + assert values == [] + +def test_new_instances(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + def test_func1(self): + self.x = 2 -def test_function_item_obj_is_instance(pytester: Pytester) -> None: + def test_func2(self): + assert not hasattr(self, "x") + + run_tests(MyTestCase, rootpath=tmp_path).assert_outcomes(passed=2) + + +def test_function_item_obj_is_instance(tmp_path: Path) -> None: """item.obj should be a bound method on unittest.TestCase function items (#5390).""" - pytester.makeconftest( - """ - def pytest_runtest_makereport(item, call): - if call.when == 'call': + checked: list[bool] = [] + + class CheckPlugin: + def pytest_runtest_makereport(self, item, call): + if call.when == "call": class_ = item.parent.obj - assert isinstance(item.obj.__self__, class_) - """ - ) - pytester.makepyfile( - """ - import unittest + checked.append(isinstance(item.obj.__self__, class_)) - class Test(unittest.TestCase): - def test_foo(self): - pass - """ - ) - result = pytester.runpytest_inprocess() - result.stdout.fnmatch_lines(["* 1 passed in*"]) + class Test(unittest.TestCase): + def test_foo(self): + pass + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(CheckPlugin(),)) + run_tests(Test, spec=spec).assert_outcomes(passed=1) + assert checked == [True] -def test_teardown(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - values = [] - def test_one(self): - pass - def tearDown(self): - self.values.append(None) - class Second(unittest.TestCase): - def test_check(self): - self.assertEqual(MyTestCase.values, [None]) - """ - ) - reprec = pytester.inline_run(testpath) - passed, skipped, failed = reprec.countoutcomes() - assert failed == 0, failed - assert passed == 2 - assert passed + skipped + failed == 2 + +def test_teardown(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + # deliberately shared class state, observed by Second below + values: list[None] = [] + + def test_one(self): + pass + + def tearDown(self): + self.values.append(None) + + class Second(unittest.TestCase): + def test_check(self): + self.assertEqual(MyTestCase.values, [None]) + + run_tests(MyTestCase, Second, rootpath=tmp_path).assert_outcomes(passed=2) -def test_teardown_issue1649(pytester: Pytester) -> None: +def test_teardown_issue1649(tmp_path: Path) -> None: """ Are TestCase objects cleaned up? Often unittest TestCase objects set attributes that are large and expensive during test run or setUp. @@ -199,140 +193,126 @@ def test_teardown_issue1649(pytester: Pytester) -> None: Regression test for #1649 (see also #12367). """ - pytester.makepyfile( - """ - import unittest - import gc - - class TestCaseObjectsShouldBeCleanedUp(unittest.TestCase): - def test_expensive(self): - self.an_expensive_obj = object() - - def test_is_it_still_alive(self): - gc.collect() - for obj in gc.get_objects(): - if type(obj).__name__ == "TestCaseObjectsShouldBeCleanedUp": - assert not hasattr(obj, "an_expensive_obj") - break - else: - assert False, "Could not find TestCaseObjectsShouldBeCleanedUp instance" - """ - ) - result = pytester.runpytest() - assert result.ret == ExitCode.OK + class TestCaseObjectsShouldBeCleanedUp(unittest.TestCase): + def test_expensive(self): + self.an_expensive_obj = object() + def test_is_it_still_alive(self): + gc.collect() + for obj in gc.get_objects(): + if type(obj).__name__ == "TestCaseObjectsShouldBeCleanedUp": + assert not hasattr(obj, "an_expensive_obj") + break + else: + assert False, "Could not find TestCaseObjectsShouldBeCleanedUp instance" -def test_unittest_skip_issue148(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest + record = run_tests(TestCaseObjectsShouldBeCleanedUp, rootpath=tmp_path) + record.assert_outcomes(passed=2) - @unittest.skip("hello") - class MyTestCase(unittest.TestCase): - @classmethod - def setUpClass(self): - xxx - def test_one(self): - pass - @classmethod - def tearDownClass(self): - xxx - """ - ) - reprec = pytester.inline_run(testpath) - reprec.assertoutcome(skipped=1) +def test_unittest_skip_issue148(tmp_path: Path) -> None: + ran = [] -def test_unittest_skip_with_autouse_fixture(pytester: Pytester) -> None: + @unittest.skip("hello") + class MyTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + ran.append("setUpClass") + + def test_one(self): + ran.append("test_one") + + @classmethod + def tearDownClass(cls): + ran.append("tearDownClass") + + run_tests(MyTestCase, rootpath=tmp_path).assert_outcomes(skipped=1) + # the original smuggled this in as a NameError on an undefined name + assert ran == [] + + +def test_unittest_skip_with_autouse_fixture(tmp_path: Path) -> None: """Autouse fixtures inside a @unittest.skipIf class should not run (#13885).""" - pytester.makepyfile( - """ - import unittest - import pytest - @unittest.skipIf(True, "skip reason") - class TestSkipped(unittest.TestCase): - @pytest.fixture(autouse=True) - def my_fixture(self): - raise RuntimeError("fixture should not run") + @unittest.skipIf(True, "skip reason") + class TestSkipped(unittest.TestCase): + @pytest.fixture(autouse=True) + def my_fixture(self): + raise RuntimeError("fixture should not run") - def test_one(self): - pass - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(skipped=1) + def test_one(self): + pass + run_tests(TestSkipped, rootpath=tmp_path).assert_outcomes(skipped=1) -def test_method_and_teardown_failing_reporting(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import unittest - class TC(unittest.TestCase): - def tearDown(self): - assert 0, "down1" - def test_method(self): - assert False, "down2" - """ - ) - result = pytester.runpytest("-s") - assert result.ret == 1 - result.stdout.fnmatch_lines( - [ - "*tearDown*", - "*assert 0*", - "*test_method*", - "*assert False*", - "*1 failed*1 error*", - ] - ) +def test_method_and_teardown_failing_reporting(tmp_path: Path) -> None: + class TC(unittest.TestCase): + def tearDown(self): + assert 0, "down1" -def test_setup_failure_is_shown(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import unittest - import pytest - class TC(unittest.TestCase): - def setUp(self): - assert 0, "down1" - def test_method(self): - print("never42") - xyz - """ - ) - result = pytester.runpytest("-s") - assert result.ret == 1 - result.stdout.fnmatch_lines(["*setUp*", "*assert 0*down1*", "*1 failed*"]) - result.stdout.no_fnmatch_line("*never42*") + def test_method(self): + assert False, "down2" + record = run_tests(TC, rootpath=tmp_path) + record.assert_outcomes(failed=1, errors=1) + call = record["test_method"].call + assert call is not None and call.failed + assert "test_method" in call.longreprtext and "down2" in call.longreprtext + teardown = record["test_method"].teardown + assert teardown is not None and teardown.failed + assert "tearDown" in teardown.longreprtext and "down1" in teardown.longreprtext -def test_setup_setUpClass(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - import pytest - class MyTestCase(unittest.TestCase): - x = 0 - @classmethod - def setUpClass(cls): - cls.x += 1 - def test_func1(self): - assert self.x == 1 - def test_func2(self): - assert self.x == 1 - @classmethod - def tearDownClass(cls): - cls.x -= 1 - def test_torn_down(): - assert MyTestCase.x == 0 - """ - ) - reprec = pytester.inline_run(testpath) - reprec.assertoutcome(passed=3) +def test_setup_failure_is_shown(tmp_path: Path) -> None: + ran = [] + + class TC(unittest.TestCase): + def setUp(self): + assert 0, "down1" + + def test_method(self): + ran.append("test_method") + + record = run_tests(TC, rootpath=tmp_path) + # a failing unittest setUp is reported in the call phase, not as an error + record.assert_outcomes(failed=1) + call = record["test_method"].call + assert call is not None + assert "setUp" in call.longreprtext and "down1" in call.longreprtext + # the test body itself must not have run (was: no_fnmatch_line("*never42*")) + assert ran == [] + + +def test_setup_setUpClass(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + x = 0 + + @classmethod + def setUpClass(cls): + cls.x += 1 + + def test_func1(self): + assert self.x == 1 + def test_func2(self): + assert self.x == 1 + + @classmethod + def tearDownClass(cls): + cls.x -= 1 + + # collection order follows the line numbers in *this* file, so + # test_torn_down has to stay defined below the class it checks + def test_torn_down(): + assert MyTestCase.x == 0 + + record = run_tests(MyTestCase, test_torn_down, rootpath=tmp_path) + record.assert_outcomes(passed=3) + + +# ensemble: --fixtures output has no in-memory equivalent def test_fixtures_setup_setUpClass_issue8394(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -357,107 +337,104 @@ def tearDownClass(cls): result.stdout.fnmatch_lines(["*no docstring available*"]) -def test_setup_class(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - import pytest - class MyTestCase(unittest.TestCase): - x = 0 - def setup_class(cls): - cls.x += 1 - def test_func1(self): - assert self.x == 1 - def test_func2(self): - assert self.x == 1 - def teardown_class(cls): - cls.x -= 1 - def test_torn_down(): - assert MyTestCase.x == 0 - """ - ) - reprec = pytester.inline_run(testpath) - reprec.assertoutcome(passed=3) +def test_setup_class(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + x = 0 + + def setup_class(cls): + cls.x += 1 + + def test_func1(self): + assert self.x == 1 + + def test_func2(self): + assert self.x == 1 + + def teardown_class(cls): + cls.x -= 1 + + # must stay below the class: collection order follows this file's lines + def test_torn_down(): + assert MyTestCase.x == 0 + + record = run_tests(MyTestCase, test_torn_down, rootpath=tmp_path) + record.assert_outcomes(passed=3) @pytest.mark.parametrize("type", ["Error", "Failure"]) -def test_testcase_adderrorandfailure_defers(pytester: Pytester, type: str) -> None: - pytester.makepyfile( - f""" - from unittest import TestCase - import pytest - class MyTestCase(TestCase): - def run(self, result): - excinfo = pytest.raises(ZeroDivisionError, lambda: 0/0) - try: - result.add{type}(self, excinfo._excinfo) - except KeyboardInterrupt: - raise - except: - pytest.fail("add{type} should not raise") - def test_hello(self): - pass - """ - ) - result = pytester.runpytest() - result.stdout.no_fnmatch_line("*should not raise*") +def test_testcase_adderrorandfailure_defers(tmp_path: Path, type: str) -> None: + raised: list[BaseException] = [] + + class MyTestCase(unittest.TestCase): + def run(self, result=None): + excinfo = pytest.raises(ZeroDivisionError, lambda: 0 / 0) + try: + getattr(result, f"add{type}")(self, excinfo._excinfo) + except KeyboardInterrupt: + raise + except BaseException as e: + # was: pytest.fail(f"add{type} should not raise") + raised.append(e) + + def test_hello(self): + pass + + record = run_tests(MyTestCase, rootpath=tmp_path) + assert raised == [] + # the deferred exception info surfaces as the call phase failure + record.assert_outcomes(failed=1) @pytest.mark.parametrize("type", ["Error", "Failure"]) -def test_testcase_custom_exception_info(pytester: Pytester, type: str) -> None: - pytester.makepyfile( - f""" - from typing import Generic, TypeVar - from unittest import TestCase - import pytest, _pytest._code - - class MyTestCase(TestCase): - def run(self, result): - excinfo = pytest.raises(ZeroDivisionError, lambda: 0/0) - # We fake an incompatible exception info. - class FakeExceptionInfo(Generic[TypeVar("E")]): - def __init__(self, *args, **kwargs): - mp.undo() - raise TypeError() - @classmethod - def from_current(cls): - return cls() - @classmethod - def from_exc_info(cls, *args, **kwargs): - return cls() - mp = pytest.MonkeyPatch() - mp.setattr(_pytest._code, 'ExceptionInfo', FakeExceptionInfo) - try: - excinfo = excinfo._excinfo - result.add{type}(self, excinfo) - finally: +def test_testcase_custom_exception_info(tmp_path: Path, type: str) -> None: + class MyTestCase(unittest.TestCase): + def run(self, result=None): + excinfo = pytest.raises(ZeroDivisionError, lambda: 0 / 0) + + # We fake an incompatible exception info. + class FakeExceptionInfo: + def __init__(self, *args, **kwargs): mp.undo() + raise TypeError - def test_hello(self): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - "NOTE: Incompatible Exception Representation*", - "*ZeroDivisionError*", - "*1 failed*", - ] - ) + # stands in for Generic[E]: only used as ExceptionInfo[...] + def __class_getitem__(cls, item): + return cls + + @classmethod + def from_current(cls): + return cls() + + @classmethod + def from_exc_info(cls, *args, **kwargs): + return cls() + + mp = pytest.MonkeyPatch() + mp.setattr(_pytest._code, "ExceptionInfo", FakeExceptionInfo) + try: + getattr(result, f"add{type}")(self, excinfo._excinfo) + finally: + mp.undo() + + def test_hello(self): + pass + + record = run_tests(MyTestCase, rootpath=tmp_path) + record.assert_outcomes(failed=1) + call = record["test_hello"].call + assert call is not None + assert "NOTE: Incompatible Exception Representation" in call.longreprtext + assert "ZeroDivisionError" in call.longreprtext -def test_testcase_totally_incompatible_exception_info(pytester: Pytester) -> None: +def test_testcase_totally_incompatible_exception_info(tmp_path: Path) -> None: import _pytest.unittest - (item,) = pytester.getitems( - """ - from unittest import TestCase - class MyTestCase(TestCase): - def test_hello(self): - pass - """ - ) + class MyTestCase(unittest.TestCase): + def test_hello(self): + pass + + (item,) = collect_tests(MyTestCase, rootpath=tmp_path) assert isinstance(item, _pytest.unittest.TestCaseFunction) item.addError(None, 42) # type: ignore[arg-type] excinfo = item._excinfo @@ -465,19 +442,33 @@ def test_hello(self): assert "ERROR: Unknown Incompatible" in str(excinfo.pop(0).getrepr()) -def test_module_level_pytestmark(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - import pytest - pytestmark = pytest.mark.xfail - class MyTestCase(unittest.TestCase): - def test_func1(self): - assert 0 - """ +def test_module_level_pytestmark(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + def test_func1(self): + assert 0 + + module = build_module( + "test_module_level_pytestmark", MyTestCase, pytestmark=pytest.mark.xfail ) - reprec = pytester.inline_run(testpath, "-s") - reprec.assertoutcome(skipped=1) + record = run_tests(module, rootpath=tmp_path) + # assertoutcome() counted the xfail report as skipped; the terminal + # category it lands in is xfailed. + record.assert_outcomes(xfailed=1) + + +def set_attributes(**attrs: object): + """Set attributes on a test method (trial's ``skip``/``todo``, ``__test__``). + + In the file based originals these were plain assignments in the class + body, which type checkers reject on real (non-``exec``'d) code. + """ + + def decorate(func): + for name, value in attrs.items(): + setattr(func, name, value) + return func + + return decorate class TestTrialUnittest: @@ -485,130 +476,141 @@ def setup_class(cls): cls.ut = pytest.importorskip("twisted.trial.unittest") # on windows trial uses a socket for a reactor and apparently doesn't close it properly # https://twistedmatrix.com/trac/ticket/9227 - cls.ignore_unclosed_socket_warning = ("-W", "always") + # (was "-W always" on the command line; an ensemble inherits the host + # suite's filterwarnings=error unless its own inicfg says otherwise) + cls.ignore_unclosed_socket_inicfg = {"filterwarnings": ["always"]} - def test_trial_testcase_runtest_not_collected(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - from twisted.trial.unittest import TestCase + def spec(self, tmp_path: Path) -> ConfigSpec: + return ConfigSpec(rootpath=tmp_path, inicfg=self.ignore_unclosed_socket_inicfg) - class TC(TestCase): - def test_hello(self): - pass - """ - ) - reprec = pytester.inline_run(*self.ignore_unclosed_socket_warning) - reprec.assertoutcome(passed=1) - pytester.makepyfile( - """ - from twisted.trial.unittest import TestCase + def test_trial_testcase_runtest_not_collected(self, tmp_path: Path) -> None: + from twisted.trial.unittest import TestCase as TrialTestCase - class TC(TestCase): - def runTest(self): - pass - """ - ) - reprec = pytester.inline_run(*self.ignore_unclosed_socket_warning) - reprec.assertoutcome(passed=1) + spec = self.spec(tmp_path) - def test_trial_exceptions_with_skips(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - from twisted.trial import unittest - import pytest - class TC(unittest.TestCase): - def test_hello(self): - pytest.skip("skip_in_method") - @pytest.mark.skipif("sys.version_info != 1") - def test_hello2(self): - pass - @pytest.mark.xfail(reason="iwanto") - def test_hello3(self): - assert 0 - def test_hello4(self): - pytest.xfail("i2wanto") - def test_trial_skip(self): - pass - test_trial_skip.skip = "trialselfskip" - - def test_trial_todo(self): - assert 0 - test_trial_todo.todo = "mytodo" - - def test_trial_todo_success(self): - pass - test_trial_todo_success.todo = "mytodo" - - class TC2(unittest.TestCase): - def setup_class(cls): - pytest.skip("skip_in_setup_class") - def test_method(self): - pass - """ - ) - result = pytester.runpytest("-rxs", *self.ignore_unclosed_socket_warning) - result.stdout.fnmatch_lines_random( - [ - "*XFAIL*test_trial_todo*", - "*trialselfskip*", - "*skip_in_setup_class*", - "*iwanto*", - "*i2wanto*", - "*sys.version_info*", - "*skip_in_method*", - "*1 failed*4 skipped*3 xfailed*", - ] - ) - assert result.ret == 1 + class TC(TrialTestCase): + def test_hello(self): + pass - def test_trial_error(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - from twisted.trial.unittest import TestCase - from twisted.internet.defer import Deferred - from twisted.internet import reactor - - class TC(TestCase): - def test_one(self): - crash - - def test_two(self): - def f(_): - crash - - d = Deferred() - d.addCallback(f) - reactor.callLater(0.3, d.callback, None) - return d - - def test_three(self): - def f(): - pass # will never get called + # trial's own inherited runTest is not collected next to test_hello + run_tests(TC, spec=spec).assert_outcomes(passed=1) + + class TCWithRunTest(TrialTestCase): + def runTest(self): + pass + + run_tests(TCWithRunTest, spec=spec).assert_outcomes(passed=1) + + def test_trial_exceptions_with_skips(self, tmp_path: Path) -> None: + from twisted.trial.unittest import TestCase as TrialTestCase + + class TC(TrialTestCase): + def test_hello(self): + pytest.skip("skip_in_method") + + @pytest.mark.skipif("sys.version_info != 1") + def test_hello2(self): + pass + + @pytest.mark.xfail(reason="iwanto") + def test_hello3(self): + assert 0 + + def test_hello4(self): + pytest.xfail("i2wanto") + + @set_attributes(skip="trialselfskip") + def test_trial_skip(self): + pass + + @set_attributes(todo="mytodo") + def test_trial_todo(self): + assert 0 + + @set_attributes(todo="mytodo") + def test_trial_todo_success(self): + pass + + class TC2(TrialTestCase): + def setup_class(cls): + pytest.skip("skip_in_setup_class") + + def test_method(self): + pass + + record = run_tests(TC, TC2, spec=self.spec(tmp_path)) + record.assert_outcomes(failed=1, skipped=4, xfailed=3) + + def reason(name: str) -> str: + item = record[name] + report = item.call if item.call is not None else item.setup + assert report is not None + return getattr(report, "wasxfail", "") + report.longreprtext + + assert "skip_in_method" in reason("test_hello") + assert "sys.version_info" in reason("test_hello2") + assert "iwanto" in reason("test_hello3") + assert "i2wanto" in reason("test_hello4") + assert "trialselfskip" in reason("test_trial_skip") + assert "mytodo" in reason("test_trial_todo") + assert record["test_trial_todo"].outcome == "xfailed" + assert record["test_trial_todo_success"].failed + assert "skip_in_setup_class" in reason("test_method") + + def test_trial_error(self, tmp_path: Path) -> None: + from twisted.internet import reactor + from twisted.internet.defer import Deferred + from twisted.trial.unittest import TestCase as TrialTestCase + + class TC(TrialTestCase): + def test_one(self): + raise NameError("crash") + + def test_two(self): + def f(_): + raise NameError("crash") + + d = Deferred() + d.addCallback(f) + reactor.callLater(0.3, d.callback, None) + return d + + def test_three(self): + def f(): + pass # will never get called + + reactor.callLater(0.3, f) + + # will crash at teardown + + def test_four(self): + def f(_): reactor.callLater(0.3, f) - # will crash at teardown - - def test_four(self): - def f(_): - reactor.callLater(0.3, f) - crash - - d = Deferred() - d.addCallback(f) - reactor.callLater(0.3, d.callback, None) - return d - # will crash both at test time and at teardown - """ - ) - result = pytester.runpytest( - "-vv", "-oconsole_output_style=classic", "-W", "ignore::DeprecationWarning" + raise NameError("crash") + + d = Deferred() + d.addCallback(f) + reactor.callLater(0.3, d.callback, None) + return d + + # will crash both at test time and at teardown + + spec = ConfigSpec( + rootpath=tmp_path, + args=("-vv", "-oconsole_output_style=classic"), + inicfg={"filterwarnings": ["ignore::DeprecationWarning"]}, ) - result.stdout.fnmatch_lines( + # this one is about what gets *rendered*, so keep the glob matching + record = run_tests(TC, spec=spec, name="test_trial_error", capture_output=True) + record.stdout.fnmatch_lines( [ - "test_trial_error.py::TC::test_four FAILED", - "test_trial_error.py::TC::test_four ERROR", - "test_trial_error.py::TC::test_one FAILED", - "test_trial_error.py::TC::test_three FAILED", - "test_trial_error.py::TC::test_two FAILED", + # the ``*`` swallows the "<- " annotation -v + # adds because the in-memory module's path is synthetic + "test_trial_error.py::TC::test_four *FAILED", + "test_trial_error.py::TC::test_four *ERROR", + "test_trial_error.py::TC::test_one *FAILED", + "test_trial_error.py::TC::test_three *FAILED", + "test_trial_error.py::TC::test_two *FAILED", "*ERRORS*", "*_ ERROR at teardown of TC.test_four _*", "*DelayedCalls*", @@ -625,6 +627,7 @@ def f(_): ] ) + # ensemble: needs a terminal to type into (pexpect) def test_trial_pdb(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ @@ -639,167 +642,146 @@ def test_hello(self): child.expect("hellopdb") child.sendeof() - def test_trial_testcase_skip_property(self, pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - from twisted.trial import unittest - class MyTestCase(unittest.TestCase): - skip = 'dont run' - def test_func(self): - pass - """ - ) - reprec = pytester.inline_run(testpath, "-s") - reprec.assertoutcome(skipped=1) + def test_trial_testcase_skip_property(self, tmp_path: Path) -> None: + from twisted.trial.unittest import TestCase as TrialTestCase - def test_trial_testfunction_skip_property(self, pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - from twisted.trial import unittest - class MyTestCase(unittest.TestCase): - def test_func(self): - pass - test_func.skip = 'dont run' - """ - ) - reprec = pytester.inline_run(testpath, "-s") - reprec.assertoutcome(skipped=1) + class MyTestCase(TrialTestCase): + skip = "dont run" - def test_trial_testcase_todo_property(self, pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - from twisted.trial import unittest - class MyTestCase(unittest.TestCase): - todo = 'dont run' - def test_func(self): - assert 0 - """ - ) - reprec = pytester.inline_run(testpath, "-s") - reprec.assertoutcome(skipped=1) + def test_func(self): + pass - def test_trial_testfunction_todo_property(self, pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - from twisted.trial import unittest - class MyTestCase(unittest.TestCase): - def test_func(self): - assert 0 - test_func.todo = 'dont run' - """ - ) - reprec = pytester.inline_run( - testpath, "-s", *self.ignore_unclosed_socket_warning - ) - reprec.assertoutcome(skipped=1) + run_tests(MyTestCase, spec=self.spec(tmp_path)).assert_outcomes(skipped=1) + + def test_trial_testfunction_skip_property(self, tmp_path: Path) -> None: + from twisted.trial.unittest import TestCase as TrialTestCase + + class MyTestCase(TrialTestCase): + @set_attributes(skip="dont run") + def test_func(self): + pass + + run_tests(MyTestCase, spec=self.spec(tmp_path)).assert_outcomes(skipped=1) + + def test_trial_testcase_todo_property(self, tmp_path: Path) -> None: + from twisted.trial.unittest import TestCase as TrialTestCase + + class MyTestCase(TrialTestCase): + todo = "dont run" + + def test_func(self): + assert 0 + + # assertoutcome() counted the xfail report as skipped + run_tests(MyTestCase, spec=self.spec(tmp_path)).assert_outcomes(xfailed=1) + def test_trial_testfunction_todo_property(self, tmp_path: Path) -> None: + from twisted.trial.unittest import TestCase as TrialTestCase -def test_djangolike_testcase(pytester: Pytester) -> None: + class MyTestCase(TrialTestCase): + @set_attributes(todo="dont run") + def test_func(self): + assert 0 + + run_tests(MyTestCase, spec=self.spec(tmp_path)).assert_outcomes(xfailed=1) + + +def test_djangolike_testcase(tmp_path: Path) -> None: # contributed from Morten Breekevold - pytester.makepyfile( - """ - from unittest import TestCase, main - - class DjangoLikeTestCase(TestCase): - - def setUp(self): - print("setUp()") - - def test_presetup_has_been_run(self): - print("test_thing()") - self.assertTrue(hasattr(self, 'was_presetup')) - - def tearDown(self): - print("tearDown()") - - def __call__(self, result=None): - try: - self._pre_setup() - except (KeyboardInterrupt, SystemExit): - raise - except Exception: - import sys - result.addError(self, sys.exc_info()) - return - super(DjangoLikeTestCase, self).__call__(result) - try: - self._post_teardown() - except (KeyboardInterrupt, SystemExit): - raise - except Exception: - import sys - result.addError(self, sys.exc_info()) - return - - def _pre_setup(self): - print("_pre_setup()") - self.was_presetup = True - - def _post_teardown(self): - print("_post_teardown()") - """ - ) - result = pytester.runpytest("-s") - assert result.ret == 0 - result.stdout.fnmatch_lines( - [ - "*_pre_setup()*", - "*setUp()*", - "*test_thing()*", - "*tearDown()*", - "*_post_teardown()*", - ] - ) + events: list[str] = [] + + class DjangoLikeTestCase(unittest.TestCase): + def setUp(self): + events.append("setUp()") + + def test_presetup_has_been_run(self): + events.append("test_thing()") + self.assertTrue(hasattr(self, "was_presetup")) + + def tearDown(self): + events.append("tearDown()") + + def __call__(self, result=None): + try: + self._pre_setup() + except (KeyboardInterrupt, SystemExit): + raise + except Exception: + result.addError(self, sys.exc_info()) + return + super().__call__(result) + try: + self._post_teardown() + except (KeyboardInterrupt, SystemExit): + raise + except Exception: + result.addError(self, sys.exc_info()) + return + + def _pre_setup(self): + events.append("_pre_setup()") + self.was_presetup = True + + def _post_teardown(self): + events.append("_post_teardown()") + + record = run_tests(DjangoLikeTestCase, rootpath=tmp_path) + record.assert_outcomes(passed=1) + # asserting the order directly, instead of globbing printed lines + assert events == [ + "_pre_setup()", + "setUp()", + "test_thing()", + "tearDown()", + "_post_teardown()", + ] -def test_unittest_not_shown_in_traceback(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import unittest - class t(unittest.TestCase): - def test_hello(self): - x = 3 - self.assertEqual(x, 4) - """ - ) - res = pytester.runpytest() - res.stdout.no_fnmatch_line("*failUnlessEqual*") +def test_unittest_not_shown_in_traceback(tmp_path: Path) -> None: + class t(unittest.TestCase): + def test_hello(self): + x = 3 + self.assertEqual(x, 4) + record = run_tests(t, rootpath=tmp_path) + record.assert_outcomes(failed=1) + call = record["test_hello"].call + assert call is not None + # the failing line is shown, unittest's own frames leading to it are not + assert "self.assertEqual(x, 4)" in call.longreprtext + assert "failUnlessEqual" not in call.longreprtext -def test_unorderable_types(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import unittest - class TestJoinEmpty(unittest.TestCase): + +def test_unorderable_types(tmp_path: Path) -> None: + class TestJoinEmpty(unittest.TestCase): + pass + + def make_test(): + class Test(unittest.TestCase): pass - def make_test(): - class Test(unittest.TestCase): - pass - Test.__name__ = "TestFoo" - return Test - TestFoo = make_test() - """ - ) - result = pytester.runpytest() - result.stdout.no_fnmatch_line("*TypeError*") - assert result.ret == ExitCode.NO_TESTS_COLLECTED + Test.__name__ = "TestFoo" + return Test + module = build_module("test_unorderable_types", TestJoinEmpty, make_test()) + # collect_tests() raises on a collection error, so an empty list really + # means "collected nothing", not "blew up on unorderable types" + assert collect_tests(module, rootpath=tmp_path) == [] -def test_unittest_typerror_traceback(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import unittest - class TestJoinEmpty(unittest.TestCase): - def test_hello(self, arg1): - pass - """ - ) - result = pytester.runpytest() - assert "TypeError" in result.stdout.str() - assert result.ret == 1 +def test_unittest_typerror_traceback(tmp_path: Path) -> None: + class TestJoinEmpty(unittest.TestCase): + def test_hello(self, arg1): + pass + record = run_tests(TestJoinEmpty, rootpath=tmp_path) + record.assert_outcomes(failed=1) + call = record["test_hello"].call + assert call is not None + assert "TypeError" in call.longreprtext + + +# ensemble: the "unittest" variant runs the module as a script (runpython) @pytest.mark.parametrize("runner", ["pytest", "unittest"]) def test_unittest_expected_failure_for_failing_test_is_xfail( pytester: Pytester, runner @@ -826,6 +808,7 @@ def test_failing_test_is_xfail(self): assert result.ret == 0 +# ensemble: the "unittest" variant runs the module as a script (runpython) @pytest.mark.parametrize("runner", ["pytest", "unittest"]) def test_unittest_expected_failure_for_passing_test_is_fail( pytester: Pytester, @@ -860,146 +843,142 @@ def test_passing_test_is_fail(self): @pytest.mark.parametrize("stmt", ["return", "yield"]) -def test_unittest_setup_interaction(pytester: Pytester, stmt: str) -> None: - pytester.makepyfile( - f""" - import unittest - import pytest - class MyTestCase(unittest.TestCase): - @pytest.fixture(scope="class", autouse=True) - @classmethod - def perclass(cls, request): - request.cls.hello = "world" - {stmt} +def test_unittest_setup_interaction(tmp_path: Path, stmt: str) -> None: + # the string template parametrized the *shape* of the fixture bodies; + # in-memory sources need both variants spelled out + if stmt == "return": - @pytest.fixture(scope="function", autouse=True) - def perfunction(self, request): - request.instance.funcname = request.function.__name__ - {stmt} + def perclass(cls, request): + request.cls.hello = "world" + return # noqa: PLR1711 - def test_method1(self): - assert self.funcname == "test_method1" - assert self.hello == "world" + def perfunction(self, request): + request.instance.funcname = request.function.__name__ + return # noqa: PLR1711 - def test_method2(self): - assert self.funcname == "test_method2" + else: - def test_classattr(self): - assert self.__class__.hello == "world" - """ + def perclass(cls, request): + request.cls.hello = "world" + yield + + def perfunction(self, request): + request.instance.funcname = request.function.__name__ + yield + + class MyTestCase(unittest.TestCase): + # set by the fixtures below + hello: str + funcname: str + + def test_method1(self): + assert self.funcname == "test_method1" + assert self.hello == "world" + + def test_method2(self): + assert self.funcname == "test_method2" + + def test_classattr(self): + assert self.__class__.hello == "world" + + # the fixtures are attached after the fact: a class body cannot read the + # enclosing function's locals under the same name it binds + MyTestCase.perclass = pytest.fixture(scope="class", autouse=True)( # type: ignore[attr-defined] + classmethod(perclass) # type: ignore[arg-type] ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*3 passed*"]) + MyTestCase.perfunction = pytest.fixture(scope="function", autouse=True)(perfunction) # type: ignore[attr-defined] + run_tests(MyTestCase, rootpath=tmp_path).assert_outcomes(passed=3) -def test_non_unittest_no_setupclass_support(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - class TestFoo(object): - x = 0 - @classmethod - def setUpClass(cls): - cls.x = 1 +def test_non_unittest_no_setupclass_support(tmp_path: Path) -> None: + class TestFoo: + x = 0 - def test_method1(self): - assert self.x == 0 + @classmethod + def setUpClass(cls): + cls.x = 1 - @classmethod - def tearDownClass(cls): - cls.x = 1 + def test_method1(self): + assert self.x == 0 - def test_not_torn_down(): - assert TestFoo.x == 0 + @classmethod + def tearDownClass(cls): + cls.x = 1 - """ - ) - reprec = pytester.inline_run(testpath) - reprec.assertoutcome(passed=2) + # must stay below the class: collection order follows this file's lines + def test_not_torn_down(): + assert TestFoo.x == 0 + record = run_tests(TestFoo, test_not_torn_down, rootpath=tmp_path) + record.assert_outcomes(passed=2) -def test_no_teardown_if_setupclass_failed(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - x = 0 +def test_no_teardown_if_setupclass_failed(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + x = 0 - @classmethod - def setUpClass(cls): - cls.x = 1 - assert False + @classmethod + def setUpClass(cls): + cls.x = 1 + assert False - def test_func1(self): - cls.x = 10 + def test_func1(self): + MyTestCase.x = 10 - @classmethod - def tearDownClass(cls): - cls.x = 100 + @classmethod + def tearDownClass(cls): + cls.x = 100 - def test_notTornDown(): - assert MyTestCase.x == 1 - """ - ) - reprec = pytester.inline_run(testpath) - reprec.assertoutcome(passed=1, failed=1) + # must stay below the class: collection order follows this file's lines + def test_notTornDown(): + assert MyTestCase.x == 1 + record = run_tests(MyTestCase, test_notTornDown, rootpath=tmp_path) + # setUpClass runs as a class scoped fixture, so its failure is a setup + # phase *error* where assertoutcome() only saw a failed report + record.assert_outcomes(passed=1, errors=1) -def test_cleanup_functions(pytester: Pytester) -> None: - """Ensure functions added with addCleanup are always called after each test ends (#6947)""" - pytester.makepyfile( - """ - import unittest - cleanups = [] +def test_cleanup_functions(tmp_path: Path) -> None: + """Ensure functions added with addCleanup are always called after each test ends (#6947)""" + cleanups: list[str] = [] - class Test(unittest.TestCase): + class Test(unittest.TestCase): + def test_func_1(self): + self.addCleanup(cleanups.append, "test_func_1") - def test_func_1(self): - self.addCleanup(cleanups.append, "test_func_1") + def test_func_2(self): + self.addCleanup(cleanups.append, "test_func_2") + assert 0 - def test_func_2(self): - self.addCleanup(cleanups.append, "test_func_2") - assert 0 + def test_func_3_check_cleanups(self): + assert cleanups == ["test_func_1", "test_func_2"] - def test_func_3_check_cleanups(self): - assert cleanups == ["test_func_1", "test_func_2"] - """ - ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( - [ - "*::test_func_1 PASSED *", - "*::test_func_2 FAILED *", - "*::test_func_3_check_cleanups PASSED *", - ] - ) + record = run_tests(Test, rootpath=tmp_path) + assert record["test_func_1"].passed + assert record["test_func_2"].failed + assert record["test_func_3_check_cleanups"].passed + assert cleanups == ["test_func_1", "test_func_2"] -def test_issue333_result_clearing(pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest +def test_issue333_result_clearing(tmp_path: Path) -> None: + class FailAfterCallPlugin: @pytest.hookimpl(wrapper=True) - def pytest_runtest_call(item): + def pytest_runtest_call(self, item): yield assert 0 - """ - ) - pytester.makepyfile( - """ - import unittest - class TestIt(unittest.TestCase): - def test_func(self): - 0/0 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(failed=1) + class TestIt(unittest.TestCase): + def test_func(self): + 0 / 0 # noqa: B018 + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(FailAfterCallPlugin(),)) + run_tests(TestIt, spec=spec).assert_outcomes(failed=1) +# ensemble canary: kept file based so that unittest collection through a real +# module import, and the path based nodeid in the report, stay covered here def test_unittest_raise_skip_issue748(pytester: Pytester) -> None: pytester.makepyfile( test_foo=""" @@ -1019,128 +998,102 @@ def test_one(self): ) -def test_unittest_skip_issue1169(pytester: Pytester) -> None: - pytester.makepyfile( - test_foo=""" - import unittest +def test_unittest_skip_issue1169(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + @unittest.skip("skipping due to reasons") + def test_skip(self): + self.fail() - class MyTestCase(unittest.TestCase): - @unittest.skip("skipping due to reasons") - def test_skip(self): - self.fail() - """ - ) - result = pytester.runpytest("-v", "-rs") - result.stdout.fnmatch_lines( - """ - *SKIP*[1]*skipping due to reasons* - *1 skipped* - """ - ) + record = run_tests(MyTestCase, rootpath=tmp_path) + record.assert_outcomes(skipped=1) + # a method level @unittest.skip is reported by unittest itself, i.e. in + # the call phase (a class level one skips in setup instead) + call = record["test_skip"].call + assert call is not None + assert "skipping due to reasons" in call.longreprtext -def test_class_method_containing_test_issue1558(pytester: Pytester) -> None: - pytester.makepyfile( - test_foo=""" - import unittest +def test_class_method_containing_test_issue1558(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + def test_should_run(self): + pass - class MyTestCase(unittest.TestCase): - def test_should_run(self): - pass - def test_should_not_run(self): - pass - test_should_not_run.__test__ = False - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + @set_attributes(__test__=False) + def test_should_not_run(self): + pass + items = collect_tests(MyTestCase, rootpath=tmp_path) + assert [item.name for item in items] == ["test_should_run"] + run_tests(MyTestCase, rootpath=tmp_path).assert_outcomes(passed=1) -@pytest.mark.parametrize("base", ["builtins.object", "unittest.TestCase"]) -def test_usefixtures_marker_on_unittest(base, pytester: Pytester) -> None: - """#3498""" - module = base.rsplit(".", 1)[0] - pytest.importorskip(module) - pytester.makepyfile( - conftest=""" - import pytest - @pytest.fixture(scope='function') - def fixture1(request, monkeypatch): - monkeypatch.setattr(request.instance, 'fixture1', True ) +@pytest.mark.parametrize( + "base", [object, unittest.TestCase], ids=["builtins.object", "unittest.TestCase"] +) +def test_usefixtures_marker_on_unittest(base, tmp_path: Path) -> None: + """#3498""" + seen: list[tuple[str, list[str]]] = [] + def node_and_marks(item): + seen.append((item.name, [mark.name for mark in item.iter_markers()])) - @pytest.fixture(scope='function') - def fixture2(request, monkeypatch): - monkeypatch.setattr(request.instance, 'fixture2', True ) + class ConftestPlugin: + @pytest.fixture(scope="function") + def fixture1(self, request, monkeypatch): + monkeypatch.setattr(request.instance, "fixture1", True) - def node_and_marks(item): - print(item.nodeid) - for mark in item.iter_markers(): - print(" ", mark) + @pytest.fixture(scope="function") + def fixture2(self, request, monkeypatch): + monkeypatch.setattr(request.instance, "fixture2", True) @pytest.fixture(autouse=True) - def my_marks(request): + def my_marks(self, request): node_and_marks(request.node) - def pytest_collection_modifyitems(items): + def pytest_collection_modifyitems(self, items): for item in items: - node_and_marks(item) - - """ - ) - - pytester.makepyfile( - f""" - import pytest - import {module} - - class Tests({base}): - fixture1 = False - fixture2 = False - - @pytest.mark.usefixtures("fixture1") - def test_one(self): - assert self.fixture1 - assert not self.fixture2 + node_and_marks(item) - @pytest.mark.usefixtures("fixture1", "fixture2") - def test_two(self): - assert self.fixture1 - assert self.fixture2 + class Tests(base): + fixture1 = False + fixture2 = False + @pytest.mark.usefixtures("fixture1") + def test_one(self): + assert self.fixture1 + assert not self.fixture2 - """ - ) + @pytest.mark.usefixtures("fixture1", "fixture2") + def test_two(self): + assert self.fixture1 + assert self.fixture2 - result = pytester.runpytest("-s") - result.assert_outcomes(passed=2) + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + run_tests(Tests, spec=spec).assert_outcomes(passed=2) + # the usefixtures marks are visible at collection *and* at fixture time + # (this is what the conftest printed for eyeballing) + assert seen.count(("test_one", ["usefixtures"])) == 2 + assert seen.count(("test_two", ["usefixtures"])) == 2 -def test_skip_setup_class(pytester: Pytester) -> None: +def test_skip_setup_class(tmp_path: Path) -> None: """Skipping tests in a class by raising unittest.SkipTest in `setUpClass` (#13985).""" - pytester.makepyfile( - """ - import unittest - class Test(unittest.TestCase): + class Test(unittest.TestCase): + @classmethod + def setUpClass(cls): + raise unittest.SkipTest("Skipping setupclass") - @classmethod - def setUpClass(cls): - raise unittest.SkipTest('Skipping setupclass') + def test_foo(self): + assert False - def test_foo(self): - assert False + def test_bar(self): + assert False - def test_bar(self): - assert False - """ - ) - result = pytester.runpytest() - result.assert_outcomes(skipped=2) + run_tests(Test, rootpath=tmp_path).assert_outcomes(skipped=2) -def test_unittest_skip_function(pytester: Pytester) -> None: +def test_unittest_skip_function(tmp_path: Path) -> None: """ Ensure raising an explicit unittest.SkipTest skips standard pytest functions. @@ -1148,39 +1101,37 @@ def test_unittest_skip_function(pytester: Pytester) -> None: but stating this support here in this test because users currently expect this to work, so if we ever break it we at least know we are breaking this use case (#13985). """ - pytester.makepyfile( - """ - import unittest - def test_foo(): - raise unittest.SkipTest('Skipping test_foo') - """ - ) - result = pytester.runpytest() - result.assert_outcomes(skipped=1) + def test_foo(): + raise unittest.SkipTest("Skipping test_foo") + run_tests(test_foo, rootpath=tmp_path).assert_outcomes(skipped=1) -def test_testcase_handles_init_exceptions(pytester: Pytester) -> None: - """ - Regression test to make sure exceptions in the __init__ method are bubbled up correctly. - See https://github.com/pytest-dev/pytest/issues/3788 - """ - pytester.makepyfile( - """ - from unittest import TestCase - import pytest - class MyTestCase(TestCase): - def __init__(self, *args, **kwargs): - raise Exception("should raise this exception") - def test_hello(self): - pass + +def test_testcase_handles_init_exceptions(tmp_path: Path) -> None: """ - ) - result = pytester.runpytest() - assert "should raise this exception" in result.stdout.str() - result.stdout.no_fnmatch_line("*ERROR at teardown of MyTestCase.test_hello*") + Regression test to make sure exceptions in the __init__ method are bubbled up correctly. + See https://github.com/pytest-dev/pytest/issues/3788 + """ + + class MyTestCase(unittest.TestCase): + def __init__(self, *args, **kwargs): + raise Exception("should raise this exception") + + def test_hello(self): + pass + record = run_tests(MyTestCase, rootpath=tmp_path) + record.assert_outcomes(errors=1) + (error,) = record.collect_errors + assert "should raise this exception" in str(error.longrepr) + # nothing was collected, so nothing ran and in particular there is no + # teardown error (was: no_fnmatch_line("*ERROR at teardown of*")) + assert record.reports == [] + +# ensemble: driven by an example script; an in-memory copy would orphan +# testing/example_scripts/unittest/test_parametrized_fixture_error_message.py def test_error_message_with_parametrized_fixtures(pytester: Pytester) -> None: pytester.copy_example("unittest/test_parametrized_fixture_error_message.py") result = pytester.runpytest() @@ -1193,6 +1144,8 @@ def test_error_message_with_parametrized_fixtures(pytester: Pytester) -> None: ) +# ensemble: driven by example scripts; in-memory copies would orphan +# testing/example_scripts/unittest/test_setup_skip*.py @pytest.mark.parametrize( "test_name, expected_outcome", [ @@ -1210,43 +1163,37 @@ def test_setup_inheritance_skipping( result.stdout.fnmatch_lines([f"* {expected_outcome} in *"]) -def test_BdbQuit(pytester: Pytester) -> None: - pytester.makepyfile( - test_foo=""" - import unittest +def test_BdbQuit(tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + def test_bdbquit(self): + import bdb - class MyTestCase(unittest.TestCase): - def test_bdbquit(self): - import bdb - raise bdb.BdbQuit() + raise bdb.BdbQuit - def test_should_not_run(self): - pass - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(failed=1, passed=1) + def test_should_not_run(self): + pass + run_tests(MyTestCase, rootpath=tmp_path).assert_outcomes(failed=1, passed=1) -def test_exit_outcome(pytester: Pytester) -> None: - pytester.makepyfile( - test_foo=""" - import pytest - import unittest - class MyTestCase(unittest.TestCase): - def test_exit_outcome(self): - pytest.exit("pytest_exit called") +def test_exit_outcome(tmp_path: Path) -> None: + ran: list[str] = [] - def test_should_not_run(self): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*Exit: pytest_exit called*", "*= no tests ran in *"]) + class MyTestCase(unittest.TestCase): + def test_exit_outcome(self): + pytest.exit("pytest_exit called") + + def test_should_not_run(self): + ran.append("test_should_not_run") + + # an ensemble has no session wrapper turning Exit into a summary line, so + # the exit surfaces as the exception it is + with pytest.raises(Exit, match="pytest_exit called"): + run_tests(MyTestCase, rootpath=tmp_path) + assert ran == [] -def test_trace(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: +def test_trace(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: calls = [] def check_call(*args, **kwargs): @@ -1261,125 +1208,105 @@ def runcall(*args, **kwargs): monkeypatch.setattr("_pytest.debugging.pytestPDB._init_pdb", check_call) - p1 = pytester.makepyfile( - """ - import unittest + class MyTestCase(unittest.TestCase): + def test(self): + self.assertEqual("foo", "foo") - class MyTestCase(unittest.TestCase): - def test(self): - self.assertEqual('foo', 'foo') - """ - ) - result = pytester.runpytest("--trace", str(p1)) + spec = ConfigSpec(rootpath=tmp_path, args=("--trace",)).with_plugins("debugging") + run_tests(MyTestCase, spec=spec).assert_outcomes(passed=1) assert len(calls) == 2 - assert result.ret == 0 -def test_pdb_teardown_called(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: +def test_pdb_teardown_called(tmp_path: Path) -> None: """Ensure tearDown() is always called when --pdb is given in the command-line. We delay the normal tearDown() calls when --pdb is given, so this ensures we are calling tearDown() eventually to avoid memory leaks when using --pdb. """ teardowns: list[str] = [] - monkeypatch.setattr( - pytest, "test_pdb_teardown_called_teardowns", teardowns, raising=False - ) - pytester.makepyfile( - """ - import unittest - import pytest + class MyTestCase(unittest.TestCase): + def tearDown(self): + teardowns.append(self.id()) - class MyTestCase(unittest.TestCase): + def test_1(self): + pass - def tearDown(self): - pytest.test_pdb_teardown_called_teardowns.append(self.id()) + def test_2(self): + pass - def test_1(self): - pass - def test_2(self): - pass - """ - ) - result = pytester.runpytest_inprocess("--pdb") - result.stdout.fnmatch_lines("* 2 passed in *") - assert teardowns == [ - "test_pdb_teardown_called.MyTestCase.test_1", - "test_pdb_teardown_called.MyTestCase.test_2", + spec = ConfigSpec(rootpath=tmp_path, args=("--pdb",)).with_plugins("debugging") + run_tests(MyTestCase, spec=spec).assert_outcomes(passed=2) + # TestCase.id() is built from __qualname__, which for a class defined in + # a test carries a "" segment + assert [teardown.split(".")[-1] for teardown in teardowns] == [ + "MyTestCase.test_1", + "MyTestCase.test_2", ] -@pytest.mark.parametrize("mark", ["@unittest.skip", "@pytest.mark.skip"]) -def test_pdb_teardown_skipped_for_functions( - pytester: Pytester, monkeypatch: MonkeyPatch, mark: str -) -> None: +@pytest.mark.parametrize( + "mark", + [ + pytest.param(unittest.skip("skipped for reasons"), id="unittest.skip"), + pytest.param(pytest.mark.skip("skipped for reasons"), id="pytest.mark.skip"), + ], +) +def test_pdb_teardown_skipped_for_functions(tmp_path: Path, mark) -> None: """ With --pdb, setUp and tearDown should not be called for tests skipped via a decorator (#7215). """ tracked: list[str] = [] - monkeypatch.setattr(pytest, "track_pdb_teardown_skipped", tracked, raising=False) - - pytester.makepyfile( - f""" - import unittest - import pytest - - class MyTestCase(unittest.TestCase): - def setUp(self): - pytest.track_pdb_teardown_skipped.append("setUp:" + self.id()) + class MyTestCase(unittest.TestCase): + def setUp(self): + tracked.append("setUp:" + self.id()) - def tearDown(self): - pytest.track_pdb_teardown_skipped.append("tearDown:" + self.id()) + def tearDown(self): + tracked.append("tearDown:" + self.id()) - {mark}("skipped for reasons") - def test_1(self): - pass + @mark + def test_1(self): + pass - """ - ) - result = pytester.runpytest_inprocess("--pdb") - result.stdout.fnmatch_lines("* 1 skipped in *") + spec = ConfigSpec(rootpath=tmp_path, args=("--pdb",)).with_plugins("debugging") + run_tests(MyTestCase, spec=spec).assert_outcomes(skipped=1) assert tracked == [] -@pytest.mark.parametrize("mark", ["@unittest.skip", "@pytest.mark.skip"]) -def test_pdb_teardown_skipped_for_classes( - pytester: Pytester, monkeypatch: MonkeyPatch, mark: str -) -> None: +@pytest.mark.parametrize( + "mark", + [ + pytest.param(unittest.skip("skipped for reasons"), id="unittest.skip"), + pytest.param(pytest.mark.skip("skipped for reasons"), id="pytest.mark.skip"), + ], +) +def test_pdb_teardown_skipped_for_classes(tmp_path: Path, mark) -> None: """ With --pdb, setUp and tearDown should not be called for tests skipped via a decorator on the class (#10060). """ tracked: list[str] = [] - monkeypatch.setattr(pytest, "track_pdb_teardown_skipped", tracked, raising=False) - - pytester.makepyfile( - f""" - import unittest - import pytest - {mark}("skipped for reasons") - class MyTestCase(unittest.TestCase): - - def setUp(self): - pytest.track_pdb_teardown_skipped.append("setUp:" + self.id()) + @mark + class MyTestCase(unittest.TestCase): + def setUp(self): + tracked.append("setUp:" + self.id()) - def tearDown(self): - pytest.track_pdb_teardown_skipped.append("tearDown:" + self.id()) + def tearDown(self): + tracked.append("tearDown:" + self.id()) - def test_1(self): - pass + def test_1(self): + pass - """ - ) - result = pytester.runpytest_inprocess("--pdb") - result.stdout.fnmatch_lines("* 1 skipped in *") + spec = ConfigSpec(rootpath=tmp_path, args=("--pdb",)).with_plugins("debugging") + run_tests(MyTestCase, spec=spec).assert_outcomes(skipped=1) assert tracked == [] +# ensemble: driven by an example script; an in-memory copy would orphan +# testing/example_scripts/unittest/test_unittest_asyncio.py def test_async_support(pytester: Pytester) -> None: pytest.importorskip("unittest.async_case") @@ -1388,6 +1315,8 @@ def test_async_support(pytester: Pytester) -> None: reprec.assertoutcome(failed=1, passed=2) +# ensemble: driven by an example script; an in-memory copy would orphan +# testing/example_scripts/unittest/test_unittest_asynctest.py @pytest.mark.skipif( sys.version_info >= (3, 11), reason="asynctest is not compatible with Python 3.11+" ) @@ -1399,6 +1328,7 @@ def test_asynctest_support(pytester: Pytester) -> None: reprec.assertoutcome(failed=1, passed=2) +# ensemble: needs a subprocess (the unawaited coroutine warning depends on gc) def test_plain_unittest_does_not_support_async(pytester: Pytester) -> None: """Async functions in plain unittest.TestCase subclasses are not supported without plugins. @@ -1422,155 +1352,144 @@ def test_plain_unittest_does_not_support_async(pytester: Pytester) -> None: result.stdout.fnmatch_lines(expected_lines) -def test_do_class_cleanups_on_success(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - values = [] - @classmethod - def setUpClass(cls): - def cleanup(): - cls.values.append(1) - cls.addClassCleanup(cleanup) - def test_one(self): - pass - def test_two(self): - pass - def test_cleanup_called_exactly_once(): - assert MyTestCase.values == [1] - """ - ) - reprec = pytester.inline_run(testpath) - passed, _skipped, failed = reprec.countoutcomes() - assert failed == 0 - assert passed == 3 +def test_do_class_cleanups_on_success(tmp_path: Path) -> None: + values: list[int] = [] + class MyTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + def cleanup(): + values.append(1) -def test_do_class_cleanups_on_setupclass_failure(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - values = [] - @classmethod - def setUpClass(cls): - def cleanup(): - cls.values.append(1) - cls.addClassCleanup(cleanup) - assert False - def test_one(self): - pass - def test_cleanup_called_exactly_once(): - assert MyTestCase.values == [1] - """ - ) - reprec = pytester.inline_run(testpath) - passed, _skipped, failed = reprec.countoutcomes() - assert failed == 1 - assert passed == 1 + cls.addClassCleanup(cleanup) + def test_one(self): + pass -def test_do_class_cleanups_on_teardownclass_failure(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - values = [] - @classmethod - def setUpClass(cls): - def cleanup(): - cls.values.append(1) - cls.addClassCleanup(cleanup) - @classmethod - def tearDownClass(cls): - assert False - def test_one(self): - pass - def test_two(self): - pass - def test_cleanup_called_exactly_once(): - assert MyTestCase.values == [1] - """ - ) - reprec = pytester.inline_run(testpath) - passed, _skipped, _failed = reprec.countoutcomes() - assert passed == 3 + def test_two(self): + pass + record = run_tests(MyTestCase, rootpath=tmp_path) + record.assert_outcomes(passed=2) + # was a trailing test function asserting this from the outside + assert values == [1] -def test_do_cleanups_on_success(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - values = [] - def setUp(self): - def cleanup(): - self.values.append(1) - self.addCleanup(cleanup) - def test_one(self): - pass - def test_two(self): - pass - def test_cleanup_called_the_right_number_of_times(): - assert MyTestCase.values == [1, 1] - """ - ) - reprec = pytester.inline_run(testpath) - passed, _skipped, failed = reprec.countoutcomes() - assert failed == 0 - assert passed == 3 +def test_do_class_cleanups_on_setupclass_failure(tmp_path: Path) -> None: + values: list[int] = [] -def test_do_cleanups_on_setup_failure(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - values = [] - def setUp(self): - def cleanup(): - self.values.append(1) - self.addCleanup(cleanup) - assert False - def test_one(self): - pass - def test_two(self): - pass - def test_cleanup_called_the_right_number_of_times(): - assert MyTestCase.values == [1, 1] - """ - ) - reprec = pytester.inline_run(testpath) - passed, _skipped, failed = reprec.countoutcomes() - assert failed == 2 - assert passed == 1 + class MyTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + def cleanup(): + values.append(1) + cls.addClassCleanup(cleanup) + assert False -def test_do_cleanups_on_teardown_failure(pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - values = [] - def setUp(self): - def cleanup(): - self.values.append(1) - self.addCleanup(cleanup) - def tearDown(self): - assert False - def test_one(self): - pass - def test_two(self): - pass - def test_cleanup_called_the_right_number_of_times(): - assert MyTestCase.values == [1, 1] - """ - ) - reprec = pytester.inline_run(testpath) - passed, _skipped, failed = reprec.countoutcomes() - assert failed == 2 - assert passed == 1 + def test_one(self): + pass + + record = run_tests(MyTestCase, rootpath=tmp_path) + # setUpClass runs as a class scoped fixture: a setup phase error + record.assert_outcomes(errors=1) + assert values == [1] + + +def test_do_class_cleanups_on_teardownclass_failure(tmp_path: Path) -> None: + values: list[int] = [] + + class MyTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + def cleanup(): + values.append(1) + + cls.addClassCleanup(cleanup) + + @classmethod + def tearDownClass(cls): + assert False + + def test_one(self): + pass + + def test_two(self): + pass + + record = run_tests(MyTestCase, rootpath=tmp_path) + # countoutcomes() ignored the teardown error the original also produced + record.assert_outcomes(passed=2, errors=1) + assert values == [1] + + +def test_do_cleanups_on_success(tmp_path: Path) -> None: + values: list[int] = [] + + class MyTestCase(unittest.TestCase): + def setUp(self): + def cleanup(): + values.append(1) + + self.addCleanup(cleanup) + + def test_one(self): + pass + + def test_two(self): + pass + + record = run_tests(MyTestCase, rootpath=tmp_path) + record.assert_outcomes(passed=2) + assert values == [1, 1] + + +def test_do_cleanups_on_setup_failure(tmp_path: Path) -> None: + values: list[int] = [] + + class MyTestCase(unittest.TestCase): + def setUp(self): + def cleanup(): + values.append(1) + + self.addCleanup(cleanup) + assert False + + def test_one(self): + pass + + def test_two(self): + pass + + record = run_tests(MyTestCase, rootpath=tmp_path) + # a unittest setUp failure is reported in the call phase, so these stay + # failures rather than becoming errors + record.assert_outcomes(failed=2) + assert values == [1, 1] + + +def test_do_cleanups_on_teardown_failure(tmp_path: Path) -> None: + values: list[int] = [] + + class MyTestCase(unittest.TestCase): + def setUp(self): + def cleanup(): + values.append(1) + + self.addCleanup(cleanup) + + def tearDown(self): + assert False + + def test_one(self): + pass + + def test_two(self): + pass + + record = run_tests(MyTestCase, rootpath=tmp_path) + record.assert_outcomes(failed=2) + assert values == [1, 1] class TestClassCleanupErrors: @@ -1581,113 +1500,98 @@ class TestClassCleanupErrors: See #11728. """ - def test_class_cleanups_failure_in_setup(self, pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - @classmethod - def setUpClass(cls): - def cleanup(n): - raise Exception(f"fail {n}") - cls.addClassCleanup(cleanup, 2) - cls.addClassCleanup(cleanup, 1) - raise Exception("fail 0") - def test(self): - pass - """ - ) - result = pytester.runpytest("-s", testpath) - result.assert_outcomes(passed=0, errors=1) - result.stdout.fnmatch_lines( - [ - "*Unittest class cleanup errors *2 sub-exceptions*", - "*Exception: fail 1", - "*Exception: fail 2", - ] - ) - result.stdout.fnmatch_lines( - [ - "* ERROR at setup of MyTestCase.test *", - "E * Exception: fail 0", - ] - ) + def test_class_cleanups_failure_in_setup(self, tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + def cleanup(n): + raise Exception(f"fail {n}") - def test_class_cleanups_failure_in_teardown(self, pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - @classmethod - def setUpClass(cls): - def cleanup(n): - raise Exception(f"fail {n}") - cls.addClassCleanup(cleanup, 2) - cls.addClassCleanup(cleanup, 1) - def test(self): - pass - """ - ) - result = pytester.runpytest("-s", testpath) - result.assert_outcomes(passed=1, errors=1) - result.stdout.fnmatch_lines( - [ - "*Unittest class cleanup errors *2 sub-exceptions*", - "*Exception: fail 1", - "*Exception: fail 2", - ] - ) + cls.addClassCleanup(cleanup, 2) + cls.addClassCleanup(cleanup, 1) + raise Exception("fail 0") - def test_class_cleanup_1_failure_in_teardown(self, pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class MyTestCase(unittest.TestCase): - @classmethod - def setUpClass(cls): - def cleanup(n): - raise Exception(f"fail {n}") - cls.addClassCleanup(cleanup, 1) - def test(self): - pass - """ - ) - result = pytester.runpytest("-s", testpath) - result.assert_outcomes(passed=1, errors=1) - result.stdout.fnmatch_lines( - [ - "*ERROR at teardown of MyTestCase.test*", - "*Exception: fail 1", - ] - ) + def test(self): + pass + record = run_tests(MyTestCase, rootpath=tmp_path) + record.assert_outcomes(passed=0, errors=1) + setup = record["test"].setup + assert setup is not None and setup.failed + text = setup.longreprtext + assert "Unittest class cleanup errors" in text + assert "2 sub-exceptions" in text + assert "Exception: fail 1" in text + assert "Exception: fail 2" in text + assert "Exception: fail 0" in text + + def test_class_cleanups_failure_in_teardown(self, tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + def cleanup(n): + raise Exception(f"fail {n}") -def test_traceback_pruning(pytester: Pytester) -> None: - """Regression test for #9610 - doesn't crash during traceback pruning.""" - pytester.makepyfile( - """ - import unittest + cls.addClassCleanup(cleanup, 2) + cls.addClassCleanup(cleanup, 1) - class MyTestCase(unittest.TestCase): - def __init__(self, test_method): - unittest.TestCase.__init__(self, test_method) + def test(self): + pass - class TestIt(MyTestCase): + record = run_tests(MyTestCase, rootpath=tmp_path) + record.assert_outcomes(passed=1, errors=1) + teardown = record["test"].teardown + assert teardown is not None and teardown.failed + text = teardown.longreprtext + assert "Unittest class cleanup errors" in text + assert "2 sub-exceptions" in text + assert "Exception: fail 1" in text + assert "Exception: fail 2" in text + + def test_class_cleanup_1_failure_in_teardown(self, tmp_path: Path) -> None: + class MyTestCase(unittest.TestCase): @classmethod - def tearDownClass(cls) -> None: - assert False + def setUpClass(cls): + def cleanup(n): + raise Exception(f"fail {n}") - def test_it(self): + cls.addClassCleanup(cleanup, 1) + + def test(self): pass - """ - ) - reprec = pytester.inline_run() - passed, _skipped, failed = reprec.countoutcomes() - assert passed == 1 - assert failed == 1 - assert reprec.ret == 1 + + record = run_tests(MyTestCase, rootpath=tmp_path) + record.assert_outcomes(passed=1, errors=1) + # was: "*ERROR at teardown of MyTestCase.test*" + teardown = record["test"].teardown + assert teardown is not None and teardown.failed + assert "Exception: fail 1" in teardown.longreprtext + + +def test_traceback_pruning(tmp_path: Path) -> None: + """Regression test for #9610 - doesn't crash during traceback pruning.""" + + class MyTestCase(unittest.TestCase): + def __init__(self, test_method): + unittest.TestCase.__init__(self, test_method) + + class TestIt(MyTestCase): + @classmethod + def tearDownClass(cls) -> None: + assert False + + def test_it(self): + pass + + record = run_tests(TestIt, rootpath=tmp_path) + # tearDownClass runs as a class scoped fixture: a teardown phase error + record.assert_outcomes(passed=1, errors=1) + teardown = record["test_it"].teardown + assert teardown is not None and teardown.failed +# ensemble canary: a module level ``raise unittest.SkipTest`` needs a real +# module import, which in-memory sources by definition do not do def test_raising_unittest_skiptest_during_collection( pytester: Pytester, ) -> None: @@ -1715,29 +1619,32 @@ def test_it2(self): pass assert reprec.ret == ExitCode.NO_TESTS_COLLECTED -def test_abstract_testcase_is_not_collected(pytester: Pytester) -> None: +def test_abstract_testcase_is_not_collected(tmp_path: Path) -> None: """Regression test for #12275.""" - pytester.makepyfile( - """ - import abc - import unittest - class TestBase(unittest.TestCase, abc.ABC): - @abc.abstractmethod - def abstract1(self): pass + class TestBase(unittest.TestCase, abc.ABC): + @abc.abstractmethod + def abstract1(self): + pass - @abc.abstractmethod - def abstract2(self): pass + @abc.abstractmethod + def abstract2(self): + pass - def test_it(self): pass + def test_it(self): + pass - class TestPartial(TestBase): - def abstract1(self): pass + class TestPartial(TestBase): + def abstract1(self): + pass - class TestConcrete(TestPartial): - def abstract2(self): pass - """ - ) - result = pytester.runpytest() - assert result.ret == ExitCode.OK - result.assert_outcomes(passed=1) + class TestConcrete(TestPartial): + def abstract2(self): + pass + + items = collect_tests(TestBase, TestPartial, TestConcrete, rootpath=tmp_path) + assert [item.nodeid.split("::", 1)[1] for item in items] == [ + "TestConcrete::test_it" + ] + record = run_tests(TestBase, TestPartial, TestConcrete, rootpath=tmp_path) + record.assert_outcomes(passed=1) From 9c0bd4e288deff45d8c6dcf2229b408ae81a5f37 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 19:39:04 +0200 Subject: [PATCH 04/30] testing: port test_mark.py to _pytest.ensemble 44 of 53 pytester-based tests ported, ~100 parametrized cases now running in memory. Marker names are the awkward part: pytest's own suite runs with strict = true, and a decorator in the enclosing test body is resolved against the *host* config long before the ensemble exists. Rather than rename the markers to ones the host happens to register - which would weaken the tests - an `ensemble_mark()` helper builds the MarkDecorator directly, keeping every original name and kwargs shape. Marks resolved *inside* an ensemble are unaffected: MARK_GEN._config is swapped to the nested config while it is configured. Two tests got stronger rather than merely equivalent: one asserting only `ret == 0` now asserts the outcome counts, and the -k selection test gains an unmarked sibling so it proves selection rather than "everything ran". 9 stay on pytester: --markers rendering served from pytest_cmdline_main (2), path arguments and arg-level collection (2), real files and conftest scoping (1), decoration-time marker resolution (1), string conditions resolved in host globals (1), absence of Directory nodes (1), and one asserting a host-anchored line number. --- testing/test_mark.py | 1331 ++++++++++++++++++++---------------------- 1 file changed, 646 insertions(+), 685 deletions(-) diff --git a/testing/test_mark.py b/testing/test_mark.py index 9b0cc44d200..1e59ea2c1aa 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -3,7 +3,10 @@ from collections.abc import Iterator import os +from pathlib import Path +import re import sys +import types from typing import cast from unittest import mock @@ -11,17 +14,45 @@ from _pytest.config import ExitCode from _pytest.config import RegisteredMarker from _pytest.config import UsageError +from _pytest.ensemble import build_module +from _pytest.ensemble import collect_tests +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import configured +from _pytest.ensemble import run_tests +from _pytest.ensemble import RunRecord from _pytest.mark import _validate_marker_names from _pytest.mark import MarkGenerator from _pytest.mark.expression import Expression from _pytest.mark.structures import _EmptyParameterSetMark from _pytest.mark.structures import EMPTY_PARAMETERSET_OPTION +from _pytest.mark.structures import Mark +from _pytest.mark.structures import MarkDecorator from _pytest.nodes import Collector from _pytest.nodes import Node from _pytest.pytester import Pytester import pytest +def ensemble_mark(name: str, *args: object, **kwargs: object) -> MarkDecorator: + """Build a mark decorator without consulting the *host* configuration. + + ``pytest.mark.`` validates the name against whatever config is + active at decoration time. Sources written inline in this file are + decorated while pytest's own suite (which runs with ``strict = true``) + is the active config, long before the ensemble they are meant to run in + exists, so marker names that are only registered inside an ensemble -- + or deliberately not registered anywhere -- have to bypass ``MARK_GEN``. + """ + return MarkDecorator(Mark(name, args, kwargs, _ispytest=True), _ispytest=True) + + +def passed_names(record: RunRecord) -> list[str]: + """The bare names of the tests that passed, in collection order.""" + return [ + nodeid.split("::")[-1] for nodeid, item in record.by_test.items() if item.passed + ] + + class TestMark: @pytest.mark.parametrize("attr", ["mark", "param"]) def test_pytest_exists_in_namespace_all(self, attr: str) -> None: @@ -53,6 +84,8 @@ def test_pytest_mark_name_starts_with_underscore(self) -> None: _ = mark._some_name +# ensemble: the subject is argument-level deduplication of the same file passed +# twice; ensemble collection is preset and has no path arguments to dedupe. def test_marked_class_run_twice(pytester: Pytester) -> None: """Test fails file is run twice that contains marked class. See issue#683. @@ -71,29 +104,28 @@ def test_1(self, abc): rec.assertoutcome(passed=6) -def test_ini_markers(pytester: Pytester) -> None: - pytester.makeini( - """ - [pytest] - markers = - a1: this is a webtest marker - a2: this is a smoke marker - """ - ) - pytester.makepyfile( - """ - def test_markers(pytestconfig): - markers = pytestconfig.getini("markers") - print(markers) - assert len(markers) >= 2 - assert markers[0].startswith("a1:") - assert markers[1].startswith("a2:") - """ +def test_ini_markers(tmp_path: Path) -> None: + def test_markers(pytestconfig): + markers = pytestconfig.getini("markers") + print(markers) + assert len(markers) >= 2 + assert markers[0].startswith("a1:") + assert markers[1].startswith("a2:") + + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={ + "markers": [ + "a1: this is a webtest marker", + "a2: this is a smoke marker", + ] + }, ) - rec = pytester.inline_run() - rec.assertoutcome(passed=1) + run_tests(test_markers, spec=spec).assert_outcomes(passed=1) +# ensemble: --markers is served from pytest_cmdline_main, which an ensemble +# (which starts from an already-parsed config) never reaches. def test_markers_option(pytester: Pytester) -> None: pytester.makeini( """ @@ -110,27 +142,22 @@ def test_markers_option(pytester: Pytester) -> None: ) -def test_ini_markers_whitespace(pytester: Pytester) -> None: - pytester.makeini( - """ - [pytest] - markers = - a1 : this is a whitespace marker - """ - ) - pytester.makepyfile( - """ - import pytest +def test_ini_markers_whitespace(tmp_path: Path) -> None: + @ensemble_mark("a1") + def test_markers(): + assert True - @pytest.mark.a1 - def test_markers(): - assert True - """ + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={"markers": ["a1 : this is a whitespace marker"]}, + args=("--strict-markers", "-m", "a1"), ) - rec = pytester.inline_run("--strict-markers", "-m", "a1") - rec.assertoutcome(passed=1) + # --strict-markers also validates the '-m' expression against the + # registered names, so this fails loudly if the whitespace is not stripped. + run_tests(test_markers, spec=spec).assert_outcomes(passed=1) +# ensemble: needs a setup.cfg on disk and a conftest scoped to a subdirectory. def test_marker_without_description(pytester: Pytester) -> None: pytester.makefile( ".cfg", @@ -151,6 +178,8 @@ def test_marker_without_description(pytester: Pytester) -> None: rec.assert_outcomes() +# ensemble: --markers again, plus a conftest that loads a plugin by module name +# from the current directory. def test_markers_option_with_plugin_in_current_dir(pytester: Pytester) -> None: pytester.makeconftest('pytest_plugins = "flip_flop"') pytester.makepyfile( @@ -177,20 +206,18 @@ def test_example(x): result.stdout.fnmatch_lines(["*flip*flop*"]) -def test_mark_on_pseudo_function(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest +def test_mark_on_pseudo_function(tmp_path: Path) -> None: + @ensemble_mark("r", lambda x: 0 / 0) + def test_hello(): + pass - @pytest.mark.r(lambda x: 0/0) - def test_hello(): - pass - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + run_tests(test_hello, rootpath=tmp_path).assert_outcomes(passed=1) +# ensemble: the whole point is a ``@pytest.mark.unregisteredmark`` decorator +# resolved against the config under test; decorators on sources written here +# resolve against the *host* config at decoration time instead. The enforcement +# itself is covered through '-m' expression validation below. @pytest.mark.parametrize( "option", [ @@ -286,52 +313,49 @@ def test_no_validation_without_strict(self) -> None: @pytest.fixture -def markexpr_pytester(pytester: Pytester) -> Pytester: - pytester.makeini( - """ - [pytest] - markers = - registered: a registered marker - """ - ) - pytester.makepyfile( - """ - import pytest +def markexpr_module() -> types.ModuleType: + @ensemble_mark("registered") + def test_registered(): + pass - @pytest.mark.registered - def test_registered(): - pass + def test_plain(): + pass - def test_plain(): - pass - """ + return build_module("test_markexpr", test_registered, test_plain) + + +@pytest.fixture +def markexpr_spec(tmp_path: Path) -> ConfigSpec: + return ConfigSpec( + rootpath=tmp_path, + inicfg={"markers": ["registered: a registered marker"]}, ) - return pytester @pytest.mark.parametrize("option", ["--strict-markers", "--strict"]) def test_strict_prohibits_unregistered_markers_in_markexpr( - markexpr_pytester: Pytester, option: str + markexpr_module: types.ModuleType, markexpr_spec: ConfigSpec, option: str ) -> None: - result = markexpr_pytester.runpytest(option, "-m", "registered or unregisteredmark") - assert result.ret == ExitCode.USAGE_ERROR - result.stderr.fnmatch_lines( - ["*Unknown marker(s) in '-m' expression: unregisteredmark*"] - ) + spec = markexpr_spec.replace(args=(option, "-m", "registered or unregisteredmark")) + with pytest.raises( + UsageError, + match=re.escape("Unknown marker(s) in '-m' expression: unregisteredmark"), + ): + run_tests(markexpr_module, spec=spec) def test_strict_allows_registered_markers_in_markexpr( - markexpr_pytester: Pytester, + markexpr_module: types.ModuleType, markexpr_spec: ConfigSpec ) -> None: - result = markexpr_pytester.runpytest("--strict-markers", "-m", "registered") - result.assert_outcomes(passed=1, deselected=1) + spec = markexpr_spec.replace(args=("--strict-markers", "-m", "registered")) + run_tests(markexpr_module, spec=spec).assert_outcomes(passed=1, deselected=1) def test_unregistered_markers_in_markexpr_allowed_without_strict( - markexpr_pytester: Pytester, + markexpr_module: types.ModuleType, markexpr_spec: ConfigSpec ) -> None: - result = markexpr_pytester.runpytest("-m", "unregisteredmark") - result.assert_outcomes(deselected=2) + spec = markexpr_spec.replace(args=("-m", "unregisteredmark")) + run_tests(markexpr_module, spec=spec).assert_outcomes(deselected=2) @pytest.mark.parametrize( @@ -346,23 +370,20 @@ def test_unregistered_markers_in_markexpr_allowed_without_strict( ], ) def test_mark_option( - expr: str, expected_passed: list[str | None], pytester: Pytester + expr: str, expected_passed: list[str | None], tmp_path: Path ) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.xyz - def test_one(): - pass - @pytest.mark.xyz2 - def test_two(): - pass - """ + @ensemble_mark("xyz") + def test_one(): + pass + + @ensemble_mark("xyz2") + def test_two(): + pass + + record = run_tests( + test_one, test_two, rootpath=tmp_path, spec=ConfigSpec(args=("-m", expr)) ) - rec = pytester.inline_run("-m", expr) - passed, _skipped, _fail = rec.listoutcomes() - passed_str = [x.nodeid.split("::")[-1] for x in passed] - assert passed_str == expected_passed + assert passed_names(record) == expected_passed @pytest.mark.parametrize( @@ -382,35 +403,36 @@ def test_two(): ids=str, ) def test_mark_option_with_kwargs( - expr: str, expected_passed: list[str | None], pytester: Pytester + expr: str, expected_passed: list[str | None], tmp_path: Path ) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.car - @pytest.mark.car(ac=True) - @pytest.mark.car(temp=4) - @pytest.mark.car(color="red") - def test_one(): - pass - @pytest.mark.car - @pytest.mark.car(ac=False) - @pytest.mark.car(temp=5) - @pytest.mark.car(color="blue") - def test_two(): - pass - @pytest.mark.car - @pytest.mark.car(ac=None) - @pytest.mark.car(temp=-5) - def test_three(): - pass + @ensemble_mark("car") + @ensemble_mark("car", ac=True) + @ensemble_mark("car", temp=4) + @ensemble_mark("car", color="red") + def test_one(): + pass - """ + @ensemble_mark("car") + @ensemble_mark("car", ac=False) + @ensemble_mark("car", temp=5) + @ensemble_mark("car", color="blue") + def test_two(): + pass + + @ensemble_mark("car") + @ensemble_mark("car", ac=None) + @ensemble_mark("car", temp=-5) + def test_three(): + pass + + record = run_tests( + test_one, + test_two, + test_three, + rootpath=tmp_path, + spec=ConfigSpec(args=("-m", expr)), ) - rec = pytester.inline_run("-m", expr) - passed, _skipped, _fail = rec.listoutcomes() - passed_str = [x.nodeid.split("::")[-1] for x in passed] - assert passed_str == expected_passed + assert passed_names(record) == expected_passed @pytest.mark.parametrize( @@ -418,29 +440,29 @@ def test_three(): [("interface", ["test_interface"]), ("not interface", ["test_nointer"])], ) def test_mark_option_custom( - expr: str, expected_passed: list[str], pytester: Pytester + expr: str, expected_passed: list[str], tmp_path: Path ) -> None: - pytester.makeconftest( - """ - import pytest - def pytest_collection_modifyitems(items): + class AddInterfaceMarker: + def pytest_collection_modifyitems(self, items): for item in items: if "interface" in item.nodeid: - item.add_marker(pytest.mark.interface) - """ - ) - pytester.makepyfile( - """ - def test_interface(): - pass - def test_nointer(): - pass - """ + item.add_marker(ensemble_mark("interface")) + + def test_interface(): + pass + + def test_nointer(): + pass + + spec = ConfigSpec(args=("-m", expr), extra_plugins=(AddInterfaceMarker(),)) + record = run_tests( + test_interface, + test_nointer, + rootpath=tmp_path, + spec=spec, + name="test_mark_option_custom", ) - rec = pytester.inline_run("-m", expr) - passed, _skipped, _fail = rec.listoutcomes() - passed_str = [x.nodeid.split("::")[-1] for x in passed] - assert passed_str == expected_passed + assert passed_names(record) == expected_passed @pytest.mark.parametrize( @@ -456,33 +478,55 @@ def test_nointer(): ], ) def test_keyword_option_custom( - expr: str, expected_passed: list[str], pytester: Pytester + expr: str, expected_passed: list[str], tmp_path: Path ) -> None: - pytester.makepyfile( - """ - def test_interface(): - pass - def test_nointer(): - pass - def test_pass(): - pass - def test_1(): - pass - def test_2(): - pass - """ + def test_interface(): + pass + + def test_nointer(): + pass + + def test_pass(): + pass + + def test_1(): + pass + + def test_2(): + pass + + # the synthesized module name is matched by -k just like a real one, so it + # is chosen to contain none of the expressions under test. + record = run_tests( + test_interface, + test_nointer, + test_pass, + test_1, + test_2, + rootpath=tmp_path, + spec=ConfigSpec(args=("-k", expr)), + name="test_keyword_custom", ) - rec = pytester.inline_run("-k", expr) - passed, _skipped, _fail = rec.listoutcomes() - passed_str = [x.nodeid.split("::")[-1] for x in passed] - assert passed_str == expected_passed + assert passed_names(record) == expected_passed + + +def test_keyword_option_considers_mark(tmp_path: Path) -> None: + @pytest.mark.foo + def test_mark(): + pass + def test_unmarked(): + pass -def test_keyword_option_considers_mark(pytester: Pytester) -> None: - pytester.copy_example("marks/marks_considered_keywords") - rec = pytester.inline_run("-k", "foo") - passed = rec.listoutcomes()[0] - assert len(passed) == 1 + record = run_tests( + test_mark, + test_unmarked, + rootpath=tmp_path, + spec=ConfigSpec(args=("-k", "foo")), + name="test_marks_as_keywords", + ) + record.assert_outcomes(passed=1, deselected=1) + assert passed_names(record) == ["test_mark"] @pytest.mark.parametrize( @@ -494,35 +538,30 @@ def test_keyword_option_considers_mark(pytester: Pytester) -> None: ], ) def test_keyword_option_parametrize( - expr: str, expected_passed: list[str], pytester: Pytester + expr: str, expected_passed: list[str], tmp_path: Path ) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize("arg", [None, 1.3, "2-3"]) - def test_func(arg): - pass - """ + @pytest.mark.parametrize("arg", [None, 1.3, "2-3"]) + def test_func(arg): + pass + + record = run_tests( + test_func, + rootpath=tmp_path, + spec=ConfigSpec(args=("-k", expr)), + name="test_keyword_parametrize", ) - rec = pytester.inline_run("-k", expr) - passed, _skipped, _fail = rec.listoutcomes() - passed_str = [x.nodeid.split("::")[-1] for x in passed] - assert passed_str == expected_passed + assert passed_names(record) == expected_passed -def test_parametrize_with_module(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize("arg", [pytest,]) - def test_func(arg): - pass - """ - ) - rec = pytester.inline_run() - passed, _skipped, _fail = rec.listoutcomes() +def test_parametrize_with_module(tmp_path: Path) -> None: + @pytest.mark.parametrize("arg", [pytest]) + def test_func(arg): + pass + + record = run_tests(test_func, rootpath=tmp_path) + record.assert_outcomes(passed=1) expected_id = "test_func[" + pytest.__name__ + "]" - assert passed[0].nodeid.split("::")[-1] == expected_id + assert passed_names(record) == [expected_id] @pytest.mark.parametrize( @@ -559,19 +598,17 @@ def test_func(arg): ], ) def test_keyword_option_wrong_arguments( - expr: str, expected_error: str, pytester: Pytester, capsys + expr: str, expected_error: str, tmp_path: Path ) -> None: - pytester.makepyfile( - """ - def test_func(arg): - pass - """ - ) - pytester.inline_run("-k", expr) - err = capsys.readouterr().err - assert expected_error in err + def test_func(arg): + pass + + with pytest.raises(UsageError, match=re.escape(expected_error)): + run_tests(test_func, rootpath=tmp_path, spec=ConfigSpec(args=("-k", expr))) +# ensemble: the subject is selecting a parametrized test by a "file.py::name" +# command line argument; ensemble collection is preset, not argument driven. def test_parametrized_collected_from_command_line(pytester: Pytester) -> None: """Parametrized test not collected if test named specified in command line issue#649.""" @@ -588,20 +625,17 @@ def test_func(arg): rec.assertoutcome(passed=3) -def test_parametrized_collect_with_wrong_args(pytester: Pytester) -> None: +def test_parametrized_collect_with_wrong_args(tmp_path: Path) -> None: """Test collect parametrized func with wrong number of args.""" - py_file = pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize('foo, bar', [(1, 2, 3)]) - def test_func(foo, bar): - pass - """ - ) + @pytest.mark.parametrize("foo, bar", [(1, 2, 3)]) + def test_func(foo, bar): + pass - result = pytester.runpytest(py_file) - result.stdout.fnmatch_lines( + module = build_module("test_parametrized_collect_with_wrong_args", test_func) + record = run_tests(module, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( [ 'test_parametrized_collect_with_wrong_args.py::test_func: in "parametrize" the number of names (2):', " ['foo', 'bar']", @@ -611,111 +645,90 @@ def test_func(foo, bar): ) -def test_parametrized_with_kwargs(pytester: Pytester) -> None: +def test_parametrized_with_kwargs(tmp_path: Path) -> None: """Test collect parametrized func with wrong number of args.""" - py_file = pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=[1,2]) - def a(request): - return request.param + @pytest.fixture(params=[1, 2]) + def a(request): + return request.param - @pytest.mark.parametrize(argnames='b', argvalues=[1, 2]) - def test_func(a, b): - pass - """ - ) + @pytest.mark.parametrize(argnames="b", argvalues=[1, 2]) + def test_func(a, b): + pass - result = pytester.runpytest(py_file) - assert result.ret == 0 + run_tests(a, test_func, rootpath=tmp_path).assert_outcomes(passed=4) -def test_parametrize_iterator(pytester: Pytester) -> None: +def test_parametrize_iterator(tmp_path: Path) -> None: """`parametrize` should work with generators (#5354).""" - py_file = pytester.makepyfile( - """\ - import pytest - def gen(): - yield 1 - yield 2 - yield 3 + def gen(): + yield 1 + yield 2 + yield 3 + + @pytest.mark.parametrize("a", gen()) + def test(a): + assert a >= 1 - @pytest.mark.parametrize('a', gen()) - def test(a): - assert a >= 1 - """ - ) - result = pytester.runpytest(py_file) - assert result.ret == 0 # should not skip any tests - result.stdout.fnmatch_lines(["*3 passed*"]) + run_tests(test, rootpath=tmp_path).assert_outcomes(passed=3) class TestFunctional: - def test_merging_markers_deep(self, pytester: Pytester) -> None: + def test_merging_markers_deep(self, tmp_path: Path) -> None: # issue 199 - propagate markers into nested classes - p = pytester.makepyfile( - """ - import pytest - class TestA(object): - pytestmark = pytest.mark.a - def test_b(self): + class TestA: + pytestmark = ensemble_mark("a") + + def test_b(self): + assert True + + class TestC: + # this one didn't get marked + def test_d(self): assert True - class TestC(object): - # this one didn't get marked - def test_d(self): - assert True - """ - ) - items, _rec = pytester.inline_genitems(p) + + items = collect_tests(TestA, rootpath=tmp_path) + assert len(items) == 2 for item in items: print(item, item.keywords) assert [x for x in item.iter_markers() if x.name == "a"] def test_mark_decorator_subclass_does_not_propagate_to_base( - self, pytester: Pytester + self, tmp_path: Path ) -> None: - p = pytester.makepyfile( - """ - import pytest + @ensemble_mark("a") + class Base: + pass - @pytest.mark.a - class Base(object): pass + @ensemble_mark("b") + class Test1(Base): + def test_foo(self): + pass - @pytest.mark.b - class Test1(Base): - def test_foo(self): pass + class Test2(Base): + def test_bar(self): + pass - class Test2(Base): - def test_bar(self): pass - """ - ) - items, _rec = pytester.inline_genitems(p) + items = collect_tests(Test1, Test2, rootpath=tmp_path) self.assert_markers(items, test_foo=("a", "b"), test_bar=("a",)) - def test_mark_should_not_pass_to_siebling_class(self, pytester: Pytester) -> None: + def test_mark_should_not_pass_to_siebling_class(self, tmp_path: Path) -> None: """#568""" - p = pytester.makepyfile( - """ - import pytest - - class TestBase(object): - def test_foo(self): - pass - @pytest.mark.b - class TestSub(TestBase): + class TestBase: + def test_foo(self): pass + @ensemble_mark("b") + class TestSub(TestBase): + pass - class TestOtherSub(TestBase): - pass + class TestOtherSub(TestBase): + pass - """ - ) - items, _rec = pytester.inline_genitems(p) + items = collect_tests(TestBase, TestSub, TestOtherSub, rootpath=tmp_path) base_item, sub_item, sub_item_other = items print(items, [x.nodeid for x in items]) # new api segregates @@ -723,47 +736,39 @@ class TestOtherSub(TestBase): assert not list(sub_item_other.iter_markers(name="b")) assert list(sub_item.iter_markers(name="b")) - def test_mark_decorator_baseclasses_merged(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest + def test_mark_decorator_baseclasses_merged(self, tmp_path: Path) -> None: + @ensemble_mark("a") + class Base: + pass - @pytest.mark.a - class Base(object): pass + @ensemble_mark("b") + class Base2(Base): + pass - @pytest.mark.b - class Base2(Base): pass + @ensemble_mark("c") + class Test1(Base2): + def test_foo(self): + pass - @pytest.mark.c - class Test1(Base2): - def test_foo(self): pass + class Test2(Base2): + @ensemble_mark("d") + def test_bar(self): + pass - class Test2(Base2): - @pytest.mark.d - def test_bar(self): pass - """ - ) - items, _rec = pytester.inline_genitems(p) + items = collect_tests(Test1, Test2, rootpath=tmp_path) self.assert_markers(items, test_foo=("a", "b", "c"), test_bar=("a", "b", "d")) - def test_mark_closest(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest - - @pytest.mark.c(location="class") - class Test: - @pytest.mark.c(location="function") - def test_has_own(self): - pass + def test_mark_closest(self, tmp_path: Path) -> None: + @ensemble_mark("c", location="class") + class Test: + @ensemble_mark("c", location="function") + def test_has_own(self): + pass - def test_has_inherited(self): - pass + def test_has_inherited(self): + pass - """ - ) - items, _rec = pytester.inline_genitems(p) - has_own, has_inherited = items + has_own, has_inherited = collect_tests(Test, rootpath=tmp_path) has_own_marker = has_own.get_closest_marker("c") has_inherited_marker = has_inherited.get_closest_marker("c") assert has_own_marker is not None @@ -772,121 +777,99 @@ def test_has_inherited(self): assert has_inherited_marker.kwargs == {"location": "class"} assert has_own.get_closest_marker("missing") is None - def test_mark_closest_default_mark_decorator(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - def test_without_mark(): - pass - """ - ) - items, _rec = pytester.inline_genitems(p) - (item,) = items + def test_mark_closest_default_mark_decorator(self, tmp_path: Path) -> None: + def test_without_mark(): + pass + + (item,) = collect_tests(test_without_mark, rootpath=tmp_path) default = pytest.mark.foo(location="default") assert item.get_closest_marker("foo", default) is default.mark - def test_mark_with_wrong_marker(self, pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - import pytest - class pytestmark(object): - pass - def test_func(): - pass - """ - ) - values = reprec.getfailedcollections() - assert len(values) == 1 - assert "TypeError" in str(values[0].longrepr) + def test_mark_with_wrong_marker(self, tmp_path: Path) -> None: + class pytestmark: + pass - def test_mark_dynamically_in_funcarg(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest + def test_func(): + pass + + module = build_module("test_wrong_marker", test_func, pytestmark=pytestmark) + record = run_tests(module, rootpath=tmp_path) + (value,) = record.collect_errors + assert "TypeError" in str(value.longrepr) + + def test_mark_dynamically_in_funcarg(self, tmp_path: Path) -> None: + class ArgPlugin: @pytest.fixture - def arg(request): - request.applymarker(pytest.mark.hello) - def pytest_terminal_summary(terminalreporter): - values = terminalreporter.stats['passed'] - terminalreporter._tw.line("keyword: %s" % values[0].keywords) - """ - ) - pytester.makepyfile( - """ - def test_func(arg): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["keyword: *hello*"]) + def arg(self, request): + request.applymarker(ensemble_mark("hello")) - def test_no_marker_match_on_unmarked_names(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest - @pytest.mark.shouldmatch - def test_marked(): - assert 1 + def test_func(arg): + pass - def test_unmarked(): - assert 1 - """ - ) - reprec = pytester.inline_run("-m", "test_unmarked", p) - passed, skipped, failed = reprec.listoutcomes() - assert len(passed) + len(skipped) + len(failed) == 0 - dlist = reprec.getcalls("pytest_deselected") - deselected_tests = dlist[0].items - assert len(deselected_tests) == 2 - - def test_keywords_at_node_level(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture(scope="session", autouse=True) - def some(request): - request.keywords["hello"] = 42 - assert "world" not in request.keywords - - @pytest.fixture(scope="function", autouse=True) - def funcsetup(request): - assert "world" in request.keywords - assert "hello" in request.keywords - - @pytest.mark.world - def test_function(): - pass - """ + spec = ConfigSpec(extra_plugins=(ArgPlugin(),)) + record = run_tests(test_func, rootpath=tmp_path, spec=spec) + record.assert_outcomes(passed=1) + # the original scraped this off the terminal summary's report keywords + call = record["test_func"].call + assert call is not None + assert "hello" in call.keywords + + def test_no_marker_match_on_unmarked_names(self, tmp_path: Path) -> None: + @ensemble_mark("shouldmatch") + def test_marked(): + assert 1 + + def test_unmarked(): + assert 1 + + record = run_tests( + test_marked, + test_unmarked, + rootpath=tmp_path, + spec=ConfigSpec(args=("-m", "test_unmarked")), ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + record.assert_outcomes(deselected=2) - def test_keyword_added_for_session(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - def pytest_collection_modifyitems(session): + def test_keywords_at_node_level(self, tmp_path: Path) -> None: + @pytest.fixture(scope="session", autouse=True) + def some(request): + request.keywords["hello"] = 42 + assert "world" not in request.keywords + + @pytest.fixture(scope="function", autouse=True) + def funcsetup(request): + assert "world" in request.keywords + assert "hello" in request.keywords + + @ensemble_mark("world") + def test_function(): + pass + + record = run_tests(some, funcsetup, test_function, rootpath=tmp_path) + record.assert_outcomes(passed=1) + + def test_keyword_added_for_session(self, tmp_path: Path) -> None: + class SessionMarkerPlugin: + def pytest_collection_modifyitems(self, session): session.add_marker("mark1") - session.add_marker(pytest.mark.mark2) - session.add_marker(pytest.mark.mark3) + session.add_marker(ensemble_mark("mark2")) + session.add_marker(ensemble_mark("mark3")) with pytest.raises(ValueError): session.add_marker(10) - """ - ) - pytester.makepyfile( - """ - def test_some(request): - assert "mark1" in request.keywords - assert "mark2" in request.keywords - assert "mark3" in request.keywords - assert 10 not in request.keywords - marker = request.node.get_closest_marker("mark1") - assert marker.name == "mark1" - assert marker.args == () - assert marker.kwargs == {} - """ - ) - reprec = pytester.inline_run("-m", "mark1") - reprec.assertoutcome(passed=1) + + def test_some(request): + assert "mark1" in request.keywords + assert "mark2" in request.keywords + assert "mark3" in request.keywords + assert 10 not in request.keywords + marker = request.node.get_closest_marker("mark1") + assert marker.name == "mark1" + assert marker.args == () + assert marker.kwargs == {} + + spec = ConfigSpec(args=("-m", "mark1"), extra_plugins=(SessionMarkerPlugin(),)) + record = run_tests(test_some, rootpath=tmp_path, spec=spec) + record.assert_outcomes(passed=1) def assert_markers(self, items, **expected) -> None: """Assert that given items have expected marker names applied to them. @@ -901,30 +884,29 @@ def assert_markers(self, items, **expected) -> None: assert markers == set(expected_markers) @pytest.mark.filterwarnings("ignore") - def test_mark_from_parameters(self, pytester: Pytester) -> None: + def test_mark_from_parameters(self, tmp_path: Path) -> None: """#1540""" - pytester.makepyfile( - """ - import pytest - - pytestmark = pytest.mark.skipif(True, reason='skip all') - - # skipifs inside fixture params - params = [pytest.mark.skipif(False, reason='dont skip')('parameter')] + # skipifs inside fixture params + params = [pytest.mark.skipif(False, reason="dont skip")("parameter")] + @pytest.fixture(params=params) + def parameter(request): + return request.param - @pytest.fixture(params=params) - def parameter(request): - return request.param - + def test_1(parameter): + assert True - def test_1(parameter): - assert True - """ + module = build_module( + "test_mark_from_parameters", + parameter, + test_1, + pytestmark=pytest.mark.skipif(True, reason="skip all"), ) - reprec = pytester.inline_run() - reprec.assertoutcome(skipped=1) + run_tests(module, rootpath=tmp_path).assert_outcomes(skipped=1) + # ensemble: string skipif conditions are evaluated in the *host* module's + # globals (in-memory sources keep this file's __globals__), so the two + # modules cannot each supply their own ``skip`` name. def test_reevaluate_dynamic_expr(self, pytester: Pytester) -> None: """#7360""" py_file1 = pytester.makepyfile( @@ -957,23 +939,26 @@ def test_should_not_skip(): class TestKeywordSelection: - def test_select_simple(self, pytester: Pytester) -> None: - file_test = pytester.makepyfile( - """ - def test_one(): - assert 0 - class TestClass(object): - def test_method_one(self): - assert 42 == 43 - """ - ) + def test_select_simple(self, tmp_path: Path) -> None: + def test_one(): + assert 0 + + class TestClass: + def test_method_one(self): + # deliberately false; the source is meant to fail + assert 42 == 43 # type: ignore[comparison-overlap] def check(keyword, name): - reprec = pytester.inline_run("-s", "-k", keyword, file_test) - _passed, _skipped, failed = reprec.listoutcomes() - assert len(failed) == 1 - assert failed[0].nodeid.split("::")[-1] == name - assert len(reprec.getcalls("pytest_deselected")) == 1 + record = run_tests( + test_one, + TestClass, + rootpath=tmp_path, + spec=ConfigSpec(args=("-k", keyword)), + name="test_simple_selection", + ) + record.assert_outcomes(failed=1, deselected=1) + (nodeid,) = record.by_test + assert nodeid.split("::")[-1] == name for keyword in ["test_one", "est_on"]: check(keyword, "test_one") @@ -990,84 +975,91 @@ def check(keyword, name): "xxx and TestClass and test_2", ], ) - def test_select_extra_keywords(self, pytester: Pytester, keyword) -> None: - p = pytester.makepyfile( - test_select=""" - def test_1(): + def test_select_extra_keywords(self, tmp_path: Path, keyword) -> None: + def test_1(): + pass + + class TestClass: + def test_2(self): pass - class TestClass(object): - def test_2(self): - pass - """ - ) - pytester.makepyfile( - conftest=""" - import pytest + + class ExtraKeywordsPlugin: @pytest.hookimpl(wrapper=True) - def pytest_pycollect_makeitem(name): + def pytest_pycollect_makeitem(self, name): item = yield if name == "TestClass": item.extra_keyword_matches.add("xxx") return item - """ + + class DeselectRecorder: + def __init__(self) -> None: + self.calls: list[list[object]] = [] + + def pytest_deselected(self, items): + self.calls.append(list(items)) + + recorder = DeselectRecorder() + spec = ConfigSpec( + args=("-k", keyword), + extra_plugins=(ExtraKeywordsPlugin(), recorder), + ) + record = run_tests( + test_1, TestClass, rootpath=tmp_path, spec=spec, name="test_select" ) - reprec = pytester.inline_run(p.parent, "-s", "-k", keyword) print("keyword", repr(keyword)) - passed, _skipped, _failed = reprec.listoutcomes() - assert len(passed) == 1 - assert passed[0].nodeid.endswith("test_2") - dlist = reprec.getcalls("pytest_deselected") - assert len(dlist) == 1 - assert dlist[0].items[0].name == "test_1" - - def test_keyword_extra(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - def test_one(): - assert 0 - test_one.mykeyword = True - """ + record.assert_outcomes(passed=1) + (nodeid,) = record.by_test + assert nodeid.endswith("test_2") + assert len(recorder.calls) == 1 + assert recorder.calls[0][0].name == "test_1" # type: ignore[attr-defined] + + def test_keyword_extra(self, tmp_path: Path) -> None: + def test_one(): + assert 0 + + setattr(test_one, "mykeyword", True) + + record = run_tests( + test_one, rootpath=tmp_path, spec=ConfigSpec(args=("-k", "mykeyword")) ) - reprec = pytester.inline_run("-k", "mykeyword", p) - _passed, _skipped, failed = reprec.countoutcomes() - assert failed == 1 + record.assert_outcomes(failed=1) @pytest.mark.xfail - def test_keyword_extra_dash(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - def test_one(): - assert 0 - test_one.mykeyword = True - """ - ) + def test_keyword_extra_dash(self, tmp_path: Path) -> None: + def test_one(): + assert 0 + + setattr(test_one, "mykeyword", True) + # with argparse the argument to an option cannot # start with '-' - reprec = pytester.inline_run("-k", "-mykeyword", p) - passed, skipped, failed = reprec.countoutcomes() - assert passed + skipped + failed == 0 + record = run_tests( + test_one, rootpath=tmp_path, spec=ConfigSpec(args=("-k", "-mykeyword")) + ) + record.assert_outcomes() @pytest.mark.parametrize( "keyword", ["__", "+", ".."], ) - def test_no_magic_values(self, pytester: Pytester, keyword: str) -> None: + def test_no_magic_values(self, tmp_path: Path, keyword: str) -> None: """Make sure the tests do not match on magic values, no double underscored values, like '__dict__' and '+'. """ - p = pytester.makepyfile( - """ - def test_one(): assert 1 - """ - ) - reprec = pytester.inline_run("-k", keyword, p) - passed, skipped, failed = reprec.countoutcomes() - dlist = reprec.getcalls("pytest_deselected") - assert passed + skipped + failed == 0 - deselected_tests = dlist[0].items - assert len(deselected_tests) == 1 + def test_one(): + assert 1 + record = run_tests( + test_one, + rootpath=tmp_path, + spec=ConfigSpec(args=("-k", keyword)), + name="test_no_magic_values", + ) + record.assert_outcomes(deselected=1) + + # ensemble: `-k` matching against directory names needs a Directory node + # above the module, which preset in-memory collection has no equivalent for. def test_no_match_directories_outside_the_suite( self, pytester: Pytester, @@ -1126,22 +1118,18 @@ def test_aliases(self) -> None: @pytest.mark.parametrize("mark", [None, "skip", "xfail"]) def test_parameterset_for_parametrize_marks( - pytester: Pytester, mark: _EmptyParameterSetMark | None + tmp_path: Path, mark: _EmptyParameterSetMark | None ) -> None: + inicfg: dict[str, object] = {} if mark is not None: - pytester.makeini( - f""" - [pytest] - {EMPTY_PARAMETERSET_OPTION}={mark} - """ - ) + inicfg[EMPTY_PARAMETERSET_OPTION] = mark - config = pytester.parseconfig() from _pytest.mark import get_empty_parameterset_mark - from _pytest.mark import pytest_configure - pytest_configure(config) - result_mark = get_empty_parameterset_mark(config, ["a"], all) + # ``configured()`` has already run _pytest.mark's pytest_configure, which + # is what the pytester version had to do by hand on a merely parsed config. + with configured(ConfigSpec(rootpath=tmp_path, inicfg=inicfg)) as config: + result_mark = get_empty_parameterset_mark(config, ["a"], all) if mark is None: # normalize to the default mark = "skip" @@ -1151,23 +1139,20 @@ def test_parameterset_for_parametrize_marks( assert result_mark.kwargs.get("run") is False -def test_parameterset_for_parametrize_marks_invalid(pytester: Pytester) -> None: - pytester.makeini( - f""" - [pytest] - {EMPTY_PARAMETERSET_OPTION}=dontcare - """ - ) - result = pytester.runpytest() - assert result.ret == pytest.ExitCode.USAGE_ERROR - result.stderr.fnmatch_lines( - [ - f"*ERROR: *: config option '{EMPTY_PARAMETERSET_OPTION}' expects one of " - "'skip' | 'xfail' | 'fail_at_collect', got 'dontcare'" - ] +def test_parameterset_for_parametrize_marks_invalid(tmp_path: Path) -> None: + spec = ConfigSpec(rootpath=tmp_path, inicfg={EMPTY_PARAMETERSET_OPTION: "dontcare"}) + expected = ( + f"config option '{EMPTY_PARAMETERSET_OPTION}' expects one of " + "'skip' | 'xfail' | 'fail_at_collect', got 'dontcare'" ) + with pytest.raises(UsageError, match=re.escape(expected)): + with configured(spec): + pass +# ensemble: asserts a host-anchored source line ("at line 3") in the rendered +# collection error, plus the INTERRUPTED exit code, neither of which an +# ensemble has. def test_parameterset_for_fail_at_collect(pytester: Pytester) -> None: pytester.makeini( f""" @@ -1209,58 +1194,45 @@ def test(): assert result.ret == ExitCode.INTERRUPTED -def test_paramset_empty_no_idfunc( - pytester: Pytester, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_paramset_empty_no_idfunc(tmp_path: Path) -> None: """An empty parameter set should not call the user provided id function (#13031).""" - p1 = pytester.makepyfile( - """ - import pytest - def idfunc(value): - raise ValueError() - @pytest.mark.parametrize("param", [], ids=idfunc) - def test(param): - pass - """ - ) - result = pytester.runpytest(p1, "-v", "-rs") - result.stdout.fnmatch_lines( - [ - "* collected 1 item", - "test_paramset_empty_no_idfunc* SKIPPED *", - "SKIPPED [1] test_paramset_empty_no_idfunc.py:5: got empty parameter set for (param)", - "*= 1 skipped in *", - ] - ) + def idfunc(value): + raise ValueError() + @pytest.mark.parametrize("param", [], ids=idfunc) + def test(param): + pass -def test_mark_expressions_no_smear(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + # collecting at all proves idfunc was never called: it would blow up. + assert len(collect_tests(test, rootpath=tmp_path)) == 1 + record = run_tests(test, rootpath=tmp_path) + record.assert_outcomes(skipped=1) + (item_record,) = record.by_test.values() + assert item_record.setup is not None + assert "got empty parameter set for (param)" in item_record.setup.longreprtext - class BaseTests(object): - def test_something(self): - pass - @pytest.mark.FOO - class TestFooClass(BaseTests): +def test_mark_expressions_no_smear(tmp_path: Path) -> None: + class BaseTests: + def test_something(self): pass - @pytest.mark.BAR - class TestBarClass(BaseTests): - pass - """ - ) + @ensemble_mark("FOO") + class TestFooClass(BaseTests): + pass + + @ensemble_mark("BAR") + class TestBarClass(BaseTests): + pass - reprec = pytester.inline_run("-m", "FOO") - passed, skipped, failed = reprec.countoutcomes() - dlist = reprec.getcalls("pytest_deselected") - assert passed == 1 - assert skipped == failed == 0 - deselected_tests = dlist[0].items - assert len(deselected_tests) == 1 + record = run_tests( + TestFooClass, + TestBarClass, + rootpath=tmp_path, + spec=ConfigSpec(args=("-m", "FOO")), + ) + record.assert_outcomes(passed=1, deselected=1) # todo: fixed # keywords smear - expected behaviour @@ -1270,12 +1242,12 @@ class TestBarClass(BaseTests): # assert skipped_k == failed_k == 0 -def test_addmarker_order(pytester) -> None: +def test_addmarker_order(tmp_path: Path) -> None: session = mock.Mock() session.own_markers = [] session.parent = None session.nodeid = "" - session.path = pytester.path + session.path = tmp_path node = Node.from_parent(session, name="Test") node.add_marker("foo") node.add_marker("bar") @@ -1285,41 +1257,40 @@ def test_addmarker_order(pytester) -> None: @pytest.mark.filterwarnings("ignore") -def test_markers_from_parametrize(pytester: Pytester) -> None: +def test_markers_from_parametrize(tmp_path: Path) -> None: """#3605""" - pytester.makepyfile( - """ - import pytest + first_custom_mark = ensemble_mark("custom_marker") + custom_mark = ensemble_mark("custom_mark") - first_custom_mark = pytest.mark.custom_marker - custom_mark = pytest.mark.custom_mark - @pytest.fixture(autouse=True) - def trigger(request): - custom_mark = list(request.node.iter_markers('custom_mark')) - print("Custom mark %s" % custom_mark) - - @custom_mark("custom mark non parametrized") - def test_custom_mark_non_parametrized(): - print("Hey from test") - - @pytest.mark.parametrize( - "obj_type", - [ - first_custom_mark("first custom mark")("template"), - pytest.param( # Think this should be recommended way? - "disk", - marks=custom_mark('custom mark1') - ), - custom_mark("custom mark2")("vm"), # Tried also this - ] - ) - def test_custom_mark_parametrized(obj_type): - print("obj_type is:", obj_type) - """ - ) + @pytest.fixture(autouse=True) + def trigger(request): + seen = list(request.node.iter_markers("custom_mark")) + print(f"Custom mark {seen}") - result = pytester.runpytest() - result.assert_outcomes(passed=4) + @custom_mark("custom mark non parametrized") + def test_custom_mark_non_parametrized(): + print("Hey from test") + + @pytest.mark.parametrize( + "obj_type", + [ + first_custom_mark("first custom mark")("template"), + pytest.param( # Think this should be recommended way? + "disk", marks=custom_mark("custom mark1") + ), + custom_mark("custom mark2")("vm"), # Tried also this + ], + ) + def test_custom_mark_parametrized(obj_type): + print("obj_type is:", obj_type) + + record = run_tests( + trigger, + test_custom_mark_non_parametrized, + test_custom_mark_parametrized, + rootpath=tmp_path, + ) + record.assert_outcomes(passed=4) def test_pytest_param_id_requires_string() -> None: @@ -1339,20 +1310,14 @@ def test_pytest_param_id_allows_none_or_string(s) -> None: @pytest.mark.parametrize("expr", ("NOT internal_err", "NOT (internal_err)", "bogus=")) -def test_marker_expr_eval_failure_handling(pytester: Pytester, expr) -> None: - foo = pytester.makepyfile( - """ - import pytest +def test_marker_expr_eval_failure_handling(tmp_path: Path, expr) -> None: + @ensemble_mark("internal_err") + def test_foo(): + pass - @pytest.mark.internal_err - def test_foo(): - pass - """ - ) - expected = f"ERROR: Wrong expression passed to '-m': {expr}: *" - result = pytester.runpytest(foo, "-m", expr) - result.stderr.fnmatch_lines([expected]) - assert result.ret == ExitCode.USAGE_ERROR + expected = f"Wrong expression passed to '-m': {expr}: " + with pytest.raises(UsageError, match=re.escape(expected)): + run_tests(test_foo, rootpath=tmp_path, spec=ConfigSpec(args=("-m", expr))) def test_mark_mro() -> None: @@ -1380,73 +1345,63 @@ class C(A, B): # @pytest.mark.issue("https://github.com/pytest-dev/pytest/issues/10447") -def test_mark_fixture_order_mro(pytester: Pytester): +def test_mark_fixture_order_mro(tmp_path: Path): """This ensures we walk marks of the mro starting with the base classes the action at a distance fixtures are taken as minimal example from a real project """ - foo = pytester.makepyfile( - """ - import pytest - - @pytest.fixture - def add_attr1(request): - request.instance.attr1 = object() - - @pytest.fixture - def add_attr2(request): - request.instance.attr2 = request.instance.attr1 + @pytest.fixture + def add_attr1(request): + request.instance.attr1 = object() + @pytest.fixture + def add_attr2(request): + request.instance.attr2 = request.instance.attr1 - @pytest.mark.usefixtures('add_attr1') - class Parent: - pass + @pytest.mark.usefixtures("add_attr1") + class Parent: + pass + @pytest.mark.usefixtures("add_attr2") + class TestThings(Parent): + def test_attrs(self): + # both attributes are injected by the usefixtures fixtures above + assert self.attr1 == self.attr2 # type: ignore[attr-defined] - @pytest.mark.usefixtures('add_attr2') - class TestThings(Parent): - def test_attrs(self): - assert self.attr1 == self.attr2 - """ - ) - result = pytester.runpytest(foo) - result.assert_outcomes(passed=1) + record = run_tests(add_attr1, add_attr2, TestThings, rootpath=tmp_path) + record.assert_outcomes(passed=1) -def test_mark_parametrize_over_staticmethod(pytester: Pytester) -> None: +def test_mark_parametrize_over_staticmethod(tmp_path: Path) -> None: """Check that applying marks works as intended on classmethods and staticmethods. Regression test for #12863. """ - pytester.makepyfile( - """ - import pytest - class TestClass: - @pytest.mark.parametrize("value", [1, 2]) - @classmethod - def test_classmethod_wrapper(cls, value: int): - assert value in [1, 2] - - @classmethod - @pytest.mark.parametrize("value", [1, 2]) - def test_classmethod_wrapper_on_top(cls, value: int): - assert value in [1, 2] - - @pytest.mark.parametrize("value", [1, 2]) - @staticmethod - def test_staticmethod_wrapper(value: int): - assert value in [1, 2] - - @staticmethod - @pytest.mark.parametrize("value", [1, 2]) - def test_staticmethod_wrapper_on_top(value: int): - assert value in [1, 2] - """ - ) - result = pytester.runpytest() - result.assert_outcomes(passed=8) + class TestClass: + @pytest.mark.parametrize("value", [1, 2]) + @classmethod + def test_classmethod_wrapper(cls, value: int): + assert value in [1, 2] + + @classmethod + @pytest.mark.parametrize("value", [1, 2]) + def test_classmethod_wrapper_on_top(cls, value: int): + assert value in [1, 2] + + @pytest.mark.parametrize("value", [1, 2]) + @staticmethod + def test_staticmethod_wrapper(value: int): + assert value in [1, 2] + + @staticmethod + @pytest.mark.parametrize("value", [1, 2]) + def test_staticmethod_wrapper_on_top(value: int): + assert value in [1, 2] + + record = run_tests(TestClass, rootpath=tmp_path) + record.assert_outcomes(passed=8) def test_fixture_disallow_on_marked_functions() -> None: @@ -1491,29 +1446,35 @@ def foo(): raise NotImplementedError() -def test_module_getattr_without_attributeerror(pytester: Pytester) -> None: +def test_module_getattr_without_attributeerror(tmp_path: Path) -> None: """ Test that a helpful warning is emitted when a module-level __getattr__ returns None instead of raising AttributeError. Regression test for https://github.com/pytest-dev/pytest/issues/8265 """ - pytester.makepyfile( - """ - def __getattr__(key): - # Bug: should raise AttributeError, but returns None - return None - def test_something(): - assert True - """ + def __getattr__(key): + # Bug: should raise AttributeError, but returns None + return None + + def test_something(): + assert True + + module = build_module( + "test_module_getattr", test_something, __getattr__=__getattr__ ) - result = pytester.runpytest("-W", "always::pytest.PytestCollectionWarning") - result.stdout.fnmatch_lines( - [ - "*PytestCollectionWarning*__getattr__*returns None*AttributeError*", - ] + spec = ConfigSpec( + rootpath=tmp_path, args=("-W", "always::pytest.PytestCollectionWarning") ) # The module is buggy (__getattr__ returns None for all attributes), - # so no tests are collected, but pytest should NOT crash with a TypeError. - assert result.ret != ExitCode.INTERNAL_ERROR + # so no tests are collected, but pytest should NOT crash with a TypeError - + # which in an ensemble would surface as the exception escaping run_tests. + record = run_tests(module, spec=spec) + record.assert_outcomes() + (warning,) = [ + w for w in record.warnings if w.category is pytest.PytestCollectionWarning + ] + assert "__getattr__" in str(warning.message) + assert "returns None" in str(warning.message) + assert "AttributeError" in str(warning.message) From a52148705f86f3e2ae63963ce8b818ec0c08ab06 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 19:39:06 +0200 Subject: [PATCH 05/30] testing: port collect.py to _pytest.ensemble 47 tests ported, and the file is 78 lines shorter for it. Rendering globs became exact nodeid lists in several places - test_ordered_by_definition_order and test_abstract_class_is_not_collected now assert the whole collected sequence rather than fnmatch-ing a few lines of --collect-only output. The build_module naming rule caught two sources: @pytest.fixture returns a FixtureFunctionDefinition whose __name__ comes from the wrapped function, so a decorated fixture passed positionally would have registered under the wrong name and left the test green with the fixture simply missing. Both are keyword members now, with a note saying why. 29 stay on pytester: real module import (10, the whole TestModule class is about the import chokepoint an ensemble bypasses), real files, packages and duplicate arguments (9), conftest scoping and hooks that never fire (4), host-anchored reportinfo (3), rendering anchored at conftest line numbers (2), and python_files discovery (1). --- testing/python/collect.py | 1352 +++++++++++++++++-------------------- 1 file changed, 637 insertions(+), 715 deletions(-) diff --git a/testing/python/collect.py b/testing/python/collect.py index dc37031ede8..7f6a0683992 100644 --- a/testing/python/collect.py +++ b/testing/python/collect.py @@ -8,17 +8,23 @@ from typing import Any import _pytest._code -from _pytest.config import ExitCode +from _pytest.ensemble import build_module from _pytest.ensemble import collect_tests +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import Ensemble from _pytest.ensemble import run_tests from _pytest.monkeypatch import MonkeyPatch from _pytest.nodes import Collector +from _pytest.nodes import Node from _pytest.pytester import Pytester from _pytest.python import Class from _pytest.python import Function import pytest +# ensemble: this whole class is about the module *import* chokepoint - real +# files on disk, import modes, import errors and their tracebacks - which is +# exactly what EnsembleModule bypasses by serving a preset object. class TestModule: def test_failing_import(self, pytester: Pytester) -> None: modcol = pytester.getmodulecol("import alksdjalskdjalkjals") @@ -150,37 +156,37 @@ def test_show_traceback_import_error_unicode(self, pytester: Pytester) -> None: class TestClass: - def test_class_with_init_warning(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - class TestClass1(object): - def __init__(self): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - "*cannot collect test class 'TestClass1' because it has " - "a __init__ constructor (from: test_class_with_init_warning.py)" - ] - ) + def test_class_with_init_warning(self, tmp_path: Path) -> None: + class TestClass1: + def __init__(self): + pass - def test_class_with_new_warning(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - class TestClass1(object): - def __new__(self): - pass - """ + module = build_module("test_class_with_init_warning", TestClass1) + record = run_tests( + module, + spec=ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["always"]}), ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - "*cannot collect test class 'TestClass1' because it has " - "a __new__ constructor (from: test_class_with_new_warning.py)" - ] + record.assert_outcomes() + assert [str(w.message) for w in record.warnings] == [ + "cannot collect test class 'TestClass1' because it has " + "a __init__ constructor (from: test_class_with_init_warning.py)" + ] + + def test_class_with_new_warning(self, tmp_path: Path) -> None: + class TestClass1: + def __new__(self): # noqa: PLW0211 + pass + + module = build_module("test_class_with_new_warning", TestClass1) + record = run_tests( + module, + spec=ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["always"]}), ) + record.assert_outcomes() + assert [str(w.message) for w in record.warnings] == [ + "cannot collect test class 'TestClass1' because it has " + "a __new__ constructor (from: test_class_with_new_warning.py)" + ] def test_class_subclassobject(self, tmp_path: Path) -> None: class test: @@ -189,18 +195,21 @@ class test: assert collect_tests(test, rootpath=tmp_path) == [] def test_class_from_parent_without_obj_resolves_by_name( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Without an explicit obj, the class is looked up on the parent's object.""" - modcol = pytester.getmodulecol( - """ - class TestGroup: - def test_method(self): - pass - """ - ) - cls = pytest.Class.from_parent(modcol, name="TestGroup") - assert cls.obj is modcol.obj.TestGroup + + class TestGroup: + def test_method(self): + pass + + module = build_module("test_from_parent", TestGroup) + with Ensemble(module, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + modcol = item.getparent(pytest.Module) + assert modcol is not None + cls = pytest.Class.from_parent(modcol, name="TestGroup") + assert cls.obj is modcol.obj.TestGroup def test_static_method(self, tmp_path: Path) -> None: """Support for collecting staticmethod tests (#2528, #2699)""" @@ -240,159 +249,145 @@ def teardown_class(cls): record.assert_outcomes(passed=1) assert events == ["setup", "teardown"] - def test_issue1035_obj_has_getattr(self, pytester: Pytester) -> None: - modcol = pytester.getmodulecol( - """ - class Chameleon(object): - def __getattr__(self, name): - return True - chameleon = Chameleon() - """ - ) - colitems = modcol.collect() - assert len(colitems) == 0 + def test_issue1035_obj_has_getattr(self, tmp_path: Path) -> None: + class Chameleon: + def __getattr__(self, name): + return True - def test_issue1579_namedtuple(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import collections + module = build_module("test_issue1035", Chameleon, chameleon=Chameleon()) + assert collect_tests(module, rootpath=tmp_path) == [] - TestCase = collections.namedtuple('TestCase', ['a']) - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - "*cannot collect test class 'TestCase' " - "because it has a __new__ constructor*" - ) + def test_issue1579_namedtuple(self, tmp_path: Path) -> None: + import collections - def test_issue2234_property(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - class TestCase(object): - @property - def prop(self): - raise NotImplementedError() - """ + TestCase = collections.namedtuple("TestCase", ["a"]) # noqa: PYI024 + + module = build_module("test_issue1579_namedtuple", TestCase) + record = run_tests( + module, + spec=ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["always"]}), ) - result = pytester.runpytest() - assert result.ret == ExitCode.NO_TESTS_COLLECTED + record.assert_outcomes() + assert [str(w.message) for w in record.warnings] == [ + "cannot collect test class 'TestCase' because it has " + "a __new__ constructor (from: test_issue1579_namedtuple.py)" + ] + + def test_issue2234_property(self, tmp_path: Path) -> None: + class TestCase: + @property + def prop(self): + raise NotImplementedError - def test_does_not_discover_properties(self, pytester: Pytester) -> None: + assert collect_tests(TestCase, rootpath=tmp_path) == [] + + def test_does_not_discover_properties(self, tmp_path: Path) -> None: """Regression test for #12446.""" - pytester.makepyfile( - """\ - class TestCase: - @property - def oops(self): - raise SystemExit('do not call me!') - """ - ) - result = pytester.runpytest() - assert result.ret == ExitCode.NO_TESTS_COLLECTED - def test_does_not_discover_instance_descriptors(self, pytester: Pytester) -> None: + class TestCase: + @property + def oops(self): + raise SystemExit("do not call me!") + + assert collect_tests(TestCase, rootpath=tmp_path) == [] + + def test_does_not_discover_instance_descriptors(self, tmp_path: Path) -> None: """Regression test for #12446.""" - pytester.makepyfile( - """\ - # not `@property`, but it acts like one - # this should cover the case of things like `@cached_property` / etc. - class MyProperty: - def __init__(self, func): - self._func = func - def __get__(self, inst, owner): - if inst is None: - return self - else: - return self._func.__get__(inst, owner)() - - class TestCase: - @MyProperty - def oops(self): - raise SystemExit('do not call me!') - """ - ) - result = pytester.runpytest() - assert result.ret == ExitCode.NO_TESTS_COLLECTED + + # not `@property`, but it acts like one + # this should cover the case of things like `@cached_property` / etc. + class MyProperty: + def __init__(self, func): + self._func = func + + def __get__(self, inst, owner): + if inst is None: + return self + else: + return self._func.__get__(inst, owner)() + + class TestCase: + @MyProperty + def oops(self): + raise SystemExit("do not call me!") + + assert collect_tests(TestCase, rootpath=tmp_path) == [] def test_does_not_eval_properties_when_collecting_tests( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Regression test for #2568. Properties on a test class must only be evaluated when a test accesses them, not during collection or fixture parsing. """ - pytester.makepyfile( - """\ - calls = [] + calls: list[int] = [] - class TestCase: - @property - def prop(self): - calls.append(1) - return len(calls) + class TestCase: + @property + def prop(self): + calls.append(1) + return len(calls) - def test_prop(self): - assert self.prop == 1 - """ - ) - result = pytester.runpytest() - result.assert_outcomes(passed=1) + def test_prop(self): + assert self.prop == 1 - def test_abstract_class_is_not_collected(self, pytester: Pytester) -> None: + record = run_tests(TestCase, rootpath=tmp_path) + record.assert_outcomes(passed=1) + assert calls == [1] + + def test_abstract_class_is_not_collected(self, tmp_path: Path) -> None: """Regression test for #12275 (non-unittest version).""" - pytester.makepyfile( - """ - import abc + import abc - class TestBase(abc.ABC): - @abc.abstractmethod - def abstract1(self): pass + class TestBase(abc.ABC): + @abc.abstractmethod + def abstract1(self): ... - @abc.abstractmethod - def abstract2(self): pass + @abc.abstractmethod + def abstract2(self): ... - def test_it(self): pass + def test_it(self): ... # noqa: B027 - class TestPartial(TestBase): - def abstract1(self): pass + class TestPartial(TestBase): + def abstract1(self): ... - class TestConcrete(TestPartial): - def abstract2(self): pass - """ + class TestConcrete(TestPartial): + def abstract2(self): ... + + record = run_tests( + TestBase, TestPartial, TestConcrete, rootpath=tmp_path, name="test_abstract" ) - result = pytester.runpytest() - assert result.ret == ExitCode.OK - result.assert_outcomes(passed=1) + record.assert_outcomes(passed=1) + assert list(record.by_test) == ["test_abstract.py::TestConcrete::test_it"] class TestFunction: - def test_getmodulecollector(self, pytester: Pytester) -> None: - item = pytester.getitem("def test_func(): pass") + def test_getmodulecollector(self, tmp_path: Path) -> None: + def test_func(): + pass + + (item,) = collect_tests(test_func, rootpath=tmp_path) modcol = item.getparent(pytest.Module) assert isinstance(modcol, pytest.Module) assert hasattr(modcol.obj, "test_func") - @pytest.mark.filterwarnings("default") - def test_function_as_object_instance_ignored(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - class A(object): - def __call__(self, tmp_path): - 0/0 - - test_a = A() - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - "collected 0 items", - "*test_function_as_object_instance_ignored.py:2: " - "*cannot collect 'test_a' because it is not a function.", - ] - ) + def test_function_as_object_instance_ignored(self, tmp_path: Path) -> None: + class A: + def __call__(self, tmp_path): + 0 / 0 # noqa: B018 + + # ensemble: the warning's file:line location is host-anchored, so only + # the message itself is asserted on. + module = build_module("test_function_as_object_instance_ignored", A, test_a=A()) + record = run_tests( + module, + spec=ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["always"]}), + ) + record.assert_outcomes() + assert [str(w.message) for w in record.warnings] == [ + "cannot collect 'test_a' because it is not a function." + ] @staticmethod def make_function(tmp_path: Path, **kwargs: Any) -> Any: @@ -424,228 +419,184 @@ def test_repr_produces_actual_test_id(self, tmp_path: Path) -> None: ) assert repr(f) == r"" - def test_issue197_parametrize_emptyset(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize('arg', []) - def test_function(arg): - pass - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(skipped=1) + def test_issue197_parametrize_emptyset(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("arg", []) + def test_function(arg): + pass - def test_single_tuple_unwraps_values(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize(('arg',), [(1,)]) - def test_function(arg): - assert arg == 1 - """ - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + run_tests(test_function, rootpath=tmp_path).assert_outcomes(skipped=1) - def test_issue213_parametrize_value_no_equal(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - class A(object): - def __eq__(self, other): - raise ValueError("not possible") - @pytest.mark.parametrize('arg', [A()]) - def test_function(arg): - assert arg.__class__.__name__ == "A" - """ - ) - reprec = pytester.inline_run("--fulltrace") - reprec.assertoutcome(passed=1) + def test_single_tuple_unwraps_values(self, tmp_path: Path) -> None: + @pytest.mark.parametrize(("arg",), [(1,)]) + def test_function(arg): + assert arg == 1 + + run_tests(test_function, rootpath=tmp_path).assert_outcomes(passed=1) + + def test_issue213_parametrize_value_no_equal(self, tmp_path: Path) -> None: + class A: + def __eq__(self, other): + raise ValueError("not possible") + + @pytest.mark.parametrize("arg", [A()]) + def test_function(arg): + assert arg.__class__.__name__ == "A" - def test_parametrize_with_non_hashable_values(self, pytester: Pytester) -> None: + run_tests(test_function, rootpath=tmp_path).assert_outcomes(passed=1) + + def test_parametrize_with_non_hashable_values(self, tmp_path: Path) -> None: """Test parametrization with non-hashable values.""" - pytester.makepyfile( - """ - archival_mapping = { - '1.0': {'tag': '1.0'}, - '1.2.2a1': {'tag': 'release-1.2.2a1'}, - } + archival_mapping = { + "1.0": {"tag": "1.0"}, + "1.2.2a1": {"tag": "release-1.2.2a1"}, + } - import pytest - @pytest.mark.parametrize('key value'.split(), - archival_mapping.items()) - def test_archival_to_version(key, value): - assert key in archival_mapping - assert value == archival_mapping[key] - """ - ) - rec = pytester.inline_run() - rec.assertoutcome(passed=2) + @pytest.mark.parametrize("key value".split(), archival_mapping.items()) + def test_archival_to_version(key, value): + assert key in archival_mapping + assert value == archival_mapping[key] + + run_tests(test_archival_to_version, rootpath=tmp_path).assert_outcomes(passed=2) def test_parametrize_with_non_hashable_values_indirect( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test parametrization with non-hashable values with indirect parametrization.""" - pytester.makepyfile( - """ - archival_mapping = { - '1.0': {'tag': '1.0'}, - '1.2.2a1': {'tag': 'release-1.2.2a1'}, - } + archival_mapping = { + "1.0": {"tag": "1.0"}, + "1.2.2a1": {"tag": "release-1.2.2a1"}, + } - import pytest + @pytest.fixture + def key(request): + return request.param - @pytest.fixture - def key(request): - return request.param + @pytest.fixture + def value(request): + return request.param - @pytest.fixture - def value(request): - return request.param - - @pytest.mark.parametrize('key value'.split(), - archival_mapping.items(), indirect=True) - def test_archival_to_version(key, value): - assert key in archival_mapping - assert value == archival_mapping[key] - """ + @pytest.mark.parametrize( + "key value".split(), archival_mapping.items(), indirect=True ) - rec = pytester.inline_run() - rec.assertoutcome(passed=2) + def test_archival_to_version(key, value): + assert key in archival_mapping + assert value == archival_mapping[key] + + record = run_tests(key, value, test_archival_to_version, rootpath=tmp_path) + record.assert_outcomes(passed=2) - def test_parametrize_overrides_fixture(self, pytester: Pytester) -> None: + def test_parametrize_overrides_fixture(self, tmp_path: Path) -> None: """Test parametrization when parameter overrides existing fixture with same name.""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def value(): - return 'value' - - @pytest.mark.parametrize('value', - ['overridden']) - def test_overridden_via_param(value): - assert value == 'overridden' - - @pytest.mark.parametrize('somevalue', ['overridden']) - def test_not_overridden(value, somevalue): - assert value == 'value' - assert somevalue == 'overridden' - - @pytest.mark.parametrize('other,value', [('foo', 'overridden')]) - def test_overridden_via_multiparam(other, value): - assert other == 'foo' - assert value == 'overridden' - """ + @pytest.fixture + def value(): + return "value" + + @pytest.mark.parametrize("value", ["overridden"]) + def test_overridden_via_param(value): + assert value == "overridden" + + @pytest.mark.parametrize("somevalue", ["overridden"]) + def test_not_overridden(value, somevalue): + assert value == "value" + assert somevalue == "overridden" + + @pytest.mark.parametrize("other,value", [("foo", "overridden")]) + def test_overridden_via_multiparam(other, value): + assert other == "foo" + assert value == "overridden" + + record = run_tests( + value, + test_overridden_via_param, + test_not_overridden, + test_overridden_via_multiparam, + rootpath=tmp_path, ) - rec = pytester.inline_run() - rec.assertoutcome(passed=3) + record.assert_outcomes(passed=3) - def test_parametrize_overrides_parametrized_fixture( - self, pytester: Pytester - ) -> None: + def test_parametrize_overrides_parametrized_fixture(self, tmp_path: Path) -> None: """Test parametrization when parameter overrides existing parametrized fixture with same name.""" - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=[1, 2]) - def value(request): - return request.param + @pytest.fixture(params=[1, 2]) + def value(request): + return request.param - @pytest.mark.parametrize('value', - ['overridden']) - def test_overridden_via_param(value): - assert value == 'overridden' - """ - ) - rec = pytester.inline_run() - rec.assertoutcome(passed=1) + @pytest.mark.parametrize("value", ["overridden"]) + def test_overridden_via_param(value): + assert value == "overridden" + + record = run_tests(value, test_overridden_via_param, rootpath=tmp_path) + record.assert_outcomes(passed=1) def test_parametrize_overrides_parametrized_fixture_with_unrelated_indirect( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test parametrization when parameter overrides existing parametrized fixture with same name, and there is an unrelated indirect param. Regression test for #13974. """ - pytester.makepyfile( - """ - import pytest - @pytest.fixture(params=["a", "b"]) - def target(request): - return request.param + @pytest.fixture(params=["a", "b"]) + def target(request): + return request.param - @pytest.fixture - def val(request): - return int(request.param) - - @pytest.mark.parametrize( - ["val", "target"], - [ - ("1", 1), - ("2", 2), - ], - indirect=["val"], - ) - def test(val, target): - assert val == target - """ + @pytest.fixture + def val(request): + return int(request.param) + + @pytest.mark.parametrize( + ["val", "target"], + [ + ("1", 1), + ("2", 2), + ], + indirect=["val"], ) - result = pytester.runpytest() - assert result.ret == 0 - result.assert_outcomes(passed=2) + def test(val, target): + assert val == target + + record = run_tests(target, val, test, rootpath=tmp_path) + record.assert_outcomes(passed=2) def test_parametrize_overrides_indirect_dependency_fixture( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Test parametrization when parameter overrides a fixture that a test indirectly depends on""" - pytester.makepyfile( - """ - import pytest + fix3_instantiated = [] - fix3_instantiated = False + @pytest.fixture + def fix1(fix2): + return fix2 + "1" - @pytest.fixture - def fix1(fix2): - return fix2 + '1' + @pytest.fixture + def fix2(fix3): + return fix3 + "2" - @pytest.fixture - def fix2(fix3): - return fix3 + '2' + @pytest.fixture + def fix3(): + fix3_instantiated.append(True) + return "3" - @pytest.fixture - def fix3(): - global fix3_instantiated - fix3_instantiated = True - return '3' - - @pytest.mark.parametrize('fix2', ['2']) - def test_it(fix1): - assert fix1 == '21' - assert not fix3_instantiated - """ - ) - rec = pytester.inline_run() - rec.assertoutcome(passed=1) + @pytest.mark.parametrize("fix2", ["2"]) + def test_it(fix1): + assert fix1 == "21" + assert not fix3_instantiated - def test_parametrize_with_mark(self, pytester: Pytester) -> None: - items = pytester.getitems( - """ - import pytest - @pytest.mark.foo - @pytest.mark.parametrize('arg', [ - 1, - pytest.param(2, marks=[pytest.mark.baz, pytest.mark.bar]) - ]) - def test_function(arg): - pass - """ + record = run_tests(fix1, fix2, fix3, test_it, rootpath=tmp_path) + record.assert_outcomes(passed=1) + assert fix3_instantiated == [] + + def test_parametrize_with_mark(self, tmp_path: Path) -> None: + @pytest.mark.foo + @pytest.mark.parametrize( + "arg", [1, pytest.param(2, marks=[pytest.mark.baz, pytest.mark.bar])] ) + def test_function(arg): + pass + + items = collect_tests(test_function, rootpath=tmp_path) keywords = [item.keywords for item in items] assert ( "foo" in keywords[0] @@ -654,186 +605,138 @@ def test_function(arg): ) assert "foo" in keywords[1] and "bar" in keywords[1] and "baz" in keywords[1] - def test_parametrize_with_empty_string_arguments(self, pytester: Pytester) -> None: - items = pytester.getitems( - """\ - import pytest + def test_parametrize_with_empty_string_arguments(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("v", ("", " ")) + @pytest.mark.parametrize("w", ("", " ")) + def test(v, w): ... - @pytest.mark.parametrize('v', ('', ' ')) - @pytest.mark.parametrize('w', ('', ' ')) - def test(v, w): ... - """ - ) + items = collect_tests(test, rootpath=tmp_path) names = {item.name for item in items} assert names == {"test[-]", "test[ -]", "test[- ]", "test[ - ]"} - def test_function_equality_with_callspec(self, pytester: Pytester) -> None: - items = pytester.getitems( - """ - import pytest - @pytest.mark.parametrize('arg', [1,2]) - def test_function(arg): - pass - """ - ) + def test_function_equality_with_callspec(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("arg", [1, 2]) + def test_function(arg): + pass + + items = collect_tests(test_function, rootpath=tmp_path) assert items[0] != items[1] assert not (items[0] == items[1]) - def test_pyfunc_call(self, pytester: Pytester) -> None: - item = pytester.getitem("def test_func(): raise ValueError") - config = item.config + def test_pyfunc_call(self, tmp_path: Path) -> None: + def test_func(): + raise ValueError - class MyPlugin1: - def pytest_pyfunc_call(self): - raise ValueError + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + config = ensemble.config - class MyPlugin2: - def pytest_pyfunc_call(self): - return True + class MyPlugin1: + def pytest_pyfunc_call(self): + raise ValueError - config.pluginmanager.register(MyPlugin1()) - config.pluginmanager.register(MyPlugin2()) - config.hook.pytest_runtest_setup(item=item) - config.hook.pytest_pyfunc_call(pyfuncitem=item) + class MyPlugin2: + def pytest_pyfunc_call(self): + return True - def test_multiple_parametrize(self, pytester: Pytester) -> None: - modcol = pytester.getmodulecol( - """ - import pytest - @pytest.mark.parametrize('x', [0, 1]) - @pytest.mark.parametrize('y', [2, 3]) - def test1(x, y): - pass - """ - ) - colitems = modcol.collect() + config.pluginmanager.register(MyPlugin1()) + config.pluginmanager.register(MyPlugin2()) + config.hook.pytest_runtest_setup(item=item) + config.hook.pytest_pyfunc_call(pyfuncitem=item) + + def test_multiple_parametrize(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("x", [0, 1]) + @pytest.mark.parametrize("y", [2, 3]) + def test1(x, y): + pass + + colitems = collect_tests(test1, rootpath=tmp_path) assert colitems[0].name == "test1[2-0]" assert colitems[1].name == "test1[2-1]" assert colitems[2].name == "test1[3-0]" assert colitems[3].name == "test1[3-1]" - def test_issue751_multiple_parametrize_with_ids(self, pytester: Pytester) -> None: - modcol = pytester.getmodulecol( - """ - import pytest - @pytest.mark.parametrize('x', [0], ids=['c']) - @pytest.mark.parametrize('y', [0, 1], ids=['a', 'b']) - class Test(object): - def test1(self, x, y): - pass - def test2(self, x, y): - pass - """ - ) - colitems = modcol.collect()[0].collect() + def test_issue751_multiple_parametrize_with_ids(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("x", [0], ids=["c"]) + @pytest.mark.parametrize("y", [0, 1], ids=["a", "b"]) + class Test: + def test1(self, x, y): + pass + + def test2(self, x, y): + pass + + colitems = collect_tests(Test, rootpath=tmp_path) assert colitems[0].name == "test1[a-c]" assert colitems[1].name == "test1[b-c]" assert colitems[2].name == "test2[a-c]" assert colitems[3].name == "test2[b-c]" - def test_parametrize_skipif(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_skipif(self, tmp_path: Path) -> None: + m = pytest.mark.skipif("True") - m = pytest.mark.skipif('True') + @pytest.mark.parametrize("x", [0, 1, pytest.param(2, marks=m)]) + def test_skip_if(x): + assert x < 2 - @pytest.mark.parametrize('x', [0, 1, pytest.param(2, marks=m)]) - def test_skip_if(x): - assert x < 2 - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 2 passed, 1 skipped in *"]) + run_tests(test_skip_if, rootpath=tmp_path).assert_outcomes(passed=2, skipped=1) - def test_parametrize_skip(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_skip(self, tmp_path: Path) -> None: + m = pytest.mark.skip("") - m = pytest.mark.skip('') + @pytest.mark.parametrize("x", [0, 1, pytest.param(2, marks=m)]) + def test_skip(x): + assert x < 2 - @pytest.mark.parametrize('x', [0, 1, pytest.param(2, marks=m)]) - def test_skip(x): - assert x < 2 - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 2 passed, 1 skipped in *"]) + run_tests(test_skip, rootpath=tmp_path).assert_outcomes(passed=2, skipped=1) - def test_parametrize_skipif_no_skip(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_skipif_no_skip(self, tmp_path: Path) -> None: + m = pytest.mark.skipif("False") - m = pytest.mark.skipif('False') + @pytest.mark.parametrize("x", [0, 1, m(2)]) + def test_skipif_no_skip(x): + assert x < 2 - @pytest.mark.parametrize('x', [0, 1, m(2)]) - def test_skipif_no_skip(x): - assert x < 2 - """ + run_tests(test_skipif_no_skip, rootpath=tmp_path).assert_outcomes( + passed=2, failed=1 ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 1 failed, 2 passed in *"]) - def test_parametrize_xfail(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_xfail(self, tmp_path: Path) -> None: + m = pytest.mark.xfail("True") - m = pytest.mark.xfail('True') + @pytest.mark.parametrize("x", [0, 1, pytest.param(2, marks=m)]) + def test_xfail(x): + assert x < 2 - @pytest.mark.parametrize('x', [0, 1, pytest.param(2, marks=m)]) - def test_xfail(x): - assert x < 2 - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 2 passed, 1 xfailed in *"]) + run_tests(test_xfail, rootpath=tmp_path).assert_outcomes(passed=2, xfailed=1) - def test_parametrize_passed(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_passed(self, tmp_path: Path) -> None: + m = pytest.mark.xfail("True") - m = pytest.mark.xfail('True') + @pytest.mark.parametrize("x", [0, 1, pytest.param(2, marks=m)]) + def test_xfail(x): + pass - @pytest.mark.parametrize('x', [0, 1, pytest.param(2, marks=m)]) - def test_xfail(x): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 2 passed, 1 xpassed in *"]) + run_tests(test_xfail, rootpath=tmp_path).assert_outcomes(passed=2, xpassed=1) - def test_parametrize_xfail_passed(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest + def test_parametrize_xfail_passed(self, tmp_path: Path) -> None: + m = pytest.mark.xfail("False") - m = pytest.mark.xfail('False') + @pytest.mark.parametrize("x", [0, 1, m(2)]) + def test_passed(x): + pass - @pytest.mark.parametrize('x', [0, 1, m(2)]) - def test_passed(x): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 3 passed in *"]) + run_tests(test_passed, rootpath=tmp_path).assert_outcomes(passed=3) - def test_function_originalname(self, pytester: Pytester) -> None: - items = pytester.getitems( - """ - import pytest + def test_function_originalname(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("arg", [1, 2]) + def test_func(arg): + pass - @pytest.mark.parametrize('arg', [1,2]) - def test_func(arg): - pass + def test_no_param(): + pass - def test_no_param(): - pass - """ - ) + items = collect_tests(test_func, test_no_param, rootpath=tmp_path) originalnames = [] for x in items: assert isinstance(x, pytest.Function) @@ -844,40 +747,46 @@ def test_no_param(): "test_no_param", ] - def test_function_with_square_brackets(self, pytester: Pytester) -> None: + def test_function_with_square_brackets(self, tmp_path: Path) -> None: """Check that functions with square brackets don't cause trouble.""" - p1 = pytester.makepyfile( - """ - locals()["test_foo[name]"] = lambda: None - """ - ) - result = pytester.runpytest("-v", str(p1)) - result.stdout.fnmatch_lines( - [ - "test_function_with_square_brackets.py::test_foo[[]name[]] PASSED *", - "*= 1 passed in *", - ] + module = build_module( + "test_function_with_square_brackets", + **{"test_foo[name]": lambda: None}, ) + record = run_tests(module, rootpath=tmp_path) + record.assert_outcomes(passed=1) + assert list(record.by_test) == [ + "test_function_with_square_brackets.py::test_foo[name]" + ] class TestSorting: - def test_check_equality(self, pytester: Pytester) -> None: - modcol = pytester.getmodulecol( - """ - def test_pass(): pass - def test_fail(): assert 0 - """ - ) - fn1 = pytester.collect_by_name(modcol, "test_pass") + def test_check_equality(self, tmp_path: Path) -> None: + def test_pass(): + pass + + def test_fail(): + assert 0 + + module = build_module("test_check_equality", test_pass, test_fail) + with Ensemble(module, rootpath=tmp_path) as ensemble: + fn1, fn3 = ensemble.collect() + # collect() is idempotent, and pytester.collect_by_name memoized: + # a second lookup of the same name is the very same node. + fn2 = ensemble.collect()[0] + assert fn2 is fn1 + # deliberately widened: comparing a Function to its Module is what + # is under test here + modcol: Node | None = fn1.getparent(pytest.Module) + assert modcol is not None + assert isinstance(fn1, pytest.Function) - fn2 = pytester.collect_by_name(modcol, "test_pass") assert isinstance(fn2, pytest.Function) assert fn1 == fn2 assert fn1 != modcol assert hash(fn1) == hash(fn2) - fn3 = pytester.collect_by_name(modcol, "test_fail") assert isinstance(fn3, pytest.Function) assert not (fn1 == fn3) assert fn1 != fn3 @@ -889,60 +798,63 @@ def test_fail(): assert 0 assert [1, 2, 3] != fn # type: ignore[comparison-overlap] assert modcol != fn - def test_allow_sane_sorting_for_decorators(self, pytester: Pytester) -> None: - modcol = pytester.getmodulecol( - """ - def dec(f): - g = lambda: f(2) - g.place_as = f - return g + def test_allow_sane_sorting_for_decorators(self, tmp_path: Path) -> None: + def dec(f): + def g(): + return f(2) + g.place_as = f # type: ignore[attr-defined] + return g - def test_b(y): - pass - test_b = dec(test_b) + def test_b(y): + pass - def test_a(y): - pass - test_a = dec(test_a) - """ + def test_a(y): + pass + + # the wrappers all carry the same name and line, so they have to be + # given their module names explicitly + module = build_module( + "test_allow_sane_sorting_for_decorators", + test_b=dec(test_b), + test_a=dec(test_a), ) - colitems = modcol.collect() + colitems = collect_tests(module, rootpath=tmp_path) assert len(colitems) == 2 assert [item.name for item in colitems] == ["test_b", "test_a"] - def test_ordered_by_definition_order(self, pytester: Pytester) -> None: - pytester.makepyfile( - """\ - class Test1: - def test_foo(self): pass - def test_bar(self): pass - class Test2: - def test_foo(self): pass - test_bar = Test1.test_bar - class Test3(Test2): - def test_baz(self): pass - """ - ) - result = pytester.runpytest("--collect-only") - result.stdout.fnmatch_lines( - [ - "*Class Test1*", - "*Function test_foo*", - "*Function test_bar*", - "*Class Test2*", - # previously the order was flipped due to Test1.test_bar reference - "*Function test_foo*", - "*Function test_bar*", - "*Class Test3*", - "*Function test_foo*", - "*Function test_bar*", - "*Function test_baz*", - ] + def test_ordered_by_definition_order(self, tmp_path: Path) -> None: + class Test1: + def test_foo(self): ... + + def test_bar(self): ... + + class Test2: + def test_foo(self): ... + + test_bar = Test1.test_bar + + class Test3(Test2): + def test_baz(self): ... + + items = collect_tests( + Test1, Test2, Test3, rootpath=tmp_path, name="test_ordered" ) + assert [item.nodeid for item in items] == [ + "test_ordered.py::Test1::test_foo", + "test_ordered.py::Test1::test_bar", + # previously the order was flipped due to Test1.test_bar reference + "test_ordered.py::Test2::test_foo", + "test_ordered.py::Test2::test_bar", + "test_ordered.py::Test3::test_foo", + "test_ordered.py::Test3::test_bar", + "test_ordered.py::Test3::test_baz", + ] class TestConftestCustomization: + # ensemble: pytest_pycollect_makemodule never fires for an ensemble - + # collect_sources constructs the EnsembleModule directly. def test_pytest_pycollect_module(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -959,6 +871,8 @@ def pytest_pycollect_makemodule(module_path, parent): result = pytester.runpytest("--collect-only") result.stdout.fnmatch_lines(["* None: b = pytester.path.joinpath("a", "b") b.mkdir(parents=True) @@ -987,56 +901,45 @@ def test_hello(): reprec = pytester.inline_run() reprec.assertoutcome(passed=1) - def test_customized_pymakeitem(self, pytester: Pytester) -> None: - b = pytester.path.joinpath("a", "b") - b.mkdir(parents=True) - b.joinpath("conftest.py").write_text( - textwrap.dedent( - """\ - import pytest - @pytest.hookimpl(wrapper=True) - def pytest_pycollect_makeitem(): - result = yield - if result: - for func in result: - func._some123 = "world" - return result - """ - ), - encoding="utf-8", - ) - b.joinpath("test_module.py").write_text( - textwrap.dedent( - """\ - import pytest + def test_customized_pymakeitem(self, tmp_path: Path) -> None: + class MakeItemPlugin: + @pytest.hookimpl(wrapper=True) + def pytest_pycollect_makeitem(self): + result = yield + if result: + for func in result: + func._some123 = "world" + return result - @pytest.fixture() - def obj(request): - return request.node._some123 - def test_hello(obj): - assert obj == "world" - """ - ), - encoding="utf-8", - ) - reprec = pytester.inline_run() - reprec.assertoutcome(passed=1) + @pytest.fixture + def obj(request): + return request.node._some123 - def test_pytest_pycollect_makeitem(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - class MyFunction(pytest.Function): - pass - def pytest_pycollect_makeitem(collector, name, obj): + def test_hello(obj): + assert obj == "world" + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(MakeItemPlugin(),)) + run_tests(obj, test_hello, spec=spec).assert_outcomes(passed=1) + + def test_pytest_pycollect_makeitem(self, tmp_path: Path) -> None: + class MyFunction(pytest.Function): + pass + + class MakeItemPlugin: + def pytest_pycollect_makeitem(self, collector, name, obj): if name == "some": return MyFunction.from_parent(name=name, parent=collector) - """ - ) - pytester.makepyfile("def some(): pass") - result = pytester.runpytest("--collect-only") - result.stdout.fnmatch_lines(["*MyFunction*some*"]) + def some(): + pass + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(MakeItemPlugin(),)) + items = collect_tests(some, spec=spec, name="test_makeitem") + assert [type(item) for item in items] == [MyFunction] + assert items[0].nodeid == "test_makeitem.py::some" + + # ensemble: pytest_collect_file never fires for an ensemble, and this needs a + # subprocess for the sys.meta_path futzing anyway. def test_issue2369_collect_module_fileext(self, pytester: Pytester) -> None: """Ensure we can collect files with weird file extensions as Python modules (#2369)""" @@ -1074,7 +977,7 @@ def test_something(): result = pytester.runpytest_subprocess() result.stdout.fnmatch_lines(["*1 passed*"]) - def test_early_ignored_attributes(self, pytester: Pytester) -> None: + def test_early_ignored_attributes(self, tmp_path: Path) -> None: """Builtin attributes should be ignored early on, even if configuration would otherwise allow them. @@ -1082,27 +985,36 @@ def test_early_ignored_attributes(self, pytester: Pytester) -> None: although it tests PytestCollectionWarning is not raised, while it would have been raised otherwise. """ - pytester.makeini( - """ - [pytest] - python_classes=* - python_functions=* - """ - ) - pytester.makepyfile( - """ - class TestEmpty: - pass - test_empty = TestEmpty() - def test_real(): - pass - """ - ) - items, rec = pytester.inline_genitems() - assert rec.ret == 0 - assert len(items) == 1 + class TestEmpty: + pass + + def test_real(): + pass + module = build_module( + "test_early_ignored_attributes", + TestEmpty, + test_real, + test_empty=TestEmpty(), + ) + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={ + "python_classes": ["*"], + "python_functions": ["*"], + "filterwarnings": ["always"], + }, + ) + record = run_tests(module, spec=spec) + # only test_real is collected, and none of the builtin module/class + # attributes was ever offered up to warn about + record.assert_outcomes(passed=1, warnings=0) + assert list(record.by_test) == ["test_early_ignored_attributes.py::test_real"] + + +# ensemble: the subject is conftest hooks being scoped to their directory; +# ensembles never load conftest files. def test_setup_only_available_in_subdir(pytester: Pytester) -> None: sub1 = pytester.mkpydir("sub1") sub2 = pytester.mkpydir("sub2") @@ -1140,6 +1052,8 @@ def pytest_runtest_teardown(item): result.assert_outcomes(passed=2) +# ensemble: re-collects from a nodeid trail through perform_collect, which +# resolves against the filesystem; an ensemble serves preset collectors. def test_modulecol_roundtrip(pytester: Pytester) -> None: modcol = pytester.getmodulecol("pass", withinit=False) trail = modcol.nodeid @@ -1157,6 +1071,8 @@ def test_skip_simple(self): assert excinfo.traceback[-2].frame.code.name == "test_skip_simple" assert not excinfo.traceback[-2].ishidden(excinfo) + # ensemble: asserts on rendered tracebacks anchored at a conftest file and + # its line numbers, plus --fulltrace, which lives in the terminal plugin. def test_traceback_argsetup(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -1182,6 +1098,7 @@ def hello(request): numentries = out.count("_ _ _ _") # separator for traceback entries assert numentries > 3 + # ensemble: about the traceback of a module *import* error. def test_traceback_error_during_import(self, pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1204,32 +1121,34 @@ def test_traceback_error_during_import(self, pytester: Pytester) -> None: result.stdout.fnmatch_lines([">*asd*", "E*NameError*"]) def test_traceback_filter_error_during_fixture_collection( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Integration test for issue #995.""" - pytester.makepyfile( - """ - import pytest - def fail_me(func): - ns = {} - exec('def w(): raise ValueError("fail me")', ns) - return ns['w'] + def fail_me(func): + ns: dict[str, Any] = {} + exec('def w(): raise ValueError("fail me")', ns) + return ns["w"] - @pytest.fixture(scope='class') - @fail_me - def fail_fixture(): - pass + @pytest.fixture(scope="class") + @fail_me + def fail_fixture(): + pass - def test_failing_fixture(fail_fixture): - pass - """ + def test_failing_fixture(fail_fixture): + pass + + # the fixture carries the name of the generated function it wraps, so + # it has to be given its module name explicitly + module = build_module( + "test_traceback_filter", + test_failing_fixture, + fail_fixture=fail_fixture, ) - result = pytester.runpytest() - assert result.ret != 0 - out = result.stdout.str() - assert "INTERNALERROR>" not in out - result.stdout.fnmatch_lines(["*ValueError: fail me*", "* 1 error in *"]) + record = run_tests(module, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(errors=1) + assert "INTERNALERROR>" not in record.output + record.stdout.fnmatch_lines(["*ValueError: fail me*", "* 1 error in *"]) def test_filter_traceback_generated_code(self) -> None: """Test that filter_traceback() works with the fact that @@ -1255,6 +1174,7 @@ def test_filter_traceback_generated_code(self) -> None: assert isinstance(traceback[-1].path, str) assert not filter_traceback(traceback[-1]) + # ensemble: needs a real importable file that is then deleted from disk. def test_filter_traceback_path_no_longer_valid(self, pytester: Pytester) -> None: """Test that filter_traceback() works with the fact that _pytest._code.code.Code.path attribute might return an str object. @@ -1287,6 +1207,9 @@ def foo(): class TestReportInfo: + # ensemble: reportinfo/location of an ensemble item is anchored at the + # *host* file that defines the source, so these three would either fail or + # have to hard-code this file's line numbers. def test_itemreport_reportinfo(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -1328,26 +1251,23 @@ def test_hello(self): pass @pytest.mark.filterwarnings( "ignore:usage of Generator.Function is deprecated, please use pytest.Function instead" ) - def test_reportinfo_with_nasty_getattr(self, pytester: Pytester) -> None: + def test_reportinfo_with_nasty_getattr(self, tmp_path: Path) -> None: # https://github.com/pytest-dev/pytest/issues/1204 - modcol = pytester.getmodulecol( - """ - # lineno 0 - class TestClass: - def __getattr__(self, name): - return "this is not an int" + class TestClass: + def __getattr__(self, name): + return "this is not an int" - def __class_getattr__(cls, name): - return "this is not an int" + def __class_getattr__(cls, name): + return "this is not an int" - def intest_foo(self): - pass + def intest_foo(self): + pass - def test_bar(self): - pass - """ - ) - classcol = pytester.collect_by_name(modcol, "TestClass") + def test_bar(self): + pass + + (item,) = collect_tests(TestClass, rootpath=tmp_path) + classcol = item.getparent(Class) assert isinstance(classcol, Class) _path, _lineno, _msg = classcol.reportinfo() func = next(iter(classcol.collect())) @@ -1355,6 +1275,10 @@ def test_bar(self): _path, _lineno, _msg = func.reportinfo() +# ensemble: the python_files half of custom discovery only exists for files on +# disk - an EnsembleModule is collected whatever it is called - and this is the +# only test covering it. See test_customized_python_discovery_functions for the +# ported half. def test_customized_python_discovery(pytester: Pytester) -> None: pytester.makeini( """ @@ -1385,105 +1309,95 @@ def check_meth(self): result.stdout.fnmatch_lines(["*2 passed*"]) -def test_customized_python_discovery_functions(pytester: Pytester) -> None: - pytester.makeini( - """ - [pytest] - python_functions=_test - """ - ) - pytester.makepyfile( - """ - def _test_underscore(): - pass - """ +def test_customized_python_discovery_functions(tmp_path: Path) -> None: + def _test_underscore(): + pass + + module = build_module( + "test_customized_python_discovery_functions", _test_underscore ) - result = pytester.runpytest("--collect-only", "-s") - result.stdout.fnmatch_lines(["*_test_underscore*"]) + spec = ConfigSpec(rootpath=tmp_path, inicfg={"python_functions": ["_test"]}) + record = run_tests(module, spec=spec) + record.assert_outcomes(passed=1) + assert list(record.by_test) == [ + "test_customized_python_discovery_functions.py::_test_underscore" + ] - result = pytester.runpytest() - assert result.ret == 0 - result.stdout.fnmatch_lines(["*1 passed*"]) +def test_unorderable_types(tmp_path: Path) -> None: + class TestJoinEmpty: + pass -def test_unorderable_types(pytester: Pytester) -> None: - pytester.makepyfile( - """ - class TestJoinEmpty(object): + def make_test(): + class Test: pass - def make_test(): - class Test(object): - pass - Test.__name__ = "TestFoo" - return Test - TestFoo = make_test() - """ - ) - result = pytester.runpytest() - result.stdout.no_fnmatch_line("*TypeError*") - assert result.ret == ExitCode.NO_TESTS_COLLECTED + Test.__name__ = "TestFoo" + return Test + + TestFoo = make_test() + + # a TypeError while ordering the collected classes would surface as a + # collection error, which collect_tests refuses to swallow + assert collect_tests(TestJoinEmpty, TestFoo, rootpath=tmp_path) == [] -@pytest.mark.filterwarnings("default::pytest.PytestCollectionWarning") -def test_dont_collect_non_function_callable(pytester: Pytester) -> None: +def test_dont_collect_non_function_callable(tmp_path: Path) -> None: """Test for issue https://github.com/pytest-dev/pytest/issues/331 In this case an INTERNALERROR occurred trying to report the failure of a test like this one because pytest failed to get the source lines. """ - pytester.makepyfile( - """ - class Oh(object): - def __call__(self): - pass - - test_a = Oh() - def test_real(): + class Oh: + def __call__(self): pass - """ + + def test_real(): + pass + + # ensemble: the warning's file:line location is host-anchored, so only the + # message itself is asserted on. + module = build_module( + "test_dont_collect_non_function_callable", Oh, test_real, test_a=Oh() ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - "*collected 1 item*", - "*test_dont_collect_non_function_callable.py:2: *cannot collect 'test_a' because it is not a function*", - "*1 passed, 1 warning in *", - ] + record = run_tests( + module, + spec=ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["always"]}), ) + record.assert_outcomes(passed=1, warnings=1) + assert list(record.by_test) == [ + "test_dont_collect_non_function_callable.py::test_real" + ] + assert [str(w.message) for w in record.warnings] == [ + "cannot collect 'test_a' because it is not a function." + ] -def test_class_injection_does_not_break_collection(pytester: Pytester) -> None: +def test_class_injection_does_not_break_collection(tmp_path: Path) -> None: """Tests whether injection during collection time will terminate testing. In this case the error should not occur if the TestClass itself is modified during collection time, and the original method list is still used for collection. """ - pytester.makeconftest( - """ - from test_inject import TestClass - def pytest_generate_tests(metafunc): - TestClass.changed_var = {} - """ - ) - pytester.makepyfile( - test_inject=''' - class TestClass(object): - def test_injection(self): - """Test being parametrized.""" - pass - ''' - ) - result = pytester.runpytest() - assert ( - "RuntimeError: dictionary changed size during iteration" - not in result.stdout.str() - ) - result.stdout.fnmatch_lines(["*1 passed*"]) + + class TestClass: + def test_injection(self): + """Test being parametrized.""" + + class InjectPlugin: + def pytest_generate_tests(self, metafunc): + TestClass.changed_var = {} # type: ignore[attr-defined] + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(InjectPlugin(),)) + # a "dictionary changed size during iteration" RuntimeError would surface + # as a collection error, which collect_tests refuses to swallow + record = run_tests(TestClass, spec=spec) + record.assert_outcomes(passed=1) +# ensemble: about a SyntaxError raised while importing a module from disk. def test_syntax_error_with_non_ascii_chars(pytester: Pytester) -> None: """Fix decoding issue while formatting SyntaxErrors during collection (#578).""" pytester.makepyfile("☃") @@ -1491,6 +1405,8 @@ def test_syntax_error_with_non_ascii_chars(pytester: Pytester) -> None: result.stdout.fnmatch_lines(["*ERROR collecting*", "*SyntaxError*", "*1 error in*"]) +# ensemble: renders a collect error for a module that fails at import, with +# --fulltrace from the terminal plugin and file-anchored line numbers. def test_collect_error_with_fulltrace(pytester: Pytester) -> None: pytester.makepyfile("assert 0") result = pytester.runpytest("--fulltrace") @@ -1510,6 +1426,8 @@ def test_collect_error_with_fulltrace(pytester: Pytester) -> None: ) +# ensemble: duplicate *path arguments* on the command line; an ensemble is +# handed objects, not paths. def test_skip_duplicates_by_default(pytester: Pytester) -> None: """Test for issue https://github.com/pytest-dev/pytest/issues/1609 (#1609) @@ -1531,6 +1449,7 @@ def test_real(): result.stdout.fnmatch_lines(["*collected 1 item*"]) +# ensemble: --keep-duplicates over duplicate path arguments, as above. def test_keep_duplicates(pytester: Pytester) -> None: """Test for issue https://github.com/pytest-dev/pytest/issues/1609 (#1609) @@ -1552,6 +1471,9 @@ def test_real(): result.stdout.fnmatch_lines(["*collected 2 item*"]) +# ensemble: everything below is about collecting real directory trees - +# packages, __init__.py files, path arguments and the resulting hierarchy - +# none of which an ensemble has. def test_package_collection_infinite_recursion(pytester: Pytester) -> None: pytester.copy_example("collect/package_infinite_recursion") result = pytester.runpytest() From b2131f9c92294fc3835459e85a2add56232161fd Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 19:39:08 +0200 Subject: [PATCH 06/30] testing: port test_runner_xunit.py to _pytest.ensemble 10 of 12 tests ported. These are setup/teardown ordering tests, so they lose the module-global smuggling and the getcalls("pytest_runtest_setup") archaeology entirely: events go into a closure list that the assertion reads directly. Every xunit setup failure moves from assertoutcome(failed=N) to assert_outcomes(errors=N) - setup_module/setup_class/setup_method failures are setup-phase - with a note at each site. One test previously asserted no outcome at all and now asserts errors=1, so it can no longer pass vacuously. 2 stay on pytester as canaries, each with a reason: one whose subject is a genuine module global shared between hooks and test bodies, and one covering path-argument collection with real file nodeids. --- testing/test_runner_xunit.py | 444 ++++++++++++++++++++++------------- 1 file changed, 276 insertions(+), 168 deletions(-) diff --git a/testing/test_runner_xunit.py b/testing/test_runner_xunit.py index 75e838a49e8..32fc7a77e6e 100644 --- a/testing/test_runner_xunit.py +++ b/testing/test_runner_xunit.py @@ -3,10 +3,20 @@ from __future__ import annotations +from pathlib import Path + +from _pytest.ensemble import build_module +from _pytest.ensemble import run_tests from _pytest.pytester import Pytester import pytest +# ensemble: kept on pytester as a canary. This is the only test here that +# exercises xunit setup through a real module import, where ``modlevel`` is a +# genuine module global shared between the xunit hooks and the test functions. +# An ensemble source keeps the *host* module's globals, so the ported tests +# below thread state through closures instead - which is exactly the mechanism +# this test is about, so it stays file-based. def test_module_and_function_setup(pytester: Pytester) -> None: reprec = pytester.inline_runsource( """ @@ -40,148 +50,172 @@ def test_module(self): assert rep.passed -def test_module_setup_failure_no_teardown(pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - values = [] - def setup_module(module): - values.append(1) - 0/0 +def test_module_setup_failure_no_teardown(tmp_path: Path) -> None: + values: list[int] = [] - def test_nothing(): - pass + def setup_module(module): + values.append(1) + raise ZeroDivisionError - def teardown_module(module): - values.append(2) - """ + def test_nothing(): ... + + def teardown_module(module): + values.append(2) + + module = build_module( + "test_xunit_module_setup_failure", setup_module, test_nothing, teardown_module ) - reprec.assertoutcome(failed=1) - calls = reprec.getcalls("pytest_runtest_setup") - assert calls[0].item.module.values == [1] + record = run_tests(module, rootpath=tmp_path) + # an xunit setup_module failure errors in the *setup* phase; the old + # assertoutcome(failed=1) counted any failed report regardless of phase. + record.assert_outcomes(errors=1) + assert record["test_nothing"].setup is not None + assert record["test_nothing"].setup.failed + # teardown_module never ran + assert values == [1] -def test_setup_function_failure_no_teardown(pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - modlevel = [] - def setup_function(function): - modlevel.append(1) - 0/0 +def test_setup_function_failure_no_teardown(tmp_path: Path) -> None: + modlevel: list[int] = [] - def teardown_function(module): - modlevel.append(2) + def setup_function(function): + modlevel.append(1) + raise ZeroDivisionError - def test_func(): - pass - """ + def teardown_function(module): + modlevel.append(2) + + def test_func(): ... + + module = build_module( + "test_xunit_function_setup_failure", + setup_function, + teardown_function, + test_func, ) - calls = reprec.getcalls("pytest_runtest_setup") - assert calls[0].item.module.modlevel == [1] + record = run_tests(module, rootpath=tmp_path) + # the original only looked at the recorded module globals; the setup + # failure is an error, and asserting it is strictly more than before + record.assert_outcomes(errors=1) + # teardown_function never ran + assert modlevel == [1] -def test_class_setup(pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - class TestSimpleClassSetup(object): - clslevel = [] - def setup_class(cls): - cls.clslevel.append(23) +def test_class_setup(tmp_path: Path) -> None: + class TestSimpleClassSetup: + clslevel: list[int] = [] - def teardown_class(cls): - cls.clslevel.pop() + def setup_class(cls): + cls.clslevel.append(23) - def test_classlevel(self): - assert self.clslevel[0] == 23 + def teardown_class(cls): + cls.clslevel.pop() - class TestInheritedClassSetupStillWorks(TestSimpleClassSetup): - def test_classlevel_anothertime(self): - assert self.clslevel == [23] + def test_classlevel(self): + assert self.clslevel[0] == 23 - def test_cleanup(): - assert not TestSimpleClassSetup.clslevel - assert not TestInheritedClassSetupStillWorks.clslevel - """ + class TestInheritedClassSetupStillWorks(TestSimpleClassSetup): + def test_classlevel_anothertime(self): + assert self.clslevel == [23] + + def test_cleanup(): + assert not TestSimpleClassSetup.clslevel + assert not TestInheritedClassSetupStillWorks.clslevel + + # collection follows argument order, so test_cleanup runs last + record = run_tests( + TestSimpleClassSetup, + TestInheritedClassSetupStillWorks, + test_cleanup, + rootpath=tmp_path, ) - reprec.assertoutcome(passed=1 + 2 + 1) + record.assert_outcomes(passed=1 + 2 + 1) + assert TestSimpleClassSetup.clslevel == [] -def test_class_setup_failure_no_teardown(pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - class TestSimpleClassSetup(object): - clslevel = [] - def setup_class(cls): - 0/0 +def test_class_setup_failure_no_teardown(tmp_path: Path) -> None: + class TestSimpleClassSetup: + clslevel: list[int] = [] - def teardown_class(cls): - cls.clslevel.append(1) + def setup_class(cls): + raise ZeroDivisionError - def test_classlevel(self): - pass + def teardown_class(cls): + cls.clslevel.append(1) - def test_cleanup(): - assert not TestSimpleClassSetup.clslevel - """ - ) - reprec.assertoutcome(failed=1, passed=1) + def test_classlevel(self): ... + def test_cleanup(): + assert not TestSimpleClassSetup.clslevel -def test_method_setup(pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - class TestSetupMethod(object): - def setup_method(self, meth): - self.methsetup = meth - def teardown_method(self, meth): - del self.methsetup + # collection follows argument order, so test_cleanup runs last + record = run_tests(TestSimpleClassSetup, test_cleanup, rootpath=tmp_path) + # setup_class fails in the *setup* phase, so this is an error, not a + # failure - assertoutcome(failed=1) did not distinguish the two. + record.assert_outcomes(errors=1, passed=1) + # teardown_class never ran + assert TestSimpleClassSetup.clslevel == [] - def test_some(self): - assert self.methsetup == self.test_some - def test_other(self): - assert self.methsetup == self.test_other - """ - ) - reprec.assertoutcome(passed=2) +def test_method_setup(tmp_path: Path) -> None: + class TestSetupMethod: + def setup_method(self, meth): + self.methsetup = meth + def teardown_method(self, meth): + del self.methsetup -def test_method_setup_failure_no_teardown(pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - class TestMethodSetup(object): - clslevel = [] - def setup_method(self, method): - self.clslevel.append(1) - 0/0 + def test_some(self): + assert self.methsetup == self.test_some - def teardown_method(self, method): - self.clslevel.append(2) + def test_other(self): + assert self.methsetup == self.test_other - def test_method(self): - pass + run_tests(TestSetupMethod, rootpath=tmp_path).assert_outcomes(passed=2) - def test_cleanup(): - assert TestMethodSetup.clslevel == [1] - """ - ) - reprec.assertoutcome(failed=1, passed=1) +def test_method_setup_failure_no_teardown(tmp_path: Path) -> None: + class TestMethodSetup: + clslevel: list[int] = [] -def test_method_setup_uses_fresh_instances(pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - class TestSelfState1(object): - memory = [] - def test_hello(self): - self.memory.append(self) + def setup_method(self, method): + self.clslevel.append(1) + raise ZeroDivisionError - def test_afterhello(self): - assert self != self.memory[0] - """ - ) - reprec.assertoutcome(passed=2, failed=0) + def teardown_method(self, method): + self.clslevel.append(2) + + def test_method(self): ... + def test_cleanup(): + assert TestMethodSetup.clslevel == [1] + # collection follows argument order, so test_cleanup runs last + record = run_tests(TestMethodSetup, test_cleanup, rootpath=tmp_path) + # setup_method fails in the *setup* phase, so this is an error, not a + # failure - assertoutcome(failed=1) did not distinguish the two. + record.assert_outcomes(errors=1, passed=1) + # teardown_method never ran + assert TestMethodSetup.clslevel == [1] + + +def test_method_setup_uses_fresh_instances(tmp_path: Path) -> None: + class TestSelfState1: + memory: list[object] = [] + + def test_hello(self): + self.memory.append(self) + + def test_afterhello(self): + assert self != self.memory[0] + + run_tests(TestSelfState1, rootpath=tmp_path).assert_outcomes(passed=2, failed=0) + + +# ensemble: kept on pytester as a canary. Together with the ported +# test_setup_fails_again_on_all_tests below it covers the same behaviour once +# more through a real file collected from a path argument, so path-based +# collection and nodeids stay exercised in this module. def test_setup_that_skips_calledagain(pytester: Pytester) -> None: p = pytester.makepyfile( """ @@ -198,39 +232,50 @@ def test_function2(): reprec.assertoutcome(skipped=2) -def test_setup_fails_again_on_all_tests(pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest - def setup_module(mod): - raise ValueError(42) - def test_function1(): - pass - def test_function2(): - pass - """ +def test_setup_fails_again_on_all_tests(tmp_path: Path) -> None: + def setup_module(mod): + raise ValueError(42) + + def test_function1(): ... + + def test_function2(): ... + + module = build_module( + "test_xunit_setup_fails_again", setup_module, test_function1, test_function2 ) - reprec = pytester.inline_run(p) - reprec.assertoutcome(failed=2) + record = run_tests(module, rootpath=tmp_path) + # the module setup failure is re-raised for every test, in the *setup* + # phase - so two errors, where assertoutcome counted two failed reports. + record.assert_outcomes(errors=2) + assert record["test_function1"].failed + assert record["test_function2"].failed -def test_setup_funcarg_setup_when_outer_scope_fails(pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest - def setup_module(mod): - raise ValueError(42) - @pytest.fixture - def hello(request): - raise ValueError("xyz43") - def test_function1(hello): - pass - def test_function2(hello): - pass - """ +def test_setup_funcarg_setup_when_outer_scope_fails(tmp_path: Path) -> None: + hello_calls: list[object] = [] + + def setup_module(mod): + raise ValueError(42) + + @pytest.fixture + def hello(request): + hello_calls.append(request) + raise ValueError("xyz43") + + def test_function1(hello): ... + + def test_function2(hello): ... + + module = build_module( + "test_xunit_outer_scope_fails", + setup_module, + hello, + test_function1, + test_function2, ) - result = pytester.runpytest(p) - result.stdout.fnmatch_lines( + record = run_tests(module, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(errors=2) + record.stdout.fnmatch_lines( [ "*function1*", "*ValueError*42*", @@ -239,48 +284,111 @@ def test_function2(hello): "*2 errors*", ] ) - result.stdout.no_fnmatch_line("*xyz43*") + record.stdout.no_fnmatch_line("*xyz43*") + # the inner fixture is never even reached + assert hello_calls == [] -@pytest.mark.parametrize("arg", ["", "arg"]) -def test_setup_teardown_function_level_with_optional_argument( - pytester: Pytester, - monkeypatch, - arg: str, -) -> None: - """Parameter to setup/teardown xunit-style functions parameter is now optional (#1728).""" - import sys +def _xunit_hooks_without_argument(trace: list[str]) -> dict[str, object]: + """The xunit functions of the test below, all without the optional argument.""" - trace_setups_teardowns: list[str] = [] - monkeypatch.setattr( - sys, "trace_setups_teardowns", trace_setups_teardowns, raising=False + def setup_module(): + trace.append("setup_module") + + def teardown_module(): + trace.append("teardown_module") + + def setup_function(): + trace.append("setup_function") + + def teardown_function(): + trace.append("teardown_function") + + def test_function_1(): ... + + def test_function_2(): ... + + class Test: + def setup_method(self): + trace.append("setup_method") + + def teardown_method(self): + trace.append("teardown_method") + + def test_method_1(self): ... + + def test_method_2(self): ... + + return dict( + setup_module=setup_module, + teardown_module=teardown_module, + setup_function=setup_function, + teardown_function=teardown_function, + test_function_1=test_function_1, + test_function_2=test_function_2, + Test=Test, ) - p = pytester.makepyfile( - f""" - import pytest - import sys - trace = sys.trace_setups_teardowns.append - def setup_module({arg}): trace('setup_module') - def teardown_module({arg}): trace('teardown_module') +def _xunit_hooks_with_argument(trace: list[str]) -> dict[str, object]: + """The xunit functions of the test below, all taking the optional argument.""" - def setup_function({arg}): trace('setup_function') - def teardown_function({arg}): trace('teardown_function') + def setup_module(arg): + trace.append("setup_module") - def test_function_1(): pass - def test_function_2(): pass + def teardown_module(arg): + trace.append("teardown_module") - class Test(object): - def setup_method(self, {arg}): trace('setup_method') - def teardown_method(self, {arg}): trace('teardown_method') + def setup_function(arg): + trace.append("setup_function") - def test_method_1(self): pass - def test_method_2(self): pass - """ + def teardown_function(arg): + trace.append("teardown_function") + + def test_function_1(): ... + + def test_function_2(): ... + + class Test: + def setup_method(self, arg): + trace.append("setup_method") + + def teardown_method(self, arg): + trace.append("teardown_method") + + def test_method_1(self): ... + + def test_method_2(self): ... + + return dict( + setup_module=setup_module, + teardown_module=teardown_module, + setup_function=setup_function, + teardown_function=teardown_function, + test_function_1=test_function_1, + test_function_2=test_function_2, + Test=Test, + ) + + +@pytest.mark.parametrize( + "hooks", + [_xunit_hooks_without_argument, _xunit_hooks_with_argument], + ids=["", "arg"], +) +def test_setup_teardown_function_level_with_optional_argument( + tmp_path: Path, + hooks, +) -> None: + """Parameter to setup/teardown xunit-style functions parameter is now optional (#1728).""" + trace_setups_teardowns: list[str] = [] + + # the members keep the order they are handed to build_module in, which is + # what decides collection order here + module = build_module( + "test_xunit_optional_argument", **hooks(trace_setups_teardowns) ) - result = pytester.inline_run(p) - result.assertoutcome(passed=4) + run_tests(module, rootpath=tmp_path).assert_outcomes(passed=4) expected = [ "setup_module", From 884d6c1300e07b3300f57de8e2ba7f93ee9c434e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 20:50:41 +0200 Subject: [PATCH 07/30] testing: port test_skipping.py to _pytest.ensemble 64 of 79 pytester-based tests ported - the highest yield in the migration so far, because this file is dominated by outcome assertions. The decisive rule for this file: short-summary lines keyed on the *nodeid* (-rx/-rX/-rE/-ra, FAILED) transfer verbatim once the synthesized module is named deliberately, while "SKIPPED [1] :: reason" renders item.location, which for an ensemble source is the host file. That single line is why the seven skip-reporting tests stayed. Two outcome assertions were hiding something. test_failing_teardown_issue9 asserted "*1 xfail*", which happily matched the "1 xpassed, 1 xfailed" summary and concealed the xpass; it is now assert_outcomes(xpassed=1, xfailed=1). Four tests failing in pytest_runtest_setup are errors, not failures. 15 stay on pytester: host-anchored skip locations (7), module-import-time behaviour (3), custom Items via pytest_collect_file (2), conftest directory scoping (1), --markers (1), and one whose subject is a module-global reachable from a string condition - evaluate_condition reads item.obj.__globals__, which for an ensemble source is this file. --- testing/test_skipping.py | 1650 ++++++++++++++++++-------------------- 1 file changed, 798 insertions(+), 852 deletions(-) diff --git a/testing/test_skipping.py b/testing/test_skipping.py index 9335f6db58c..2b34b700b87 100644 --- a/testing/test_skipping.py +++ b/testing/test_skipping.py @@ -1,9 +1,16 @@ # mypy: allow-untyped-defs from __future__ import annotations +from pathlib import Path import textwrap from _pytest._code import ExceptionInfo +from _pytest.ensemble import build_module +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import Ensemble +from _pytest.ensemble import run_tests +from _pytest.ensemble import RunRecord +from _pytest.outcomes import Exit from _pytest.pytester import Pytester from _pytest.runner import runtestprotocol from _pytest.skipping import evaluate_skip_marks @@ -12,179 +19,174 @@ import pytest +def setup_longrepr(record: RunRecord, name: str) -> str: + """The rendered longrepr of a test's setup report. + + Skip reasons and setup errors live here; the ensemble equivalent of + matching them in the terminal's short summary. + """ + setup = record[name].setup + assert setup is not None + return setup.longreprtext + + class TestEvaluation: - def test_no_marker(self, pytester: Pytester) -> None: - item = pytester.getitem("def test_func(): pass") - skipped = evaluate_skip_marks(item) - assert not skipped + def test_no_marker(self, tmp_path: Path) -> None: + def test_func(): + pass - def test_marked_xfail_no_args(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.xfail - def test_func(): - pass - """ - ) - xfailed = evaluate_xfail_marks(item) - assert xfailed - assert xfailed.reason == "" - assert xfailed.run + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + skipped = evaluate_skip_marks(item) + assert not skipped - def test_marked_skipif_no_args(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.skipif - def test_func(): - pass - """ - ) - skipped = evaluate_skip_marks(item) - assert skipped - assert skipped.reason == "" + def test_marked_xfail_no_args(self, tmp_path: Path) -> None: + @pytest.mark.xfail + def test_func(): + pass - def test_marked_one_arg(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.skipif("hasattr(os, 'sep')") - def test_func(): - pass - """ - ) - skipped = evaluate_skip_marks(item) - assert skipped - assert skipped.reason == "condition: hasattr(os, 'sep')" + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + xfailed = evaluate_xfail_marks(item) + assert xfailed + assert xfailed.reason == "" + assert xfailed.run - def test_marked_one_arg_with_reason(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.skipif("hasattr(os, 'sep')", attr=2, reason="hello world") - def test_func(): - pass - """ + def test_marked_skipif_no_args(self, tmp_path: Path) -> None: + # A bare `skipif` (no condition) is deliberately not a valid call. + @pytest.mark.skipif # type: ignore[arg-type] + def test_func(): + pass + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "" + + def test_marked_one_arg(self, tmp_path: Path) -> None: + @pytest.mark.skipif("hasattr(os, 'sep')") + def test_func(): + pass + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "condition: hasattr(os, 'sep')" + + def test_marked_one_arg_with_reason(self, tmp_path: Path) -> None: + @pytest.mark.skipif( # type: ignore[call-arg] + "hasattr(os, 'sep')", attr=2, reason="hello world" ) - skipped = evaluate_skip_marks(item) - assert skipped - assert skipped.reason == "hello world" - - def test_marked_one_arg_twice(self, pytester: Pytester) -> None: - lines = [ - """@pytest.mark.skipif("not hasattr(os, 'murks')")""", - """@pytest.mark.skipif(condition="hasattr(os, 'murks')")""", - ] - for i in range(2): - item = pytester.getitem( - f""" - import pytest - {lines[i]} - {lines[(i + 1) % 2]} - def test_func(): - pass - """ - ) + def test_func(): + pass + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "hello world" + + def test_marked_one_arg_twice(self, tmp_path: Path) -> None: + # The original generated both stacking orders from the same two + # source lines; here they are spelled out, one function each. + @pytest.mark.skipif("not hasattr(os, 'murks')") + @pytest.mark.skipif(condition="hasattr(os, 'murks')") + def test_func_string_first(): + pass + + @pytest.mark.skipif(condition="hasattr(os, 'murks')") + @pytest.mark.skipif("not hasattr(os, 'murks')") + def test_func_keyword_first(): + pass + + for func in (test_func_string_first, test_func_keyword_first): + with Ensemble(func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "condition: not hasattr(os, 'murks')" + + def test_marked_one_arg_twice2(self, tmp_path: Path) -> None: + @pytest.mark.skipif("hasattr(os, 'murks')") + @pytest.mark.skipif("not hasattr(os, 'murks')") + def test_func(): + pass + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() skipped = evaluate_skip_marks(item) assert skipped assert skipped.reason == "condition: not hasattr(os, 'murks')" - def test_marked_one_arg_twice2(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.skipif("hasattr(os, 'murks')") - @pytest.mark.skipif("not hasattr(os, 'murks')") - def test_func(): - pass - """ - ) - skipped = evaluate_skip_marks(item) - assert skipped - assert skipped.reason == "condition: not hasattr(os, 'murks')" + def test_marked_skipif_with_boolean_without_reason(self, tmp_path: Path) -> None: + @pytest.mark.skipif(False) + def test_func(): + pass - def test_marked_skipif_with_boolean_without_reason( - self, pytester: Pytester - ) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.skipif(False) - def test_func(): - pass - """ - ) - with pytest.raises(pytest.fail.Exception) as excinfo: - evaluate_skip_marks(item) + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + with pytest.raises(pytest.fail.Exception) as excinfo: + evaluate_skip_marks(item) assert excinfo.value.msg is not None assert ( """Error evaluating 'skipif': you need to specify reason=STRING when using booleans as conditions.""" in excinfo.value.msg ) - def test_marked_skipif_with_invalid_boolean(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest + def test_marked_skipif_with_invalid_boolean(self, tmp_path: Path) -> None: + class InvalidBool: + def __bool__(self): + raise TypeError("INVALID") - class InvalidBool: - def __bool__(self): - raise TypeError("INVALID") + @pytest.mark.skipif(InvalidBool(), reason="xxx") # type: ignore[arg-type] + def test_func(): + pass - @pytest.mark.skipif(InvalidBool(), reason="xxx") - def test_func(): - pass - """ - ) - with pytest.raises(pytest.fail.Exception) as excinfo: - evaluate_skip_marks(item) + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + with pytest.raises(pytest.fail.Exception) as excinfo: + evaluate_skip_marks(item) assert excinfo.value.msg is not None assert "Error evaluating 'skipif' condition as a boolean" in excinfo.value.msg assert "INVALID" in excinfo.value.msg - def test_skipif_class(self, pytester: Pytester) -> None: - (item,) = pytester.getitems( - """ - import pytest - class TestClass(object): - pytestmark = pytest.mark.skipif("config._hackxyz") - def test_func(self): - pass - """ - ) - item.config._hackxyz = 3 # type: ignore[attr-defined] - skipped = evaluate_skip_marks(item) - assert skipped - assert skipped.reason == "condition: config._hackxyz" + def test_skipif_class(self, tmp_path: Path) -> None: + class TestClass: + pytestmark = pytest.mark.skipif("config._hackxyz") - def test_skipif_markeval_namespace(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest + def test_func(self): + pass - def pytest_markeval_namespace(): + with Ensemble(TestClass, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + item.config._hackxyz = 3 # type: ignore[attr-defined] + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "condition: config._hackxyz" + + def test_skipif_markeval_namespace(self, tmp_path: Path) -> None: + class ConftestPlugin: + def pytest_markeval_namespace(self): return {"color": "green"} - """ - ) - p = pytester.makepyfile( - """ - import pytest - @pytest.mark.skipif("color == 'green'") - def test_1(): - assert True + @pytest.mark.skipif("color == 'green'") + def test_1(): + assert True - @pytest.mark.skipif("color == 'red'") - def test_2(): - assert True - """ - ) - res = pytester.runpytest(p) - assert res.ret == 0 - res.stdout.fnmatch_lines(["*1 skipped*"]) - res.stdout.fnmatch_lines(["*1 passed*"]) + @pytest.mark.skipif("color == 'red'") + def test_2(): + assert True + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(test_1, test_2, spec=spec) + # Stronger than the two separate "*1 skipped*"/"*1 passed*" matches: + # this also pins that nothing else happened. + record.assert_outcomes(passed=1, skipped=1) + + # ensemble: the point is that a conftest deeper in the tree overrides the + # namespace of one above it, and ensembles have no directory scoping. def test_skipif_markeval_namespace_multiple(self, pytester: Pytester) -> None: """Keys defined by ``pytest_markeval_namespace()`` in nested plugins override top-level ones.""" root = pytester.mkdir("root") @@ -268,112 +270,101 @@ def test_bar(): reprec = pytester.inline_run("-vs", "--capture=no") reprec.assertoutcome(skipped=3) - def test_skipif_markeval_namespace_ValueError(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - - def pytest_markeval_namespace(): + def test_skipif_markeval_namespace_ValueError(self, tmp_path: Path) -> None: + class ConftestPlugin: + def pytest_markeval_namespace(self): return True - """ - ) - p = pytester.makepyfile( - """ - import pytest - @pytest.mark.skipif("color == 'green'") - def test_1(): - assert True - """ - ) - res = pytester.runpytest(p) - assert res.ret == 1 - res.stdout.fnmatch_lines( - [ - "*ValueError: pytest_markeval_namespace() needs to return a dict, got True*" - ] + @pytest.mark.skipif("color == 'green'") + def test_1(): + assert True + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(test_1, spec=spec) + # The ValueError escapes pytest_runtest_setup, so this is an error, + # not a failure - the nonzero exit code of the original did not say + # which. + record.assert_outcomes(errors=1) + assert ( + "ValueError: pytest_markeval_namespace() needs to return a dict, got True" + in setup_longrepr(record, "test_1") ) class TestXFail: @pytest.mark.parametrize("strict", [True, False]) - def test_xfail_simple(self, pytester: Pytester, strict: bool) -> None: - item = pytester.getitem( - f""" - import pytest - @pytest.mark.xfail(strict={strict}) - def test_func(): - assert 0 - """ - ) - reports = runtestprotocol(item, log=False) + def test_xfail_simple(self, tmp_path: Path, strict: bool) -> None: + @pytest.mark.xfail(strict=strict) + def test_func(): + assert 0 + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + reports = runtestprotocol(item, log=False) assert len(reports) == 3 callreport = reports[1] assert callreport.skipped assert callreport.wasxfail == "" - def test_xfail_xpassed(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.xfail(reason="this is an xfail") - def test_func(): - assert 1 - """ - ) - reports = runtestprotocol(item, log=False) + def test_xfail_xpassed(self, tmp_path: Path) -> None: + @pytest.mark.xfail(reason="this is an xfail") + def test_func(): + assert 1 + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + reports = runtestprotocol(item, log=False) assert len(reports) == 3 callreport = reports[1] assert callreport.passed assert callreport.wasxfail == "this is an xfail" - def test_xfail_using_platform(self, pytester: Pytester) -> None: + def test_xfail_using_platform(self, tmp_path: Path) -> None: """Verify that platform can be used with xfail statements.""" - item = pytester.getitem( - """ - import pytest - @pytest.mark.xfail("platform.platform() == platform.platform()") - def test_func(): - assert 0 - """ - ) - reports = runtestprotocol(item, log=False) + + @pytest.mark.xfail("platform.platform() == platform.platform()") + def test_func(): + assert 0 + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + reports = runtestprotocol(item, log=False) assert len(reports) == 3 callreport = reports[1] assert callreport.wasxfail - def test_xfail_xpassed_strict(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.xfail(strict=True, reason="nope") - def test_func(): - assert 1 - """ - ) - reports = runtestprotocol(item, log=False) + def test_xfail_xpassed_strict(self, tmp_path: Path) -> None: + @pytest.mark.xfail(strict=True, reason="nope") + def test_func(): + assert 1 + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + reports = runtestprotocol(item, log=False) assert len(reports) == 3 callreport = reports[1] assert callreport.failed assert str(callreport.longrepr) == "[XPASS(strict)] nope" assert not hasattr(callreport, "wasxfail") - def test_xfail_run_anyway(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail - def test_func(): - assert 0 - def test_func2(): - pytest.xfail("hello") - """ - ) - result = pytester.runpytest("--runxfail") - result.stdout.fnmatch_lines( + def test_xfail_run_anyway(self, tmp_path: Path) -> None: + @pytest.mark.xfail + def test_func(): + assert 0 + + def test_func2(): + pytest.xfail("hello") + + spec = ConfigSpec(rootpath=tmp_path, args=("--runxfail",)) + record = run_tests(test_func, test_func2, spec=spec, capture_output=True) + # --runxfail replaces pytest.xfail with a no-op, so test_func2 passes. + record.assert_outcomes(failed=1, passed=1) + record.stdout.fnmatch_lines( ["*def test_func():*", "*assert 0*", "*1 failed*1 pass*"] ) + # ensemble: the expected `-rs` line names the source file and the line the + # skip mark sits on, which for an ensemble source is this very file. @pytest.mark.parametrize( "test_input,expected", [ @@ -401,71 +392,77 @@ def test_skip_location() -> None: result = pytester.runpytest(*test_input) result.stdout.fnmatch_lines(expected) - def test_xfail_evalfalse_but_fails(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.xfail('False') - def test_func(): - assert 0 - """ - ) - reports = runtestprotocol(item, log=False) + def test_xfail_evalfalse_but_fails(self, tmp_path: Path) -> None: + @pytest.mark.xfail("False") + def test_func(): + assert 0 + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + reports = runtestprotocol(item, log=False) callreport = reports[1] assert callreport.failed assert not hasattr(callreport, "wasxfail") assert "xfail" in callreport.keywords - def test_xfail_not_report_default(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - test_one=""" - import pytest - @pytest.mark.xfail - def test_this(): - assert 0 - """ + def test_xfail_not_report_default(self, tmp_path: Path) -> None: + @pytest.mark.xfail + def test_this(): + assert 0 + + spec = ConfigSpec(rootpath=tmp_path, args=("-v",)) + record = run_tests( + build_module("test_one", test_this=test_this), + spec=spec, + capture_output=True, ) - pytester.runpytest(p, "-v") # result.stdout.fnmatch_lines([ # "*HINT*use*-r*" # ]) + record.assert_outcomes(xfailed=1) + # Without a report char there is no short summary section at all. + record.stdout.no_fnmatch_line("*short test summary info*") - def test_xfail_not_run_xfail_reporting(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - test_one=""" - import pytest - @pytest.mark.xfail(run=False, reason="noway") - def test_this(): - assert 0 - @pytest.mark.xfail("True", run=False) - def test_this_true(): - assert 0 - @pytest.mark.xfail("False", run=False, reason="huh") - def test_this_false(): - assert 1 - """ + def test_xfail_not_run_xfail_reporting(self, tmp_path: Path) -> None: + @pytest.mark.xfail(run=False, reason="noway") + def test_this(): + assert 0 + + @pytest.mark.xfail("True", run=False) + def test_this_true(): + assert 0 + + @pytest.mark.xfail("False", run=False, reason="huh") + def test_this_false(): + assert 1 + + spec = ConfigSpec(rootpath=tmp_path, args=("-rx",)) + record = run_tests( + build_module( + "test_one", + test_this=test_this, + test_this_true=test_this_true, + test_this_false=test_this_false, + ), + spec=spec, + capture_output=True, ) - result = pytester.runpytest(p, "-rx") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*test_one*test_this - *NOTRUN* noway", "*test_one*test_this_true - *NOTRUN* condition: True", "*1 passed*", ] ) + record.assert_outcomes(passed=1, xfailed=2) def test_xfail_not_run_does_not_format_traceback( - self, pytester: Pytester, monkeypatch: pytest.MonkeyPatch + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - item = pytester.getitem( - """ - import pytest + @pytest.mark.xfail(run=False, reason="noway") + def test_func(): + assert 0 - @pytest.mark.xfail(run=False, reason="noway") - def test_func(): - assert 0 - """ - ) getrepr = ExceptionInfo.getrepr styles = [] @@ -473,78 +470,96 @@ def spy_getrepr(self, *args, **kwargs): styles.append(kwargs["style"]) return getrepr(self, *args, **kwargs) - monkeypatch.setattr(ExceptionInfo, "getrepr", spy_getrepr) - - reports = runtestprotocol(item, log=False) + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + monkeypatch.setattr(ExceptionInfo, "getrepr", spy_getrepr) + reports = runtestprotocol(item, log=False) assert reports[0].skipped assert styles == ["value"] - def test_xfail_not_run_no_setup_run(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - test_one=""" - import pytest - @pytest.mark.xfail(run=False, reason="hello") - def test_this(): - assert 0 - def setup_module(mod): - raise ValueError(42) - """ + def test_xfail_not_run_no_setup_run(self, tmp_path: Path) -> None: + @pytest.mark.xfail(run=False, reason="hello") + def test_this(): + assert 0 + + def setup_module(mod): + raise ValueError(42) + + spec = ConfigSpec(rootpath=tmp_path, args=("-rx",)) + record = run_tests( + build_module("test_one", test_this=test_this, setup_module=setup_module), + spec=spec, + capture_output=True, ) - result = pytester.runpytest(p, "-rx") - result.stdout.fnmatch_lines(["*test_one*test_this*NOTRUN*hello", "*1 xfailed*"]) + record.stdout.fnmatch_lines(["*test_one*test_this*NOTRUN*hello", "*1 xfailed*"]) + record.assert_outcomes(xfailed=1) - def test_xfail_xpass(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - test_one=""" - import pytest - @pytest.mark.xfail - def test_that(): - assert 1 - """ + def test_xfail_xpass(self, tmp_path: Path) -> None: + @pytest.mark.xfail + def test_that(): + assert 1 + + spec = ConfigSpec(rootpath=tmp_path, args=("-rX",)) + record = run_tests( + build_module("test_one", test_that=test_that), + spec=spec, + capture_output=True, ) - result = pytester.runpytest(p, "-rX") - result.stdout.fnmatch_lines(["*XPASS*test_that*", "*1 xpassed*"]) - assert result.ret == 0 + record.stdout.fnmatch_lines(["*XPASS*test_that*", "*1 xpassed*"]) + # An xpass is not a failure, which is what `ret == 0` stood for. + record.assert_outcomes(xpassed=1) - def test_xfail_imperative(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest - def test_this(): - pytest.xfail("hello") - """ + def test_xfail_imperative(self, tmp_path: Path) -> None: + def test_this(): + pytest.xfail("hello") + + record = run_tests(test_this, rootpath=tmp_path) + record.assert_outcomes(xfailed=1) + record = run_tests( + test_this, + spec=ConfigSpec(rootpath=tmp_path, args=("-rx",)), + capture_output=True, ) - result = pytester.runpytest(p) - result.stdout.fnmatch_lines(["*1 xfailed*"]) - result = pytester.runpytest(p, "-rx") - result.stdout.fnmatch_lines(["*XFAIL*test_this*hello*"]) - result = pytester.runpytest(p, "--runxfail") - result.stdout.fnmatch_lines(["*1 pass*"]) - - def test_xfail_imperative_in_setup_function(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest - def setup_function(function): - pytest.xfail("hello") + record.stdout.fnmatch_lines(["*XFAIL*test_this*hello*"]) + record = run_tests( + test_this, spec=ConfigSpec(rootpath=tmp_path, args=("--runxfail",)) + ) + record.assert_outcomes(passed=1) - def test_this(): - assert 0 - """ + def test_xfail_imperative_in_setup_function(self, tmp_path: Path) -> None: + def setup_function(function): + pytest.xfail("hello") + + def test_this(): + assert 0 + + module = build_module( + "test_one", setup_function=setup_function, test_this=test_this ) - result = pytester.runpytest(p) - result.stdout.fnmatch_lines(["*1 xfailed*"]) - result = pytester.runpytest(p, "-rx") - result.stdout.fnmatch_lines(["*XFAIL*test_this*hello*"]) - result = pytester.runpytest(p, "--runxfail") - result.stdout.fnmatch_lines( + record = run_tests(module, rootpath=tmp_path) + record.assert_outcomes(xfailed=1) + record = run_tests( + module, + spec=ConfigSpec(rootpath=tmp_path, args=("-rx",)), + capture_output=True, + ) + record.stdout.fnmatch_lines(["*XFAIL*test_this*hello*"]) + record = run_tests( + module, + spec=ConfigSpec(rootpath=tmp_path, args=("--runxfail",)), + capture_output=True, + ) + record.stdout.fnmatch_lines( """ *def test_this* *1 fail* """ ) + record.assert_outcomes(failed=1) + # ensemble: not a test - the name does not start with `test`, so it has + # never been collected; left untouched rather than resurrected here. def xtest_dynamic_xfail_set_during_setup(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ @@ -560,367 +575,309 @@ def test_that(): result = pytester.runpytest(p, "-rxX") result.stdout.fnmatch_lines(["*XFAIL*test_this*", "*XPASS*test_that*"]) - def test_dynamic_xfail_no_run(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest - @pytest.fixture - def arg(request): - request.applymarker(pytest.mark.xfail(run=False)) - def test_this(arg): - assert 0 - """ - ) - result = pytester.runpytest(p, "-rxX") - result.stdout.fnmatch_lines(["*XFAIL*test_this*NOTRUN*"]) + def test_dynamic_xfail_no_run(self, tmp_path: Path) -> None: + @pytest.fixture + def arg(request): + request.applymarker(pytest.mark.xfail(run=False)) - def test_dynamic_xfail_set_during_funcarg_setup(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest - @pytest.fixture - def arg(request): - request.applymarker(pytest.mark.xfail) - def test_this2(arg): - assert 0 - """ - ) - result = pytester.runpytest(p) - result.stdout.fnmatch_lines(["*1 xfailed*"]) + def test_this(arg): + assert 0 + + spec = ConfigSpec(rootpath=tmp_path, args=("-rxX",)) + record = run_tests(arg, test_this, spec=spec, capture_output=True) + record.stdout.fnmatch_lines(["*XFAIL*test_this*NOTRUN*"]) + record.assert_outcomes(xfailed=1) + + def test_dynamic_xfail_set_during_funcarg_setup(self, tmp_path: Path) -> None: + @pytest.fixture + def arg(request): + request.applymarker(pytest.mark.xfail) + + def test_this2(arg): + assert 0 + + record = run_tests(arg, test_this2, rootpath=tmp_path) + record.assert_outcomes(xfailed=1) - def test_dynamic_xfail_set_during_runtest_failed(self, pytester: Pytester) -> None: + def test_dynamic_xfail_set_during_runtest_failed(self, tmp_path: Path) -> None: # Issue #7486. - p = pytester.makepyfile( - """ - import pytest - def test_this(request): - request.node.add_marker(pytest.mark.xfail(reason="xfail")) - assert 0 - """ - ) - result = pytester.runpytest(p) - result.assert_outcomes(xfailed=1) + def test_this(request): + request.node.add_marker(pytest.mark.xfail(reason="xfail")) + assert 0 + + record = run_tests(test_this, rootpath=tmp_path) + record.assert_outcomes(xfailed=1) def test_dynamic_xfail_set_during_runtest_passed_strict( - self, pytester: Pytester + self, tmp_path: Path ) -> None: # Issue #7486. - p = pytester.makepyfile( - """ - import pytest - def test_this(request): - request.node.add_marker(pytest.mark.xfail(reason="xfail", strict=True)) - """ - ) - result = pytester.runpytest(p) - result.assert_outcomes(failed=1) + def test_this(request): + request.node.add_marker(pytest.mark.xfail(reason="xfail", strict=True)) + + record = run_tests(test_this, rootpath=tmp_path) + record.assert_outcomes(failed=1) @pytest.mark.parametrize( - "expected, actual, matchline", + "expected, actual, outcome", [ - ("TypeError", "TypeError", "*1 xfailed*"), - ("(AttributeError, TypeError)", "TypeError", "*1 xfailed*"), - ("TypeError", "IndexError", "*1 failed*"), - ("(AttributeError, TypeError)", "IndexError", "*1 failed*"), + (TypeError, TypeError, "xfailed"), + ((AttributeError, TypeError), TypeError, "xfailed"), + (TypeError, IndexError, "failed"), + ((AttributeError, TypeError), IndexError, "failed"), ], ) - def test_xfail_raises( - self, expected, actual, matchline, pytester: Pytester - ) -> None: - p = pytester.makepyfile( - f""" - import pytest - @pytest.mark.xfail(raises={expected}) - def test_raises(): - raise {actual}() - """ - ) - result = pytester.runpytest(p) - result.stdout.fnmatch_lines([matchline]) + def test_xfail_raises(self, expected, actual, outcome, tmp_path: Path) -> None: + @pytest.mark.xfail(raises=expected) + def test_raises(): + raise actual() + + record = run_tests(test_raises, rootpath=tmp_path) + # Stronger than the single summary-line match of the original. + record.assert_outcomes(**{outcome: 1}) - def test_strict_sanity(self, pytester: Pytester) -> None: + def test_strict_sanity(self, tmp_path: Path) -> None: """Sanity check for xfail(strict=True): a failing test should behave exactly like a normal xfail.""" - p = pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail(reason='unsupported feature', strict=True) - def test_foo(): - assert 0 - """ - ) - result = pytester.runpytest(p, "-rxX") - result.stdout.fnmatch_lines(["*XFAIL*unsupported feature*"]) - assert result.ret == 0 + + @pytest.mark.xfail(reason="unsupported feature", strict=True) + def test_foo(): + assert 0 + + spec = ConfigSpec(rootpath=tmp_path, args=("-rxX",)) + record = run_tests(test_foo, spec=spec, capture_output=True) + record.stdout.fnmatch_lines(["*XFAIL*unsupported feature*"]) + # `ret == 0` stood for "nothing failed". + record.assert_outcomes(xfailed=1) @pytest.mark.parametrize("strict", [True, False]) - def test_strict_xfail(self, pytester: Pytester, strict: bool) -> None: - p = pytester.makepyfile( - f""" - import pytest + def test_strict_xfail(self, tmp_path: Path, strict: bool) -> None: + executed = [] - @pytest.mark.xfail(reason='unsupported feature', strict={strict}) - def test_foo(): - with open('foo_executed', 'w', encoding='utf-8'): - pass # make sure test executes - """ + @pytest.mark.xfail(reason="unsupported feature", strict=strict) + def test_foo(): + executed.append(True) # make sure test executes + + spec = ConfigSpec(rootpath=tmp_path, args=("-rxX",)) + record = run_tests( + build_module("test_strict_xfail", test_foo=test_foo), + spec=spec, + capture_output=True, ) - result = pytester.runpytest(p, "-rxX") if strict: - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( ["*test_foo*", "*XPASS(strict)*unsupported feature*"] ) + record.assert_outcomes(failed=1) else: - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*test_strict_xfail*", "XPASS test_strict_xfail.py::test_foo - unsupported feature", ] ) - assert result.ret == (1 if strict else 0) - assert pytester.path.joinpath("foo_executed").exists() + record.assert_outcomes(xpassed=1) + assert executed == [True] @pytest.mark.parametrize("strict", [True, False]) - def test_strict_xfail_condition(self, pytester: Pytester, strict: bool) -> None: - p = pytester.makepyfile( - f""" - import pytest + def test_strict_xfail_condition(self, tmp_path: Path, strict: bool) -> None: + @pytest.mark.xfail(False, reason="unsupported feature", strict=strict) + def test_foo(): + pass - @pytest.mark.xfail(False, reason='unsupported feature', strict={strict}) - def test_foo(): - pass - """ - ) - result = pytester.runpytest(p, "-rxX") - result.stdout.fnmatch_lines(["*1 passed*"]) - assert result.ret == 0 + # `-rxX` is dropped: it only selects short summary lines, which this + # no longer matches on, and it is a terminal plugin option. + record = run_tests(test_foo, rootpath=tmp_path) + record.assert_outcomes(passed=1) @pytest.mark.parametrize("strict", [True, False]) - def test_xfail_condition_keyword(self, pytester: Pytester, strict: bool) -> None: - p = pytester.makepyfile( - f""" - import pytest + def test_xfail_condition_keyword(self, tmp_path: Path, strict: bool) -> None: + @pytest.mark.xfail(condition=False, reason="unsupported feature", strict=strict) + def test_foo(): + pass - @pytest.mark.xfail(condition=False, reason='unsupported feature', strict={strict}) - def test_foo(): - pass - """ - ) - result = pytester.runpytest(p, "-rxX") - result.stdout.fnmatch_lines(["*1 passed*"]) - assert result.ret == 0 + record = run_tests(test_foo, rootpath=tmp_path) + record.assert_outcomes(passed=1) @pytest.mark.parametrize("strict_val", ["true", "false"]) @pytest.mark.parametrize("option_name", ["strict_xfail", "strict"]) def test_strict_xfail_default_from_file( - self, pytester: Pytester, strict_val: str, option_name: str + self, tmp_path: Path, strict_val: str, option_name: str ) -> None: - pytester.makeini( - f""" - [pytest] - {option_name} = {strict_val} - """ - ) - p = pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail(reason='unsupported feature') - def test_foo(): - pass - """ - ) - result = pytester.runpytest(p, "-rxX") - strict = strict_val == "true" - result.stdout.fnmatch_lines(["*1 failed*" if strict else "*1 xpassed*"]) - assert result.ret == (1 if strict else 0) + @pytest.mark.xfail(reason="unsupported feature") + def test_foo(): + pass - def test_xfail_markeval_namespace(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest + spec = ConfigSpec(rootpath=tmp_path, inicfg={option_name: strict_val}) + record = run_tests(test_foo, spec=spec) + strict = strict_val == "true" + if strict: + record.assert_outcomes(failed=1) + else: + record.assert_outcomes(xpassed=1) - def pytest_markeval_namespace(): + def test_xfail_markeval_namespace(self, tmp_path: Path) -> None: + class ConftestPlugin: + def pytest_markeval_namespace(self): return {"color": "green"} - """ - ) - p = pytester.makepyfile( - """ - import pytest - - @pytest.mark.xfail("color == 'green'") - def test_1(): - assert False - @pytest.mark.xfail("color == 'red'") - def test_2(): - assert False - """ - ) - res = pytester.runpytest(p) - assert res.ret == 1 - res.stdout.fnmatch_lines(["*1 failed*"]) - res.stdout.fnmatch_lines(["*1 xfailed*"]) - - -class TestXFailwithSetupTeardown: - def test_failing_setup_issue9(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def setup_function(func): - assert 0 + @pytest.mark.xfail("color == 'green'") + def test_1(): + assert False - @pytest.mark.xfail - def test_func(): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 xfail*"]) + @pytest.mark.xfail("color == 'red'") + def test_2(): + assert False - def test_failing_teardown_issue9(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def teardown_function(func): - assert 0 + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(test_1, test_2, spec=spec) + record.assert_outcomes(failed=1, xfailed=1) - @pytest.mark.xfail - def test_func(): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 xfail*"]) +class TestXFailwithSetupTeardown: + def test_failing_setup_issue9(self, tmp_path: Path) -> None: + def setup_function(func): + assert 0 -class TestSkip: - def test_skip_class(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip - class TestSomething(object): - def test_foo(self): - pass - def test_bar(self): - pass + @pytest.mark.xfail + def test_func(): + pass - def test_baz(): - pass - """ + record = run_tests( + build_module( + "test_one", setup_function=setup_function, test_func=test_func + ), + rootpath=tmp_path, ) - rec = pytester.inline_run() - rec.assertoutcome(skipped=2, passed=1) + record.assert_outcomes(xfailed=1) - def test_skips_on_false_string(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip('False') - def test_foo(): - pass - """ - ) - rec = pytester.inline_run() - rec.assertoutcome(skipped=1) + def test_failing_teardown_issue9(self, tmp_path: Path) -> None: + def teardown_function(func): + assert 0 - def test_arg_as_reason(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip('testing stuff') - def test_bar(): - pass - """ - ) - result = pytester.runpytest("-rs") - result.stdout.fnmatch_lines(["*testing stuff*", "*1 skipped*"]) + @pytest.mark.xfail + def test_func(): + pass - def test_skip_no_reason(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip - def test_foo(): - pass - """ + record = run_tests( + build_module( + "test_one", teardown_function=teardown_function, test_func=test_func + ), + rootpath=tmp_path, ) - result = pytester.runpytest("-rs") - result.stdout.fnmatch_lines(["*unconditional skip*", "*1 skipped*"]) + # The call phase passes (so: xpassed) and only the teardown turns into + # an xfail; "*1 xfail*" matched the "1 xpassed, 1 xfailed" summary and + # hid the xpass. + record.assert_outcomes(xpassed=1, xfailed=1) - def test_skip_with_reason(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip(reason="for lolz") - def test_bar(): - pass - """ - ) - result = pytester.runpytest("-rs") - result.stdout.fnmatch_lines(["*for lolz*", "*1 skipped*"]) - def test_only_skips_marked_test(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip - def test_foo(): - pass - @pytest.mark.skip(reason="nothing in particular") - def test_bar(): +class TestSkip: + def test_skip_class(self, tmp_path: Path) -> None: + @pytest.mark.skip + class TestSomething: + def test_foo(self): pass - def test_baz(): - assert True - """ - ) - result = pytester.runpytest("-rs") - result.stdout.fnmatch_lines(["*nothing in particular*", "*1 passed*2 skipped*"]) - def test_strict_and_skip(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip - def test_hello(): + def test_bar(self): pass - """ - ) - result = pytester.runpytest("-rs", "--strict-markers") - result.stdout.fnmatch_lines(["*unconditional skip*", "*1 skipped*"]) - def test_wrong_skip_usage(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip(False, reason="I thought this was skipif") - def test_hello(): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - "*TypeError: *__init__() got multiple values for argument 'reason'" - " - maybe you meant pytest.mark.skipif?" - ] + def test_baz(): + pass + + record = run_tests(TestSomething, test_baz, rootpath=tmp_path) + record.assert_outcomes(skipped=2, passed=1) + + def test_skips_on_false_string(self, tmp_path: Path) -> None: + @pytest.mark.skip("False") + def test_foo(): + pass + + record = run_tests(test_foo, rootpath=tmp_path) + record.assert_outcomes(skipped=1) + + def test_arg_as_reason(self, tmp_path: Path) -> None: + @pytest.mark.skip("testing stuff") + def test_bar(): + pass + + record = run_tests(test_bar, rootpath=tmp_path) + record.assert_outcomes(skipped=1) + assert "testing stuff" in setup_longrepr(record, "test_bar") + + def test_skip_no_reason(self, tmp_path: Path) -> None: + @pytest.mark.skip + def test_foo(): + pass + + record = run_tests(test_foo, rootpath=tmp_path) + record.assert_outcomes(skipped=1) + assert "unconditional skip" in setup_longrepr(record, "test_foo") + + def test_skip_with_reason(self, tmp_path: Path) -> None: + @pytest.mark.skip(reason="for lolz") + def test_bar(): + pass + + record = run_tests(test_bar, rootpath=tmp_path) + record.assert_outcomes(skipped=1) + assert "for lolz" in setup_longrepr(record, "test_bar") + + def test_only_skips_marked_test(self, tmp_path: Path) -> None: + @pytest.mark.skip + def test_foo(): + pass + + @pytest.mark.skip(reason="nothing in particular") + def test_bar(): + pass + + def test_baz(): + assert True + + record = run_tests(test_foo, test_bar, test_baz, rootpath=tmp_path) + record.assert_outcomes(passed=1, skipped=2) + assert "nothing in particular" in setup_longrepr(record, "test_bar") + + def test_strict_and_skip(self, tmp_path: Path) -> None: + @pytest.mark.skip + def test_hello(): + pass + + spec = ConfigSpec(rootpath=tmp_path, args=("--strict-markers",)) + record = run_tests(test_hello, spec=spec) + record.assert_outcomes(skipped=1) + assert "unconditional skip" in setup_longrepr(record, "test_hello") + + def test_wrong_skip_usage(self, tmp_path: Path) -> None: + # Deliberately wrong: `skip` takes no condition. + @pytest.mark.skip(False, reason="I thought this was skipif") # type: ignore[call-overload] + def test_hello(): + pass + + record = run_tests(test_hello, rootpath=tmp_path) + # The TypeError escapes pytest_runtest_setup: an error, not a failure. + record.assert_outcomes(errors=1) + longrepr = setup_longrepr(record, "test_hello") + assert "TypeError: " in longrepr + assert ( + "got multiple values for argument 'reason'" + " - maybe you meant pytest.mark.skipif?" in longrepr ) class TestSkipif: - def test_skipif_conditional(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.skipif("hasattr(os, 'sep')") - def test_func(): - pass - """ - ) - x = pytest.raises(pytest.skip.Exception, lambda: pytest_runtest_setup(item)) + def test_skipif_conditional(self, tmp_path: Path) -> None: + @pytest.mark.skipif("hasattr(os, 'sep')") + def test_func(): + pass + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + x = pytest.raises(pytest.skip.Exception, lambda: pytest_runtest_setup(item)) assert x.value.msg == "condition: hasattr(os, 'sep')" + # ensemble: the expected `-rs` line names the file the skipped function + # lives in, which for an ensemble source is this very file. @pytest.mark.parametrize( "params", ["\"hasattr(sys, 'platform')\"", 'True, reason="invalid platform"'] ) @@ -937,18 +894,18 @@ def test_that(): result.stdout.fnmatch_lines(["*SKIP*1*test_foo.py*platform*", "*1 skipped*"]) assert result.ret == 0 - def test_skipif_using_platform(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - @pytest.mark.skipif("platform.platform() == platform.platform()") - def test_func(): - pass - """ - ) - with pytest.raises(pytest.skip.Exception): - pytest_runtest_setup(item) + def test_skipif_using_platform(self, tmp_path: Path) -> None: + @pytest.mark.skipif("platform.platform() == platform.platform()") + def test_func(): + pass + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + with pytest.raises(pytest.skip.Exception): + pytest_runtest_setup(item) + + # ensemble: the `SKIP` half of this expects a `-rs` line naming the file the + # skipped function lives in, which for an ensemble source is this very file. @pytest.mark.parametrize( "marker, msg1, msg2", [("skipif", "SKIP", "skipped"), ("xfail", "XPASS", "xpassed")], @@ -972,40 +929,36 @@ def test_foobar(): assert result.ret == 0 -def test_skip_not_report_default(pytester: Pytester) -> None: - p = pytester.makepyfile( - test_one=""" - import pytest - def test_this(): - pytest.skip("hello") - """ - ) - result = pytester.runpytest(p, "-v") - result.stdout.fnmatch_lines( - [ - # "*HINT*use*-r*", - "*1 skipped*" - ] +def test_skip_not_report_default(tmp_path: Path) -> None: + def test_this(): + pytest.skip("hello") + + spec = ConfigSpec(rootpath=tmp_path, args=("-v",)) + record = run_tests( + build_module("test_one", test_this=test_this), spec=spec, capture_output=True ) + # "*HINT*use*-r*", + record.assert_outcomes(skipped=1) + # Without a report char there is no short summary section at all. + record.stdout.no_fnmatch_line("*short test summary info*") -def test_skipif_class(pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest +def test_skipif_class(tmp_path: Path) -> None: + class TestClass: + pytestmark = pytest.mark.skipif("True") - class TestClass(object): - pytestmark = pytest.mark.skipif("True") - def test_that(self): - assert 0 - def test_though(self): - assert 0 - """ - ) - result = pytester.runpytest(p) - result.stdout.fnmatch_lines(["*2 skipped*"]) + def test_that(self): + assert 0 + + def test_though(self): + assert 0 + + record = run_tests(TestClass, rootpath=tmp_path) + record.assert_outcomes(skipped=2) +# ensemble: asserts the exact `file:line` of every skip, and both the skipping +# helper module and the reported locations are host-anchored here. def test_skipped_reasons_functional(pytester: Pytester) -> None: pytester.makepyfile( test_one=""" @@ -1044,6 +997,8 @@ def doskip(reason): assert result.ret == 0 +# ensemble: the folded `-rs` line names the file the skipped tests live in, +# which for an ensemble source is this very file. def test_skipped_folding(pytester: Pytester) -> None: pytester.makepyfile( test_one=""" @@ -1063,66 +1018,72 @@ def test_method(self): assert result.ret == 0 -def test_reportchars(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def test_1(): - assert 0 - @pytest.mark.xfail - def test_2(): - assert 0 - @pytest.mark.xfail - def test_3(): - pass - def test_4(): - pytest.skip("four") - """ - ) - result = pytester.runpytest("-rfxXs") - result.stdout.fnmatch_lines( +def test_reportchars(tmp_path: Path) -> None: + def test_1(): + assert 0 + + @pytest.mark.xfail + def test_2(): + assert 0 + + @pytest.mark.xfail + def test_3(): + pass + + def test_4(): + pytest.skip("four") + + spec = ConfigSpec(rootpath=tmp_path, args=("-rfxXs",)) + record = run_tests(test_1, test_2, test_3, test_4, spec=spec, capture_output=True) + record.stdout.fnmatch_lines( ["FAIL*test_1*", "XFAIL*test_2*", "XPASS*test_3*", "SKIP*four*"] ) -def test_reportchars_error(pytester: Pytester) -> None: - pytester.makepyfile( - conftest=""" - def pytest_runtest_teardown(): +def test_reportchars_error(tmp_path: Path) -> None: + class ConftestPlugin: + def pytest_runtest_teardown(self): assert 0 - """, - test_simple=""" - def test_foo(): - pass - """, + + def test_foo(): + pass + + spec = ConfigSpec( + rootpath=tmp_path, args=("-rE",), extra_plugins=(ConftestPlugin(),) ) - result = pytester.runpytest("-rE") - result.stdout.fnmatch_lines(["ERROR*test_foo*"]) + record = run_tests( + build_module("test_simple", test_foo=test_foo), spec=spec, capture_output=True + ) + record.stdout.fnmatch_lines(["ERROR*test_foo*"]) -def test_reportchars_all(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def test_1(): - assert 0 - @pytest.mark.xfail - def test_2(): - assert 0 - @pytest.mark.xfail - def test_3(): - pass - def test_4(): - pytest.skip("four") - @pytest.fixture - def fail(): - assert 0 - def test_5(fail): - pass - """ +def test_reportchars_all(tmp_path: Path) -> None: + def test_1(): + assert 0 + + @pytest.mark.xfail + def test_2(): + assert 0 + + @pytest.mark.xfail + def test_3(): + pass + + def test_4(): + pytest.skip("four") + + @pytest.fixture + def fail(): + assert 0 + + def test_5(fail): + pass + + spec = ConfigSpec(rootpath=tmp_path, args=("-ra",)) + record = run_tests( + test_1, test_2, test_3, test_4, fail, test_5, spec=spec, capture_output=True ) - result = pytester.runpytest("-ra") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "SKIP*four*", "XFAIL*test_2*", @@ -1133,37 +1094,38 @@ def test_5(fail): ) -def test_reportchars_all_error(pytester: Pytester) -> None: - pytester.makepyfile( - conftest=""" - def pytest_runtest_teardown(): +def test_reportchars_all_error(tmp_path: Path) -> None: + class ConftestPlugin: + def pytest_runtest_teardown(self): assert 0 - """, - test_simple=""" - def test_foo(): - pass - """, + + def test_foo(): + pass + + spec = ConfigSpec( + rootpath=tmp_path, args=("-ra",), extra_plugins=(ConftestPlugin(),) + ) + record = run_tests( + build_module("test_simple", test_foo=test_foo), spec=spec, capture_output=True ) - result = pytester.runpytest("-ra") - result.stdout.fnmatch_lines(["ERROR*test_foo*"]) + record.stdout.fnmatch_lines(["ERROR*test_foo*"]) -def test_errors_in_xfail_skip_expressions(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skipif("asd") - def test_nameerror(): - pass - @pytest.mark.xfail("syntax error") - def test_syntax(): - pass +def test_errors_in_xfail_skip_expressions(tmp_path: Path) -> None: + @pytest.mark.skipif("asd") + def test_nameerror(): + pass - def test_func(): - pass - """ + @pytest.mark.xfail("syntax error") + def test_syntax(): + pass + + def test_func(): + pass + + record = run_tests( + test_nameerror, test_syntax, test_func, rootpath=tmp_path, capture_output=True ) - result = pytester.runpytest() expected = [ "*ERROR*test_nameerror*", @@ -1182,9 +1144,13 @@ def test_func(): "SyntaxError: invalid syntax", "*1 pass*2 errors*", ] - result.stdout.fnmatch_lines(expected) + record.stdout.fnmatch_lines(expected) + record.assert_outcomes(passed=1, errors=2) +# ensemble: string conditions are eval'd in the *source function's* __globals__, +# which for an ensemble source is this module, not the synthesized one - so the +# module-global `x` this test is about cannot be set up. def test_xfail_skipif_with_globals(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1202,6 +1168,8 @@ def test_boolean(): result.stdout.fnmatch_lines(["*SKIP*x == 3*", "*XFAIL*test_boolean*x == 3*"]) +# ensemble: `--markers` is served from pytest_cmdline_main, which an ensemble +# never runs. def test_default_markers(pytester: Pytester) -> None: result = pytester.runpytest("--markers") result.stdout.fnmatch_lines( @@ -1212,111 +1180,89 @@ def test_default_markers(pytester: Pytester) -> None: ) -def test_xfail_test_setup_exception(pytester: Pytester) -> None: - pytester.makeconftest( - """ - def pytest_runtest_setup(): - 0 / 0 - """ - ) - p = pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail - def test_func(): - assert 0 - """ - ) - result = pytester.runpytest(p) - assert result.ret == 0 - assert "xfailed" in result.stdout.str() - result.stdout.no_fnmatch_line("*xpassed*") +def test_xfail_test_setup_exception(tmp_path: Path) -> None: + class ConftestPlugin: + def pytest_runtest_setup(self): + 0 / 0 # noqa: B018 + @pytest.mark.xfail + def test_func(): + assert 0 -def test_imperativeskip_on_xfail_test(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail - def test_that_fails(): - assert 0 + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + record = run_tests(test_func, spec=spec) + # Stronger than "xfailed in stdout and xpassed not in stdout", and covers + # `ret == 0` too: an xfail is the only thing that happened. + record.assert_outcomes(xfailed=1) - @pytest.mark.skipif("True") - def test_hello(): - pass - """ - ) - pytester.makeconftest( - """ - import pytest - def pytest_runtest_setup(item): + +def test_imperativeskip_on_xfail_test(tmp_path: Path) -> None: + class ConftestPlugin: + def pytest_runtest_setup(self, item): pytest.skip("abc") - """ + + @pytest.mark.xfail + def test_that_fails(): + assert 0 + + @pytest.mark.skipif("True") + def test_hello(): + pass + + spec = ConfigSpec( + rootpath=tmp_path, args=("-rsxX",), extra_plugins=(ConftestPlugin(),) ) - result = pytester.runpytest("-rsxX") - result.stdout.fnmatch_lines_random( + record = run_tests(test_that_fails, test_hello, spec=spec, capture_output=True) + record.stdout.fnmatch_lines_random( """ *SKIP*abc* *SKIP*condition: True* *2 skipped* """ ) + record.assert_outcomes(skipped=2) class TestBooleanCondition: - def test_skipif(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skipif(True, reason="True123") - def test_func1(): - pass - @pytest.mark.skipif(False, reason="True123") - def test_func2(): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - """ - *1 passed*1 skipped* - """ - ) + def test_skipif(self, tmp_path: Path) -> None: + @pytest.mark.skipif(True, reason="True123") + def test_func1(): + pass - def test_skipif_noreason(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skipif(True) - def test_func(): - pass - """ - ) - result = pytester.runpytest("-rs") - result.stdout.fnmatch_lines( - """ - *1 error* - """ - ) + @pytest.mark.skipif(False, reason="True123") + def test_func2(): + pass - def test_xfail(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail(True, reason="True123") - def test_func(): - assert 0 - """ - ) - result = pytester.runpytest("-rxs") - result.stdout.fnmatch_lines( + record = run_tests(test_func1, test_func2, rootpath=tmp_path) + record.assert_outcomes(passed=1, skipped=1) + + def test_skipif_noreason(self, tmp_path: Path) -> None: + @pytest.mark.skipif(True) + def test_func(): + pass + + record = run_tests(test_func, rootpath=tmp_path) + # The missing-reason failure happens in setup, so it is an error. + record.assert_outcomes(errors=1) + + def test_xfail(self, tmp_path: Path) -> None: + @pytest.mark.xfail(True, reason="True123") + def test_func(): + assert 0 + + spec = ConfigSpec(rootpath=tmp_path, args=("-rxs",)) + record = run_tests(test_func, spec=spec, capture_output=True) + record.stdout.fnmatch_lines( """ *XFAIL*True123* *1 xfail* """ ) + record.assert_outcomes(xfailed=1) +# ensemble: the item is produced by a `pytest_collect_file` hook, and an +# ensemble serves a preset collection tree instead of walking files. def test_xfail_item(pytester: Pytester) -> None: # Ensure pytest.xfail works with non-Python Item pytester.makeconftest( @@ -1339,6 +1285,8 @@ def pytest_collect_file(file_path, parent): assert xfailed +# ensemble: the skip has to happen while the module is being imported, and an +# ensemble module is handed over as an object rather than imported. def test_module_level_skip_error(pytester: Pytester) -> None: """Verify that using pytest.skip at module level causes a collection error.""" pytester.makepyfile( @@ -1356,6 +1304,7 @@ def test_func(): ) +# ensemble: same - the skip is raised at module import time. def test_module_level_skip_with_allow_module_level(pytester: Pytester) -> None: """Verify that using pytest.skip(allow_module_level=True) is allowed.""" pytester.makepyfile( @@ -1371,6 +1320,7 @@ def test_func(): result.stdout.fnmatch_lines(["*SKIP*skip_module_level"]) +# ensemble: same - the TypeError is raised at module import time. def test_invalid_skip_keyword_parameter(pytester: Pytester) -> None: """Verify that using pytest.skip() with unknown parameter raises an error.""" pytester.makepyfile( @@ -1386,6 +1336,8 @@ def test_func(): result.stdout.fnmatch_lines(["*TypeError:*['unknown']*"]) +# ensemble: the item is produced by a `pytest_collect_file` hook, and an +# ensemble serves a preset collection tree instead of walking files. def test_mark_xfail_item(pytester: Pytester) -> None: # Ensure pytest.mark.xfail works with non-Python Item pytester.makeconftest( @@ -1413,17 +1365,19 @@ def pytest_collect_file(file_path, parent): assert xfailed -def test_summary_list_after_errors(pytester: Pytester) -> None: +def test_summary_list_after_errors(tmp_path: Path) -> None: """Ensure the list of errors/fails/xfails/skips appears after tracebacks in terminal reporting.""" - pytester.makepyfile( - """ - import pytest - def test_fail(): - assert 0 - """ + + def test_fail(): + assert 0 + + spec = ConfigSpec(rootpath=tmp_path, args=("-ra",)) + record = run_tests( + build_module("test_summary_list_after_errors", test_fail=test_fail), + spec=spec, + capture_output=True, ) - result = pytester.runpytest("-ra") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "=* FAILURES *=", "*= short test summary info =*", @@ -1440,6 +1394,9 @@ def test_importorskip() -> None: pytest.importorskip("doesnotexist") +# ensemble: asserts the skip's `tests/test_1.py:2` location relative to a +# `--rootdir` below it; both the real path layout and the location are +# host-anchored. def test_relpath_rootdir(pytester: Pytester) -> None: pytester.makepyfile( **{ @@ -1457,6 +1414,8 @@ def test_pass(): ) +# ensemble: same - the expected line pins `tests/test_1.py:2` as the reported +# skip location. def test_skip_from_fixture(pytester: Pytester) -> None: pytester.makepyfile( **{ @@ -1478,42 +1437,29 @@ def arg(): ) -def test_skip_using_reason_works_ok(pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest +def test_skip_using_reason_works_ok(tmp_path: Path) -> None: + def test_skipping_reason(): + pytest.skip(reason="skippedreason") - def test_skipping_reason(): - pytest.skip(reason="skippedreason") - """ - ) - result = pytester.runpytest(p) - result.stdout.no_fnmatch_line("*PytestDeprecationWarning*") - result.assert_outcomes(skipped=1) + record = run_tests(test_skipping_reason, rootpath=tmp_path) + # `warnings=0` is what the no_fnmatch_line on PytestDeprecationWarning was + # after, checked against the recorded warnings rather than the rendering. + record.assert_outcomes(skipped=1, warnings=0) -def test_fail_using_reason_works_ok(pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest +def test_fail_using_reason_works_ok(tmp_path: Path) -> None: + def test_failing_reason(): + pytest.fail(reason="failedreason") - def test_failing_reason(): - pytest.fail(reason="failedreason") - """ - ) - result = pytester.runpytest(p) - result.stdout.no_fnmatch_line("*PytestDeprecationWarning*") - result.assert_outcomes(failed=1) + record = run_tests(test_failing_reason, rootpath=tmp_path) + record.assert_outcomes(failed=1, warnings=0) -def test_exit_with_reason_works_ok(pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest +def test_exit_with_reason_works_ok(tmp_path: Path) -> None: + def test_exit_reason_only(): + pytest.exit(reason="foo") - def test_exit_reason_only(): - pytest.exit(reason="foo") - """ - ) - result = pytester.runpytest(p) - result.stdout.fnmatch_lines("*_pytest.outcomes.Exit: foo*") + # An ensemble has no `wrap_session` catching Exit and rendering it, so the + # exception itself is what the rendered `Exit: foo` line stood for. + with pytest.raises(Exit, match=r"^foo$"): + run_tests(test_exit_reason_only, rootpath=tmp_path) From 7e17b14182654052a2b777161e9841228b842c71 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 20:52:54 +0200 Subject: [PATCH 08/30] testing: port test_runner.py to _pytest.ensemble 34 of 53 pytester-based tests ported. Report-shape assertions map onto RunRecord.reports and the ItemRecord phases directly, and ret == 1 has a faithful stand-in in session.testsfailed - which is exactly what wrap_session turns into the exit code. Two tests carried a conftest defining `class Function(pytest.Function)`. Node-class customization by name has not been consulted in years - python.py instantiates Function directly - and every assertion that would have noticed was already commented out. Verified empirically with a real conftest on disk before dropping it. 19 stay on pytester: import-time behaviour of a real module (7), wrap_session/main() (4), capture (2), session bail-out mid-run (2), custom Collector via pytest_collect_file (1), module-level hooks (1), subprocess (1), real stream encoding (1). The two session bail-out tests are worth noting: they look like plain assert_outcomes ports, but run_items has no shouldstop handling, so a port would have quietly asserted a fully-run session instead. --- testing/test_runner.py | 852 ++++++++++++++++++++++------------------- 1 file changed, 448 insertions(+), 404 deletions(-) diff --git a/testing/test_runner.py b/testing/test_runner.py index 3cf6be69de9..aa4570e2850 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -1,6 +1,7 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Callable from functools import partial import inspect import os @@ -15,6 +16,12 @@ from _pytest._code import ExceptionInfo from _pytest._code.code import ExceptionChainRepr from _pytest.config import ExitCode +from _pytest.ensemble import build_module +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import Ensemble +from _pytest.ensemble import EnsembleModule +from _pytest.ensemble import run_tests +from _pytest.ensemble import Source from _pytest.monkeypatch import MonkeyPatch from _pytest.outcomes import OutcomeException from _pytest.pytester import Pytester @@ -25,40 +32,67 @@ from exceptiongroup import ExceptionGroup +#: A runner for the runtest protocol of a single item, as the test classes +#: below hand out from ``getrunner``. +ProtocolRunner = Callable[[pytest.Item], list[reports.TestReport]] + + +def runitem( + getrunner: Callable[[], ProtocolRunner], + tmp_path: Path, + *sources: Source, +) -> list[reports.TestReport]: + """Collect exactly one item from in-memory sources and run it through the + protocol runner the calling test class provides. + + The ensemble replacement for :meth:`Pytester.runitem`, which does the same + thing with a file on disk and looks the runner up on the calling instance. + """ + with Ensemble(*sources, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + return getrunner()(item) + + class TestSetupState: - def test_setup(self, pytester: Pytester) -> None: - item = pytester.getitem("def test_func(): pass") - ss = item.session._setupstate - values = [1] - ss.setup(item) - ss.addfinalizer(values.pop, item) - assert values - ss.teardown_exact(None) - assert not values - - def test_teardown_exact_stack_empty(self, pytester: Pytester) -> None: - item = pytester.getitem("def test_func(): pass") - ss = item.session._setupstate - ss.setup(item) - ss.teardown_exact(None) - ss.teardown_exact(None) - ss.teardown_exact(None) - - def test_setup_fails_and_failure_is_cached(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - def setup_module(mod): - raise ValueError(42) - def test_func(): pass - """ - ) - ss = item.session._setupstate - with pytest.raises(ValueError): + def test_setup(self, tmp_path: Path) -> None: + def test_func(): ... + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + ss = item.session._setupstate + values = [1] ss.setup(item) - with pytest.raises(ValueError): + ss.addfinalizer(values.pop, item) + assert values + ss.teardown_exact(None) + assert not values + + def test_teardown_exact_stack_empty(self, tmp_path: Path) -> None: + def test_func(): ... + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + ss = item.session._setupstate ss.setup(item) + ss.teardown_exact(None) + ss.teardown_exact(None) + ss.teardown_exact(None) - def test_teardown_multiple_one_fails(self, pytester: Pytester) -> None: + def test_setup_fails_and_failure_is_cached(self, tmp_path: Path) -> None: + def setup_module(mod): + raise ValueError(42) + + def test_func(): ... + + with Ensemble(setup_module, test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + ss = item.session._setupstate + with pytest.raises(ValueError): + ss.setup(item) + with pytest.raises(ValueError): + ss.setup(item) + + def test_teardown_multiple_one_fails(self, tmp_path: Path) -> None: r = [] def fin1(): @@ -70,39 +104,46 @@ def fin2(): def fin3(): r.append("fin3") - item = pytester.getitem("def test_func(): pass") - ss = item.session._setupstate - ss.setup(item) - ss.addfinalizer(fin1, item) - ss.addfinalizer(fin2, item) - ss.addfinalizer(fin3, item) - with pytest.raises(Exception) as err: - ss.teardown_exact(None) - assert err.value.args == ("oops",) - assert r == ["fin3", "fin1"] + def test_func(): ... - def test_teardown_multiple_fail(self, pytester: Pytester) -> None: + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + ss = item.session._setupstate + ss.setup(item) + ss.addfinalizer(fin1, item) + ss.addfinalizer(fin2, item) + ss.addfinalizer(fin3, item) + with pytest.raises(Exception) as err: + ss.teardown_exact(None) + assert err.value.args == ("oops",) + assert r == ["fin3", "fin1"] + + def test_teardown_multiple_fail(self, tmp_path: Path) -> None: def fin1(): raise Exception("oops1") def fin2(): raise Exception("oops2") - item = pytester.getitem("def test_func(): pass") - ss = item.session._setupstate - ss.setup(item) - ss.addfinalizer(fin1, item) - ss.addfinalizer(fin2, item) - with pytest.raises(ExceptionGroup) as err: - ss.teardown_exact(None) - - # Note that finalizers are run LIFO, but because FIFO is more intuitive for - # users we reverse the order of messages, and see the error from fin1 first. - err1, err2 = err.value.exceptions - assert err1.args == ("oops1",) - assert err2.args == ("oops2",) + def test_func(): ... - def test_teardown_multiple_scopes_one_fails(self, pytester: Pytester) -> None: + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + ss = item.session._setupstate + ss.setup(item) + ss.addfinalizer(fin1, item) + ss.addfinalizer(fin2, item) + with pytest.raises(ExceptionGroup) as err: + ss.teardown_exact(None) + + # Note that finalizers are run LIFO, but because FIFO is more intuitive + # for users we reverse the order of messages, and see the error from + # fin1 first. + err1, err2 = err.value.exceptions + assert err1.args == ("oops1",) + assert err2.args == ("oops2",) + + def test_teardown_multiple_scopes_one_fails(self, tmp_path: Path) -> None: module_teardown = [] def fin_func(): @@ -111,35 +152,51 @@ def fin_func(): def fin_module(): module_teardown.append("fin_module") - item = pytester.getitem("def test_func(): pass") - mod = item.listchain()[-2] - ss = item.session._setupstate - ss.setup(item) - ss.addfinalizer(fin_module, mod) - ss.addfinalizer(fin_func, item) - with pytest.raises(Exception, match="oops1"): - ss.teardown_exact(None) - assert module_teardown == ["fin_module"] + def test_func(): ... + + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + mod = item.listchain()[-2] + ss = item.session._setupstate + ss.setup(item) + ss.addfinalizer(fin_module, mod) + ss.addfinalizer(fin_func, item) + with pytest.raises(Exception, match="oops1"): + ss.teardown_exact(None) + assert module_teardown == ["fin_module"] - def test_teardown_multiple_scopes_several_fail(self, pytester) -> None: + def test_teardown_multiple_scopes_several_fail(self, tmp_path: Path) -> None: def raiser(exc): raise exc - item = pytester.getitem("def test_func(): pass") - mod = item.listchain()[-2] - ss = item.session._setupstate - ss.setup(item) - ss.addfinalizer(partial(raiser, KeyError("from module scope")), mod) - ss.addfinalizer(partial(raiser, TypeError("from function scope 1")), item) - ss.addfinalizer(partial(raiser, ValueError("from function scope 2")), item) - - with pytest.raises(ExceptionGroup, match="errors during test teardown") as e: - ss.teardown_exact(None) - mod, func = e.value.exceptions - assert isinstance(mod, KeyError) - assert isinstance(func.exceptions[0], TypeError) - assert isinstance(func.exceptions[1], ValueError) + def test_func(): ... + with Ensemble(test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + mod = item.listchain()[-2] + ss = item.session._setupstate + ss.setup(item) + ss.addfinalizer(partial(raiser, KeyError("from module scope")), mod) + ss.addfinalizer(partial(raiser, TypeError("from function scope 1")), item) + ss.addfinalizer(partial(raiser, ValueError("from function scope 2")), item) + + with pytest.raises( + ExceptionGroup, match="errors during test teardown" + ) as e: + ss.teardown_exact(None) + # renamed from ``mod``/``func``: the module collector above already + # holds ``mod``, and mypy now checks this function because it takes an + # annotated ``tmp_path`` rather than a bare ``pytester`` + mod_exc, func_exc = e.value.exceptions + assert isinstance(mod_exc, KeyError) + assert isinstance(func_exc.exceptions[0], TypeError) + assert isinstance(func_exc.exceptions[1], ValueError) + + # ensemble: the subject is a *collector* whose ``setup()`` raises, which is + # what caches the exception in ``SetupState``. Ensembles only collect + # modules/classes/functions, so there is no way to hand one a custom + # ``pytest.Collector`` (nor to serve one from ``pytest_collect_file``, since + # the session's collect report is preset). def test_cached_exception_doesnt_get_longer(self, pytester: Pytester) -> None: """Regression test for #12204 (the "BTW" case).""" pytester.makepyfile(test="") @@ -179,26 +236,25 @@ def pytest_collect_file(file_path, parent): class BaseFunctionalTests: - def test_passfunction(self, pytester: Pytester) -> None: - reports = pytester.runitem( - """ - def test_func(): - pass - """ - ) + def getrunner(self) -> ProtocolRunner: + """The runtest protocol runner; provided by the concrete subclass.""" + raise NotImplementedError + + def test_passfunction(self, tmp_path: Path) -> None: + def test_func(): ... + + reports = runitem(self.getrunner, tmp_path, test_func) rep = reports[1] assert rep.passed assert not rep.failed assert rep.outcome == "passed" assert not rep.longrepr - def test_failfunction(self, pytester: Pytester) -> None: - reports = pytester.runitem( - """ - def test_func(): - assert 0 - """ - ) + def test_failfunction(self, tmp_path: Path) -> None: + def test_func(): + assert 0 + + reports = runitem(self.getrunner, tmp_path, test_func) rep = reports[1] assert not rep.passed assert not rep.skipped @@ -207,14 +263,11 @@ def test_func(): assert rep.outcome == "failed" # assert isinstance(rep.longrepr, ReprExceptionInfo) - def test_skipfunction(self, pytester: Pytester) -> None: - reports = pytester.runitem( - """ - import pytest - def test_func(): - pytest.skip("hello") - """ - ) + def test_skipfunction(self, tmp_path: Path) -> None: + def test_func(): + pytest.skip("hello") + + reports = runitem(self.getrunner, tmp_path, test_func) rep = reports[1] assert not rep.failed assert not rep.passed @@ -227,16 +280,13 @@ def test_func(): # assert rep.skipped.location.path # assert not rep.skipped.failurerepr - def test_skip_in_setup_function(self, pytester: Pytester) -> None: - reports = pytester.runitem( - """ - import pytest - def setup_function(func): - pytest.skip("hello") - def test_func(): - pass - """ - ) + def test_skip_in_setup_function(self, tmp_path: Path) -> None: + def setup_function(func): + pytest.skip("hello") + + def test_func(): ... + + reports = runitem(self.getrunner, tmp_path, setup_function, test_func) print(reports) rep = reports[0] assert not rep.failed @@ -248,16 +298,13 @@ def test_func(): assert len(reports) == 2 assert reports[1].passed # teardown - def test_failure_in_setup_function(self, pytester: Pytester) -> None: - reports = pytester.runitem( - """ - import pytest - def setup_function(func): - raise ValueError(42) - def test_func(): - pass - """ - ) + def test_failure_in_setup_function(self, tmp_path: Path) -> None: + def setup_function(func): + raise ValueError(42) + + def test_func(): ... + + reports = runitem(self.getrunner, tmp_path, setup_function, test_func) rep = reports[0] assert not rep.skipped assert not rep.passed @@ -265,16 +312,13 @@ def test_func(): assert rep.when == "setup" assert len(reports) == 2 - def test_failure_in_teardown_function(self, pytester: Pytester) -> None: - reports = pytester.runitem( - """ - import pytest - def teardown_function(func): - raise ValueError(42) - def test_func(): - pass - """ - ) + def test_failure_in_teardown_function(self, tmp_path: Path) -> None: + def teardown_function(func): + raise ValueError(42) + + def test_func(): ... + + reports = runitem(self.getrunner, tmp_path, teardown_function, test_func) print(reports) assert len(reports) == 3 rep = reports[2] @@ -285,22 +329,16 @@ def test_func(): # assert rep.longrepr.reprcrash.lineno == 3 # assert rep.longrepr.reprtraceback.reprentries - def test_custom_failure_repr(self, pytester: Pytester) -> None: - pytester.makepyfile( - conftest=""" - import pytest - class Function(pytest.Function): - def repr_failure(self, excinfo): - return "hello" - """ - ) - reports = pytester.runitem( - """ - import pytest - def test_func(): - assert 0 - """ - ) + def test_custom_failure_repr(self, tmp_path: Path) -> None: + # The original wrote a conftest defining ``class Function(pytest.Function)`` + # with a custom ``repr_failure``. Node class customization by name was + # removed long ago - ``python.py`` instantiates ``Function`` directly - so + # that conftest was never consulted, and every assertion that would have + # noticed is commented out below. Nothing is lost by dropping it. + def test_func(): + assert 0 + + reports = runitem(self.getrunner, tmp_path, test_func) rep = reports[1] assert not rep.skipped assert not rep.passed @@ -310,60 +348,73 @@ def test_func(): # assert rep.failed.where.path.basename == "test_func.py" # assert rep.failed.failurerepr == "hello" - def test_teardown_final_returncode(self, pytester: Pytester) -> None: - rec = pytester.inline_runsource( - """ - def test_func(): - pass - def teardown_function(func): - raise ValueError(42) - """ - ) - assert rec.ret == 1 + def test_teardown_final_returncode(self, tmp_path: Path) -> None: + def test_func(): ... - def test_logstart_logfinish_hooks(self, pytester: Pytester) -> None: - rec = pytester.inline_runsource( - """ - import pytest - def test_func(): - pass - """ - ) - reps = rec.getcalls("pytest_runtest_logstart pytest_runtest_logfinish") - assert [x._name for x in reps] == [ + def teardown_function(func): + raise ValueError(42) + + with Ensemble(test_func, teardown_function, rootpath=tmp_path) as ensemble: + record = ensemble.run() + # ``inline_run`` reported ``ret == 1`` here; an ensemble has no + # ``wrap_session`` to turn the session into an exit code, so assert + # on the counter that exit code is computed from. + assert ensemble.session.testsfailed == 1 + # the call passes and the teardown fails, so this is an error + record.assert_outcomes(passed=1, errors=1) + + def test_logstart_logfinish_hooks(self, tmp_path: Path) -> None: + events: list[tuple[str, str, tuple[str, int | None, str]]] = [] + + class LogHooks: + def pytest_runtest_logstart(self, nodeid, location): + events.append(("pytest_runtest_logstart", nodeid, location)) + + def pytest_runtest_logfinish(self, nodeid, location): + events.append(("pytest_runtest_logfinish", nodeid, location)) + + def test_func(): ... + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(LogHooks(),)) + with Ensemble(test_func, spec=spec) as ensemble: + (item,) = ensemble.collect() + ensemble.run() + + assert [name for name, _, _ in events] == [ "pytest_runtest_logstart", "pytest_runtest_logfinish", ] - for rep in reps: - assert rep.nodeid == "test_logstart_logfinish_hooks.py::test_func" - assert rep.location == ("test_logstart_logfinish_hooks.py", 1, "test_func") + for _, nodeid, location in events: + assert nodeid == item.nodeid == "test_ensemble.py::test_func" + # the path and line number are host-anchored - the item's code + # object really lives in this file - so only the name transfers + # verbatim; the rest is asserted against the item it came from. + assert location == item.location + assert location[2] == "test_func" - def test_exact_teardown_issue90(self, pytester: Pytester) -> None: - rec = pytester.inline_runsource( - """ - import pytest + def test_exact_teardown_issue90(self, tmp_path: Path) -> None: + class TestClass: + def test_method(self): ... - class TestClass(object): - def test_method(self): - pass - def teardown_class(cls): - raise Exception() + def teardown_class(cls): + raise Exception() - def test_func(): - import sys - # on python2 exc_info is kept till a function exits - # so we would end up calling test functions while - # sys.exc_info would return the indexerror - # from guessing the lastitem - excinfo = sys.exc_info() - import traceback - assert excinfo[0] is None, \ - traceback.format_exception(*excinfo) - def teardown_function(func): - raise ValueError(42) - """ - ) - reps = rec.getreports("pytest_runtest_logreport") + def test_func(): + import traceback + + # on python2 exc_info is kept till a function exits + # so we would end up calling test functions while + # sys.exc_info would return the indexerror + # from guessing the lastitem + excinfo = sys.exc_info() + assert excinfo[0] is None, traceback.format_exception(*excinfo) + + def teardown_function(func): + raise ValueError(42) + + # collection follows argument order: the class first, then test_func + record = run_tests(TestClass, test_func, teardown_function, rootpath=tmp_path) + reps = record.reports print(reps) for i in range(2): assert reps[i].nodeid.endswith("test_method") @@ -378,21 +429,17 @@ def teardown_function(func): assert reps[5].nodeid.endswith("test_func") assert reps[5].failed - def test_exact_teardown_issue1206(self, pytester: Pytester) -> None: + def test_exact_teardown_issue1206(self, tmp_path: Path) -> None: """Issue shadowing error with wrong number of arguments on teardown_method.""" - rec = pytester.inline_runsource( - """ - import pytest - class TestClass(object): - def teardown_method(self, x, y, z): - pass + class TestClass: + def teardown_method(self, x, y, z): ... - def test_method(self): - assert True - """ - ) - reps = rec.getreports("pytest_runtest_logreport") + def test_method(self): + assert True + + record = run_tests(TestClass, rootpath=tmp_path) + reps = record.reports print(reps) assert len(reps) == 3 # @@ -410,31 +457,26 @@ def test_method(self): longrepr = reps[2].longrepr assert isinstance(longrepr, ExceptionChainRepr) assert longrepr.reprcrash - assert longrepr.reprcrash.message in ( - "TypeError: teardown_method() missing 2 required positional arguments: 'y' and 'z'", - # Python >= 3.10 - "TypeError: TestClass.teardown_method() missing 2 required positional arguments: 'y' and 'z'", + # the qualname python puts in front is host-anchored - the class is + # defined inside this test, so it reads + # ``test_exact_teardown_issue1206..TestClass.teardown_method`` + assert longrepr.reprcrash.message.startswith("TypeError: ") + assert longrepr.reprcrash.message.endswith( + "teardown_method() missing 2 required positional arguments: 'y' and 'z'" ) def test_failure_in_setup_function_ignores_custom_repr( - self, pytester: Pytester + self, tmp_path: Path ) -> None: - pytester.makepyfile( - conftest=""" - import pytest - class Function(pytest.Function): - def repr_failure(self, excinfo): - assert 0 - """ - ) - reports = pytester.runitem( - """ - def setup_function(func): - raise ValueError(42) - def test_func(): - pass - """ - ) + # As in test_custom_failure_repr above, the conftest ``Function`` subclass + # the original defined has not been consulted by the collection machinery + # for a long time, so dropping it changes nothing that was asserted. + def setup_function(func): + raise ValueError(42) + + def test_func(): ... + + reports = runitem(self.getrunner, tmp_path, setup_function, test_func) assert len(reports) == 2 rep = reports[0] print(rep) @@ -446,98 +488,76 @@ def test_func(): # assert rep.outcome.where.path.basename == "test_func.py" # assert isinstance(rep.failed.failurerepr, PythonFailureRepr) - def test_systemexit_does_not_bail_out(self, pytester: Pytester) -> None: + def test_systemexit_does_not_bail_out(self, tmp_path: Path) -> None: + def test_func(): + raise SystemExit(42) + try: - reports = pytester.runitem( - """ - def test_func(): - raise SystemExit(42) - """ - ) + reports = runitem(self.getrunner, tmp_path, test_func) except SystemExit: assert False, "runner did not catch SystemExit" rep = reports[1] assert rep.failed assert rep.when == "call" - def test_exit_propagates(self, pytester: Pytester) -> None: - try: - pytester.runitem( - """ - import pytest - def test_func(): - raise pytest.exit.Exception() - """ - ) - except pytest.exit.Exception: - pass - else: - assert False, "did not raise" + def test_exit_propagates(self, tmp_path: Path) -> None: + def test_func(): + raise pytest.exit.Exception() + + with pytest.raises(pytest.exit.Exception): + runitem(self.getrunner, tmp_path, test_func) class TestExecutionNonForked(BaseFunctionalTests): - def getrunner(self): + def getrunner(self) -> ProtocolRunner: def f(item): return runner.runtestprotocol(item, log=False) return f - def test_keyboardinterrupt_propagates(self, pytester: Pytester) -> None: - try: - pytester.runitem( - """ - def test_func(): - raise KeyboardInterrupt("fake") - """ - ) - except KeyboardInterrupt: - pass - else: - assert False, "did not raise" + def test_keyboardinterrupt_propagates(self, tmp_path: Path) -> None: + def test_func(): + raise KeyboardInterrupt("fake") + + with pytest.raises(KeyboardInterrupt): + runitem(self.getrunner, tmp_path, test_func) def test_keyboardinterrupt_clears_request_and_funcargs( - self, pytester: Pytester + self, tmp_path: Path ) -> None: """Ensure that an item's fixtures are cleared quickly even if exiting early due to a keyboard interrupt (#13626).""" - item = pytester.getitem( - """ - import pytest - @pytest.fixture - def resource(): - return object() + @pytest.fixture + def resource(): + return object() - def test_func(resource): - raise KeyboardInterrupt("fake") - """ - ) - assert isinstance(item, pytest.Function) - assert item._request - assert item.funcargs == {} + def test_func(resource): + raise KeyboardInterrupt("fake") - try: - runner.runtestprotocol(item, log=False) - except KeyboardInterrupt: - pass - else: - assert False, "did not raise" + with Ensemble(resource, test_func, rootpath=tmp_path) as ensemble: + (item,) = ensemble.collect() + assert isinstance(item, pytest.Function) + assert item._request + assert item.funcargs == {} - assert not cast(object, item._request) - assert not item.funcargs + with pytest.raises(KeyboardInterrupt): + runner.runtestprotocol(item, log=False) + + assert not cast(object, item._request) + assert not item.funcargs class TestSessionReports: - def test_collect_result(self, pytester: Pytester) -> None: - col = pytester.getmodulecol( - """ - def test_func1(): - pass - class TestClass(object): - pass - """ - ) - rep = runner.collect_one_node(col) + def test_collect_result(self, tmp_path: Path) -> None: + def test_func1(): ... + + class TestClass: ... + + module = build_module("test_collect_result", test_func1, TestClass) + with Ensemble(rootpath=tmp_path) as ensemble: + col = EnsembleModule.from_parent(ensemble.session, obj=module) + rep = runner.collect_one_node(col) assert not rep.failed assert not rep.skipped assert rep.passed @@ -597,6 +617,10 @@ def raise_assertion(): # then something like the following functional tests makes sense +# ensemble: the subject is a ``pytest_runtest_setup`` hook defined at module +# level in the test file. Module-level hooks are registered by +# ``consider_module`` when the module is imported, and an ensemble module is +# served from memory instead of imported, so its hooks are never registered. @pytest.mark.xfail def test_runtest_in_module_ordering(pytester: Pytester) -> None: p1 = pytester.makepyfile( @@ -656,6 +680,9 @@ def test_pytest_fail() -> None: assert s.startswith("Failed") +# ensemble: ``pytest.exit`` from ``pytest_configure`` is rendered onto stderr by +# ``wrap_session``, which ensembles do not run - the Exit would simply propagate +# out of ``configured()``. def test_pytest_exit_msg(pytester: Pytester) -> None: pytester.makeconftest( """ @@ -679,6 +706,9 @@ def _strip_resource_warnings(lines): ] +# ensemble: about the process return code and the stderr message, both produced +# by ``wrap_session``; inside an ensemble ``pytest.exit`` just propagates as +# ``Exit``. def test_pytest_exit_returncode(pytester: Pytester) -> None: pytester.makepyfile( """\ @@ -710,22 +740,27 @@ def pytest_sessionstart(): assert result.ret == 98 -def test_pytest_fail_notrace_runtest(pytester: Pytester) -> None: +def test_pytest_fail_notrace_runtest(tmp_path: Path) -> None: """Test pytest.fail(..., pytrace=False) does not show tracebacks during test run.""" - pytester.makepyfile( - """ - import pytest - def test_hello(): - pytest.fail("hello", pytrace=False) - def teardown_function(function): - pytest.fail("world", pytrace=False) - """ + + def test_hello(): + pytest.fail("hello", pytrace=False) + + def teardown_function(function): + pytest.fail("world", pytrace=False) + + record = run_tests( + test_hello, teardown_function, rootpath=tmp_path, capture_output=True ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["world", "hello"]) - result.stdout.no_fnmatch_line("*def teardown_function*") + # the call fails and the teardown errors + record.assert_outcomes(failed=1, errors=1) + # errors are summarized before failures, hence "world" before "hello" + record.stdout.fnmatch_lines(["world", "hello"]) + record.stdout.no_fnmatch_line("*def teardown_function*") +# ensemble: the failure happens while the test module is imported, and an +# ensemble module is built in memory rather than imported. def test_pytest_fail_notrace_collection(pytester: Pytester) -> None: """Test pytest.fail(..., pytrace=False) does not show tracebacks during collection.""" pytester.makepyfile( @@ -741,24 +776,23 @@ def some_internal_function(): result.stdout.no_fnmatch_line("*def some_internal_function()*") -def test_pytest_fail_notrace_non_ascii(pytester: Pytester) -> None: +def test_pytest_fail_notrace_non_ascii(tmp_path: Path) -> None: """Fix pytest.fail with pytrace=False with non-ascii characters (#1178). This tests with native and unicode strings containing non-ascii chars. """ - pytester.makepyfile( - """\ - import pytest - def test_hello(): - pytest.fail('oh oh: ☺', pytrace=False) - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*test_hello*", "oh oh: ☺"]) - result.stdout.no_fnmatch_line("*def test_hello*") + def test_hello(): + pytest.fail("oh oh: ☺", pytrace=False) + + record = run_tests(test_hello, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(failed=1) + record.stdout.fnmatch_lines(["*test_hello*", "oh oh: ☺"]) + record.stdout.no_fnmatch_line("*def test_hello*") +# ensemble: entirely about the exit status of a run, which is computed by +# ``wrap_session`` from a session an ensemble does not have. def test_pytest_no_tests_collected_exit_status(pytester: Pytester) -> None: result = pytester.runpytest() result.stdout.fnmatch_lines(["*collected 0 items*"]) @@ -840,6 +874,7 @@ def test_module_not_found_skips_by_default(self) -> None: "TestImportOrSkipExcType_test_module_not_found_skips_without_warning" ) + # ensemble: needs a real importable module on sys.path that raises on import. def test_import_error_is_propagated_by_default(self, pytester: Pytester) -> None: fn = pytester.makepyfile("raise ImportError('some specific problem')") pytester.syspathinsert() @@ -847,6 +882,7 @@ def test_import_error_is_propagated_by_default(self, pytester: Pytester) -> None with pytest.raises(ImportError, match="some specific problem"): pytest.importorskip(fn.stem) + # ensemble: needs a real importable module on sys.path that raises on import. def test_import_error_can_be_captured_explicitly(self, pytester: Pytester) -> None: fn = pytester.makepyfile("raise ImportError('some specific problem')") pytester.syspathinsert() @@ -854,6 +890,7 @@ def test_import_error_can_be_captured_explicitly(self, pytester: Pytester) -> No with pytest.raises(pytest.skip.Exception): pytest.importorskip(fn.stem, exc_type=ImportError) + # ensemble: needs a second, real module whose import raises ImportError. def test_import_error_integration(self, pytester: Pytester) -> None: pytester.makepyfile( """ @@ -887,6 +924,8 @@ def test_importorskip_dev_module(monkeypatch) -> None: assert False, f"spurious skip: {ExceptionInfo.from_current()}" +# ensemble: the skip is raised while the test module is imported, which is what +# turns it into a collection-level skip; an ensemble module is already built. def test_importorskip_module_level(pytester: Pytester) -> None: """`importorskip` must be able to skip entire modules when used at module level.""" pytester.makepyfile( @@ -902,6 +941,7 @@ def test_foo(): result.stdout.fnmatch_lines(["*collected 0 items / 1 skipped*"]) +# ensemble: as above, a module level skip raised during import. def test_importorskip_custom_reason(pytester: Pytester) -> None: """Make sure custom reasons are used.""" pytester.makepyfile( @@ -918,6 +958,7 @@ def test_foo(): result.stdout.fnmatch_lines(["*collected 0 items / 1 skipped*"]) +# ensemble: runs pytest in a subprocess. def test_pytest_cmdline_main(pytester: Pytester) -> None: p = pytester.makepyfile( """ @@ -936,6 +977,9 @@ def test_hello(): assert ret == 0 +# ensemble: the subject is writing a non-ascii longrepr to the *real* output +# stream and its encoding. An ensemble renders into a StringIO, where a +# UnicodeEncodeError can never happen, so the ported test would be vacuous. def test_unicode_in_longrepr(pytester: Pytester) -> None: pytester.makeconftest( """\ @@ -959,34 +1003,37 @@ def test_out(): assert "UnicodeEncodeError" not in result.stderr.str() -def test_failure_in_setup(pytester: Pytester) -> None: - pytester.makepyfile( - """ - def setup_module(): - 0/0 - def test_func(): - pass - """ - ) - result = pytester.runpytest("--tb=line") - result.stdout.no_fnmatch_line("*def setup_module*") +def test_failure_in_setup(tmp_path: Path) -> None: + def setup_module(): + raise ZeroDivisionError + def test_func(): ... -def test_makereport_getsource(pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_foo(): - if False: pass - else: assert False - """ - ) - result = pytester.runpytest() - result.stdout.no_fnmatch_line("*INTERNALERROR*") - result.stdout.fnmatch_lines(["*else: assert False*"]) + # ``--tb`` is a terminal plugin option, so this needs the rendering config + spec = ConfigSpec(rootpath=tmp_path, args=("--tb=line",)) + record = run_tests(setup_module, test_func, spec=spec, capture_output=True) + # the xunit setup failure is a setup phase error + record.assert_outcomes(errors=1) + record.stdout.no_fnmatch_line("*def setup_module*") + + +def test_makereport_getsource(tmp_path: Path) -> None: + # the assertion below matches the rendered source line verbatim, so this + # block is kept off the formatter + # fmt: off + def test_foo(): + if False: pass # type: ignore[unreachable] # noqa: E701 + else: assert False # noqa: E701 + # fmt: on + + record = run_tests(test_foo, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(failed=1) + record.stdout.no_fnmatch_line("*INTERNALERROR*") + record.stdout.fnmatch_lines(["*else: assert False*"]) def test_makereport_getsource_dynamic_code( - pytester: Pytester, monkeypatch: MonkeyPatch + tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: """Test that exception in dynamically generated code doesn't break getting the source line.""" import inspect @@ -1001,21 +1048,18 @@ def findsource(obj): monkeypatch.setattr(inspect, "findsource", findsource) - pytester.makepyfile( - """ - import pytest + @pytest.fixture + def foo(missing): ... - @pytest.fixture - def foo(missing): - pass + def test_fix(foo): + assert False - def test_fix(foo): - assert False - """ - ) - result = pytester.runpytest("-vv") - result.stdout.no_fnmatch_line("*INTERNALERROR*") - result.stdout.fnmatch_lines(["*test_fix*", "*fixture*'missing'*not found*"]) + spec = ConfigSpec(rootpath=tmp_path, args=("-vv",)) + record = run_tests(foo, test_fix, spec=spec, capture_output=True) + # the fixture is missing, so setup errors rather than the call failing + record.assert_outcomes(errors=1) + record.stdout.no_fnmatch_line("*INTERNALERROR*") + record.stdout.fnmatch_lines(["*test_fix*", "*fixture*'missing'*not found*"]) def test_store_except_info_on_error() -> None: @@ -1054,67 +1098,60 @@ def runtest(self): assert not hasattr(sys, "last_traceback") -def test_current_test_env_var(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: +def test_current_test_env_var(tmp_path: Path) -> None: + # the smuggling through a ``sys`` attribute the original needed to get the + # values out of a separately imported module is a plain closure here pytest_current_test_vars: list[tuple[str, str]] = [] - monkeypatch.setattr( - sys, "pytest_current_test_vars", pytest_current_test_vars, raising=False - ) - pytester.makepyfile( - """ - import pytest - import sys - import os - @pytest.fixture - def fix(): - sys.pytest_current_test_vars.append(('setup', os.environ['PYTEST_CURRENT_TEST'])) - yield - sys.pytest_current_test_vars.append(('teardown', os.environ['PYTEST_CURRENT_TEST'])) + @pytest.fixture + def fix(): + pytest_current_test_vars.append(("setup", os.environ["PYTEST_CURRENT_TEST"])) + yield + pytest_current_test_vars.append(("teardown", os.environ["PYTEST_CURRENT_TEST"])) - def test(fix): - sys.pytest_current_test_vars.append(('call', os.environ['PYTEST_CURRENT_TEST'])) - """ - ) - result = pytester.runpytest_inprocess() - assert result.ret == 0 - test_id = "test_current_test_env_var.py::test" + def test(fix): + pytest_current_test_vars.append(("call", os.environ["PYTEST_CURRENT_TEST"])) + + record = run_tests(fix, test, rootpath=tmp_path) + record.assert_outcomes(passed=1) + test_id = "test_ensemble.py::test" assert pytest_current_test_vars == [ ("setup", test_id + " (setup)"), ("call", test_id + " (call)"), ("teardown", test_id + " (teardown)"), ] + # the inner run deletes the variable outright when the item is done, so it + # is gone even though the host run set it for this very test assert "PYTEST_CURRENT_TEST" not in os.environ class TestReportContents: """Test user-level API of ``TestReport`` objects.""" - def getrunner(self): + def getrunner(self) -> ProtocolRunner: return lambda item: runner.runtestprotocol(item, log=False) - def test_longreprtext_pass(self, pytester: Pytester) -> None: - reports = pytester.runitem( - """ - def test_func(): - pass - """ - ) + def test_longreprtext_pass(self, tmp_path: Path) -> None: + def test_func(): ... + + reports = runitem(self.getrunner, tmp_path, test_func) rep = reports[1] assert rep.longreprtext == "" - def test_longreprtext_skip(self, pytester: Pytester) -> None: + def test_longreprtext_skip(self, tmp_path: Path) -> None: """TestReport.longreprtext can handle non-str ``longrepr`` attributes (#7559)""" - reports = pytester.runitem( - """ - import pytest - def test_func(): - pytest.skip() - """ - ) + + def test_func(): + pytest.skip() + + reports = runitem(self.getrunner, tmp_path, test_func) _, call_rep, _ = reports assert isinstance(call_rep.longrepr, tuple) assert "Skipped" in call_rep.longreprtext + # ensemble: the skip is raised while the module is imported, and an ensemble + # module is built in memory instead of imported, so there is no collect + # report to carry it. def test_longreprtext_collect_skip(self, pytester: Pytester) -> None: """CollectReport.longreprtext can handle non-str ``longrepr`` attributes (#7559)""" pytester.makepyfile( @@ -1129,17 +1166,17 @@ def test_longreprtext_collect_skip(self, pytester: Pytester) -> None: assert isinstance(call.report.longrepr, tuple) assert "Skipped" in call.report.longreprtext - def test_longreprtext_failure(self, pytester: Pytester) -> None: - reports = pytester.runitem( - """ - def test_func(): - x = 1 - assert x == 4 - """ - ) + def test_longreprtext_failure(self, tmp_path: Path) -> None: + def test_func(): + x = 1 + assert x == 4 + + reports = runitem(self.getrunner, tmp_path, test_func) rep = reports[1] assert "assert 1 == 4" in rep.longreprtext + # ensemble: the subject is the captured stdout/stderr on the reports, and + # the capture plugin is not loaded inside an ensemble. def test_captured_text(self, pytester: Pytester) -> None: reports = pytester.runitem( """ @@ -1170,6 +1207,8 @@ def test_func(fix): assert call.capstderr == "setup: stderr\ncall: stderr\n" assert teardown.capstderr == "setup: stderr\ncall: stderr\nteardown: stderr\n" + # ensemble: without the capture plugin ``capstdout``/``capstderr`` are empty + # no matter what the test does, so a ported version would assert nothing. def test_no_captured_text(self, pytester: Pytester) -> None: reports = pytester.runitem( """ @@ -1181,14 +1220,11 @@ def test_func(): assert rep.capstdout == "" assert rep.capstderr == "" - def test_longrepr_type(self, pytester: Pytester) -> None: - reports = pytester.runitem( - """ - import pytest - def test_func(): - pytest.fail(pytrace=False) - """ - ) + def test_longrepr_type(self, tmp_path: Path) -> None: + def test_func(): + pytest.fail(pytrace=False) + + reports = runitem(self.getrunner, tmp_path, test_func) rep = reports[1] assert isinstance(rep.longrepr, ExceptionChainRepr) @@ -1208,6 +1244,8 @@ def func() -> None: assert str(excinfo.value) == expected +# ensemble: ``PYTEST_VERSION`` is set and restored around ``main()``, which an +# ensemble never goes through - it builds its config directly. def test_pytest_version_env_var(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: monkeypatch.setenv("PYTEST_VERSION", "old version") pytester.makepyfile( @@ -1225,6 +1263,10 @@ def test(): assert os.environ["PYTEST_VERSION"] == "old version" +# ensemble: the subject is the session bailing out mid-run on ``--maxfail`` and +# tearing higher scoped fixtures down against the last item. ``run_items`` +# drives ``pytest_runtest_protocol`` per item without the ``shouldstop`` / +# ``shouldfail`` handling ``pytest_runtestloop`` does, so nothing bails. def test_teardown_session_failed(pytester: Pytester) -> None: """Test that higher-scoped fixture teardowns run in the context of the last item after the test session bails early due to --maxfail. @@ -1250,6 +1292,8 @@ def test_bar(): pass result.assert_outcomes(failed=1, errors=1) +# ensemble: as above, plus ``--stepwise``, whose plugin (and the cache it needs) +# an ensemble config does not load. def test_teardown_session_stopped(pytester: Pytester) -> None: """Test that higher-scoped fixture teardowns run in the context of the last item after the test session bails early due to --stepwise. From 18946c104f3390fb38c1e588b5cb6d622a312266 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 20:52:55 +0200 Subject: [PATCH 09/30] testing: port test_warnings.py to _pytest.ensemble 32 of 51 pytester-based tests ported, asserting on typed WarningMessage objects instead of scraping the -rw summary block. The expected obstacle did not materialize. Config._catch_configured_warnings prepends `always` filters for (Pending)DeprecationWarning, so they beat the host suite's inherited `filterwarnings = error` - which means the whole TestDeprecationWarningsByDefault class, whose docstring explains that all its runs are in a subprocess precisely so they do not inherit filters, ports with no ini override at all. Six subprocess runs became in-process. Several assertions got stronger rather than merely equivalent: "the warnings summary is absent from stdout" became warnings=0 plus an empty record.warnings, and the collection warning test now asserts it was reported with when="collect". 19 stay on pytester: warning file/line/func or a summary keyed on real paths (8), real importable plugin modules (4), a fresh interpreter (2), Config.parse phases an ensemble skips (2), and three one-offs. --- testing/test_warnings.py | 1089 +++++++++++++++++++++----------------- 1 file changed, 594 insertions(+), 495 deletions(-) diff --git a/testing/test_warnings.py b/testing/test_warnings.py index d27776d561b..f1ce4e0f8e5 100644 --- a/testing/test_warnings.py +++ b/testing/test_warnings.py @@ -1,11 +1,20 @@ # mypy: allow-untyped-defs from __future__ import annotations +import dataclasses import os +from pathlib import Path import sys import warnings from _pytest.config import ExitCode +from _pytest.config import UsageError +from _pytest.ensemble import build_module +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import Ensemble +from _pytest.ensemble import run_tests +from _pytest.ensemble import RunRecord +from _pytest.ensemble import Source from _pytest.fixtures import FixtureRequest from _pytest.pytester import Pytester import pytest @@ -14,6 +23,21 @@ WARNINGS_SUMMARY_HEADER = "warnings summary" +class WarningCollector: + """Ensemble plugin recording the full ``pytest_warning_recorded`` payload. + + ``RunRecord.warnings`` keeps only the ``WarningMessage``; the ``when`` + and ``nodeid`` a warning was reported with are what several of these + tests are about, so they are collected here instead. + """ + + def __init__(self) -> None: + self.collected: list[tuple[str, str, str, tuple[str, int, str] | None]] = [] + + def pytest_warning_recorded(self, warning_message, when, nodeid, location): + self.collected.append((str(warning_message.message), when, nodeid, location)) + + @pytest.fixture def pyfile_with_warnings(pytester: Pytester, request: FixtureRequest) -> str: """Create a test file which calls a function in a module which generates warnings.""" @@ -38,6 +62,9 @@ def foo(): return str(test_file) +# ensemble: the subject is the rendered warnings summary, whose per-warning +# lines quote the file and line the warning was raised at; those are anchored +# in this host file for an ensemble source, so the patterns do not transfer. @pytest.mark.filterwarnings("default::UserWarning", "default::RuntimeWarning") def test_normal_flow(pytester: Pytester, pyfile_with_warnings) -> None: """Check that the warnings section is displayed.""" @@ -55,101 +82,100 @@ def test_normal_flow(pytester: Pytester, pyfile_with_warnings) -> None: ) -@pytest.mark.filterwarnings("always::UserWarning") -def test_setup_teardown_warnings(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import warnings - import pytest +def emit_module_warnings() -> int: + """Stand-in for the helper module imported by ``pyfile_with_warnings``.""" + warnings.warn(UserWarning("user warning")) + warnings.warn(RuntimeWarning("runtime warning")) + return 1 - @pytest.fixture - def fix(): - warnings.warn(UserWarning("warning during setup")) - yield - warnings.warn(UserWarning("warning during teardown")) - def test_func(fix): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - f"*== {WARNINGS_SUMMARY_HEADER} ==*", - "*test_setup_teardown_warnings.py:6: UserWarning: warning during setup", - '*warnings.warn(UserWarning("warning during setup"))', - "*test_setup_teardown_warnings.py:8: UserWarning: warning during teardown", - '*warnings.warn(UserWarning("warning during teardown"))', - "* 1 passed, 2 warnings*", - ] +def test_setup_teardown_warnings(tmp_path: Path) -> None: + @pytest.fixture + def fix(): + warnings.warn(UserWarning("warning during setup")) + yield + warnings.warn(UserWarning("warning during teardown")) + + def test_func(fix): + pass + + record = run_tests( + fix, + test_func, + rootpath=tmp_path, + spec=ConfigSpec(inicfg={"filterwarnings": ["always::UserWarning"]}), ) + # The original matched the two rendered warning lines; the file and line + # they quote are host-anchored, so the messages are asserted directly. + record.assert_outcomes(passed=1, warnings=2) + setup_warning, teardown_warning = record.warnings + assert setup_warning.category is UserWarning + assert str(setup_warning.message) == "warning during setup" + assert teardown_warning.category is UserWarning + assert str(teardown_warning.message) == "warning during teardown" @pytest.mark.parametrize("method", ["cmdline", "ini"]) -def test_as_errors(pytester: Pytester, pyfile_with_warnings, method) -> None: - args = ("-W", "error") if method == "cmdline" else () - if method == "ini": - pytester.makeini( - """ - [pytest] - filterwarnings=error - """ - ) - # Use a subprocess, since changing logging level affects other threads - # (xdist). - result = pytester.runpytest_subprocess(*args, pyfile_with_warnings) - result.stdout.fnmatch_lines( +def test_as_errors(tmp_path: Path, method) -> None: + # The original needed a subprocess because ``-W error`` on a real command + # line changes the process-wide filters; an ensemble's filters live in a + # ``warnings.catch_warnings`` block scoped to its own config. + def test_func(): + assert emit_module_warnings() == 1 + + if method == "cmdline": + spec = ConfigSpec(rootpath=tmp_path, args=("-W", "error")) + else: + spec = ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["error"]}) + record = run_tests(test_func, spec=spec, capture_output=True) + record.assert_outcomes(failed=1) + record.stdout.fnmatch_lines( [ "E UserWarning: user warning", - "as_errors_module.py:3: UserWarning", "* 1 failed in *", ] ) @pytest.mark.parametrize("method", ["cmdline", "ini"]) -def test_ignore(pytester: Pytester, pyfile_with_warnings, method) -> None: - args = ("-W", "ignore") if method == "cmdline" else () - if method == "ini": - pytester.makeini( - """ - [pytest] - filterwarnings= ignore - """ - ) - - result = pytester.runpytest(*args, pyfile_with_warnings) - result.stdout.fnmatch_lines(["* 1 passed in *"]) - assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() - - -@pytest.mark.filterwarnings("always::UserWarning") -def test_unicode(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import warnings - import pytest +def test_ignore(tmp_path: Path, method) -> None: + def test_func(): + assert emit_module_warnings() == 1 + if method == "cmdline": + spec = ConfigSpec(rootpath=tmp_path, args=("-W", "ignore")) + else: + spec = ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["ignore"]}) + record = run_tests(test_func, spec=spec) + # Stronger than the original's "no warnings summary was rendered": not a + # single warning was recorded. + record.assert_outcomes(passed=1, warnings=0) + assert record.warnings == [] - @pytest.fixture - def fix(): - warnings.warn("测试") - yield - def test_func(fix): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - f"*== {WARNINGS_SUMMARY_HEADER} ==*", - "*test_unicode.py:7: UserWarning: \u6d4b\u8bd5*", - "* 1 passed, 1 warning*", - ] +def test_unicode(tmp_path: Path) -> None: + @pytest.fixture + def fix(): + warnings.warn("测试") + yield + + def test_func(fix): + pass + + record = run_tests( + fix, + test_func, + rootpath=tmp_path, + spec=ConfigSpec(inicfg={"filterwarnings": ["always::UserWarning"]}), ) + record.assert_outcomes(passed=1, warnings=1) + (warning,) = record.warnings + assert warning.category is UserWarning + assert str(warning.message) == "测试" +# ensemble: the pre-installed filter is registered by module-level code run +# at import time, and an ensemble module body is never executed. @pytest.mark.skip("issue #13485") def test_works_with_filterwarnings(pytester: Pytester) -> None: """Ensure our warnings capture does not mess with pre-installed filters (#2430).""" @@ -176,103 +202,109 @@ def test_my_warning(self): @pytest.mark.parametrize("default_config", ["ini", "cmdline"]) -def test_filterwarnings_mark(pytester: Pytester, default_config) -> None: +def test_filterwarnings_mark(tmp_path: Path, default_config) -> None: """Test ``filterwarnings`` mark works and takes precedence over command line and ini options.""" - if default_config == "ini": - pytester.makeini( - """ - [pytest] - filterwarnings = always::RuntimeWarning - """ - ) - pytester.makepyfile( - """ - import warnings - import pytest - @pytest.mark.filterwarnings('ignore::RuntimeWarning') - def test_ignore_runtime_warning(): - warnings.warn(RuntimeWarning()) + @pytest.mark.filterwarnings("ignore::RuntimeWarning") + def test_ignore_runtime_warning(): + warnings.warn(RuntimeWarning()) - @pytest.mark.filterwarnings('error') - def test_warning_error(): - warnings.warn(RuntimeWarning()) + @pytest.mark.filterwarnings("error") + def test_warning_error(): + warnings.warn(RuntimeWarning()) - def test_show_warning(): - warnings.warn(RuntimeWarning()) - """ - ) - result = pytester.runpytest( - "-W always::RuntimeWarning" if default_config == "cmdline" else "" + def test_show_warning(): + warnings.warn(RuntimeWarning()) + + if default_config == "ini": + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={"filterwarnings": ["always::RuntimeWarning"]}, + ) + else: + spec = ConfigSpec(rootpath=tmp_path, args=("-W", "always::RuntimeWarning")) + record = run_tests( + test_ignore_runtime_warning, test_warning_error, test_show_warning, spec=spec ) - result.stdout.fnmatch_lines(["*= 1 failed, 2 passed, 1 warning in *"]) + record.assert_outcomes(passed=2, failed=1, warnings=1) + assert record["test_warning_error"].failed + (warning,) = record.warnings + assert warning.category is RuntimeWarning -def test_non_string_warning_argument(pytester: Pytester) -> None: +def test_non_string_warning_argument(tmp_path: Path) -> None: """Non-str argument passed to warning breaks pytest (#2956)""" - pytester.makepyfile( - """\ - import warnings - import pytest - def test(): - warnings.warn(UserWarning(1, 'foo')) - """ + def test(): + warnings.warn(UserWarning(1, "foo")) + + record = run_tests( + test, + rootpath=tmp_path, + spec=ConfigSpec(args=("-W", "always::UserWarning")), ) - result = pytester.runpytest("-W", "always::UserWarning") - result.stdout.fnmatch_lines(["*= 1 passed, 1 warning in *"]) + record.assert_outcomes(passed=1, warnings=1) + (warning,) = record.warnings + assert warning.category is UserWarning + assert isinstance(warning.message, UserWarning) + assert warning.message.args == (1, "foo") -def test_filterwarnings_mark_registration(pytester: Pytester) -> None: +def test_filterwarnings_mark_registration(tmp_path: Path) -> None: """Ensure filterwarnings mark is registered""" - pytester.makepyfile( - """ - import pytest - - @pytest.mark.filterwarnings('error') - def test_func(): - pass - """ - ) - result = pytester.runpytest("--strict-markers") - assert result.ret == 0 + @pytest.mark.filterwarnings("error") + def test_func(): + pass -@pytest.mark.filterwarnings("always::UserWarning") -def test_warning_recorded_hook(pytester: Pytester) -> None: - pytester.makeconftest( - """ - def pytest_configure(config): - config.issue_config_time_warning(UserWarning("config warning"), stacklevel=2) - """ + record = run_tests( + test_func, rootpath=tmp_path, spec=ConfigSpec(args=("--strict-markers",)) ) - pytester.makepyfile( - """ - import pytest, warnings + # ``--strict-markers`` turns an unregistered mark into a collection error, + # so a clean pass is exactly the original's ``ret == 0``. + record.assert_outcomes(passed=1) + assert record.collect_errors == [] - warnings.warn(UserWarning("collect warning")) - @pytest.fixture - def fix(): - warnings.warn(UserWarning("setup warning")) - yield 1 - warnings.warn(UserWarning("teardown warning")) +def test_warning_recorded_hook(tmp_path: Path) -> None: + class ConfigWarner: + """Stands in for the conftest of the original.""" - def test_func(fix): - warnings.warn(UserWarning("call warning")) - assert fix == 1 - """ - ) + def pytest_configure(self, config): + config.issue_config_time_warning( + UserWarning("config warning"), stacklevel=2 + ) - collected = [] + class CollectWarner: + """Stands in for the module-level ``warnings.warn`` of the original; + an ensemble module body is never executed, so the collect-phase + warning is issued from a collection hook instead.""" - class WarningCollector: - def pytest_warning_recorded(self, warning_message, when, nodeid, location): - collected.append((str(warning_message.message), when, nodeid, location)) + def pytest_collection_modifyitems(self): + warnings.warn(UserWarning("collect warning")) - result = pytester.runpytest(plugins=[WarningCollector()]) - result.stdout.fnmatch_lines(["*1 passed*"]) + @pytest.fixture + def fix(): + warnings.warn(UserWarning("setup warning")) + yield 1 + warnings.warn(UserWarning("teardown warning")) + + def test_func(fix): + warnings.warn(UserWarning("call warning")) + assert fix == 1 + + warning_collector = WarningCollector() + record = run_tests( + build_module("test_warning_recorded_hook", fix=fix, test_func=test_func), + spec=ConfigSpec( + rootpath=tmp_path, + inicfg={"filterwarnings": ["always::UserWarning"]}, + extra_plugins=(ConfigWarner(), CollectWarner(), warning_collector), + ), + ) + record.assert_outcomes(passed=1) + collected = warning_collector.collected expected = [ ("config warning", "config", ""), @@ -297,272 +329,288 @@ def pytest_warning_recorded(self, warning_message, when, nodeid, location): assert collected_result[3] is None, str(collected) -@pytest.mark.filterwarnings("always::UserWarning") -def test_collection_warnings(pytester: Pytester) -> None: +def test_collection_warnings(tmp_path: Path) -> None: """Check that we also capture warnings issued during test collection (#3251).""" - pytester.makepyfile( - """ - import warnings - warnings.warn(UserWarning("collection warning")) - - def test_foo(): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - f"*== {WARNINGS_SUMMARY_HEADER} ==*", - " *collection_warnings.py:3: UserWarning: collection warning", - ' warnings.warn(UserWarning("collection warning"))', - "* 1 passed, 1 warning*", - ] + class CollectWarner: + """The original warns from the module body, which runs while the + module is imported for collection; an ensemble module body is never + executed, so the warning is issued from a collection hook.""" + + def pytest_collection_modifyitems(self): + warnings.warn(UserWarning("collection warning")) + + def test_foo(): + pass + + warning_collector = WarningCollector() + record = run_tests( + test_foo, + spec=ConfigSpec( + rootpath=tmp_path, + inicfg={"filterwarnings": ["always::UserWarning"]}, + extra_plugins=(CollectWarner(), warning_collector), + ), ) + record.assert_outcomes(passed=1, warnings=1) + (warning,) = record.warnings + assert warning.category is UserWarning + assert str(warning.message) == "collection warning" + # Stronger than the original: the warning is reported as a collect-phase + # one, not merely rendered somewhere in the summary. + assert warning_collector.collected == [("collection warning", "collect", "", None)] -@pytest.mark.filterwarnings("always::UserWarning") -def test_mark_regex_escape(pytester: Pytester) -> None: +def test_mark_regex_escape(tmp_path: Path) -> None: """@pytest.mark.filterwarnings should not try to escape regex characters (#3936)""" - pytester.makepyfile( - r""" - import pytest, warnings - @pytest.mark.filterwarnings(r"ignore:some \(warning\)") - def test_foo(): - warnings.warn(UserWarning("some (warning)")) - """ + @pytest.mark.filterwarnings(r"ignore:some \(warning\)") + def test_foo(): + warnings.warn(UserWarning("some (warning)")) + + record = run_tests( + test_foo, + rootpath=tmp_path, + spec=ConfigSpec(inicfg={"filterwarnings": ["always::UserWarning"]}), ) - result = pytester.runpytest() - assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + record.assert_outcomes(passed=1, warnings=0) + assert record.warnings == [] @pytest.mark.filterwarnings("default::pytest.PytestWarning") @pytest.mark.parametrize("ignore_pytest_warnings", ["no", "ini", "cmdline"]) -def test_hide_pytest_internal_warnings( - pytester: Pytester, ignore_pytest_warnings -) -> None: +def test_hide_pytest_internal_warnings(tmp_path: Path, ignore_pytest_warnings) -> None: """Make sure we can ignore internal pytest warnings using a warnings filter.""" - pytester.makepyfile( - """ - import pytest - import warnings + def test_bar(): warnings.warn(pytest.PytestWarning("some internal warning")) - def test_bar(): - pass - """ - ) + # As in the original, the "no" case relies on the enclosing + # ``default::pytest.PytestWarning`` mark: the inner run inherits the + # process-global filters of the run driving it. + inicfg: dict[str, object] = {} + args: tuple[str, ...] = () if ignore_pytest_warnings == "ini": - pytester.makeini( - """ - [pytest] - filterwarnings = ignore::pytest.PytestWarning - """ - ) - args = ( - ["-W", "ignore::pytest.PytestWarning"] - if ignore_pytest_warnings == "cmdline" - else [] + inicfg = {"filterwarnings": ["ignore::pytest.PytestWarning"]} + elif ignore_pytest_warnings == "cmdline": + args = ("-W", "ignore::pytest.PytestWarning") + record = run_tests( + test_bar, + spec=ConfigSpec(rootpath=tmp_path, inicfg=inicfg, args=args), ) - result = pytester.runpytest(*args) if ignore_pytest_warnings != "no": - assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + record.assert_outcomes(passed=1, warnings=0) + assert record.warnings == [] else: - result.stdout.fnmatch_lines( - [ - f"*== {WARNINGS_SUMMARY_HEADER} ==*", - "*test_hide_pytest_internal_warnings.py:4: PytestWarning: some internal warning", - "* 1 passed, 1 warning *", - ] - ) + record.assert_outcomes(passed=1, warnings=1) + (warning,) = record.warnings + assert warning.category is pytest.PytestWarning + assert str(warning.message) == "some internal warning" @pytest.mark.parametrize("ignore_on_cmdline", [True, False]) -def test_option_precedence_cmdline_over_ini( - pytester: Pytester, ignore_on_cmdline -) -> None: +def test_option_precedence_cmdline_over_ini(tmp_path: Path, ignore_on_cmdline) -> None: """Filters defined in the command-line should take precedence over filters in config files (#3946).""" - pytester.makeini( - """ - [pytest] - filterwarnings = error::UserWarning - """ - ) - pytester.makepyfile( - """ - import warnings - def test(): - warnings.warn(UserWarning('hello')) - """ + + def test(): + warnings.warn(UserWarning("hello")) + + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={"filterwarnings": ["error::UserWarning"]}, + args=("-W", "ignore") if ignore_on_cmdline else (), ) - args = ["-W", "ignore"] if ignore_on_cmdline else [] - result = pytester.runpytest(*args) + record = run_tests(test, spec=spec) if ignore_on_cmdline: - result.stdout.fnmatch_lines(["* 1 passed in*"]) + record.assert_outcomes(passed=1, warnings=0) else: - result.stdout.fnmatch_lines(["* 1 failed in*"]) + record.assert_outcomes(failed=1) -def test_option_precedence_mark(pytester: Pytester) -> None: +def test_option_precedence_mark(tmp_path: Path) -> None: """Filters defined by marks should always take precedence (#3946).""" - pytester.makeini( - """ - [pytest] - filterwarnings = ignore - """ - ) - pytester.makepyfile( - """ - import pytest, warnings - @pytest.mark.filterwarnings('error') - def test(): - warnings.warn(UserWarning('hello')) - """ + + @pytest.mark.filterwarnings("error") + def test(): + warnings.warn(UserWarning("hello")) + + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={"filterwarnings": ["ignore"]}, + args=("-W", "ignore"), ) - result = pytester.runpytest("-W", "ignore") - result.stdout.fnmatch_lines(["* 1 failed in*"]) + record = run_tests(test, spec=spec) + record.assert_outcomes(failed=1) -def test_accept_unknown_category(pytester: Pytester) -> None: +def test_accept_unknown_category(tmp_path: Path, recwarn) -> None: """Category types that can't be imported don't cause failure (#13732).""" - pytester.makeini( - """ - [pytest] - filterwarnings = - always:Failed to import filter module.*:pytest.PytestConfigWarning - ignore::foobar.Foobar - """ - ) - pytester.makepyfile( - """ - def test(): - pass - """ - ) - result = pytester.runpytest_subprocess("-W", "ignore::bizbaz.Bizbaz") - result.stdout.fnmatch_lines( - [ - f"*== {WARNINGS_SUMMARY_HEADER} ==*", - "*PytestConfigWarning: Failed to import filter module 'foobar': ignore::foobar.Foobar", - "*PytestConfigWarning: Failed to import filter module 'bizbaz': ignore::bizbaz.Bizbaz", - "* 1 passed, * warning*", - ] + + def test(): + pass + + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={ + "filterwarnings": [ + "always:Failed to import filter module.*:pytest.PytestConfigWarning", + "ignore::foobar.Foobar", + ] + }, + args=("-W", "ignore::bizbaz.Bizbaz"), ) + # The filters are (re)applied on entering every warnings-catching block, + # so the same PytestConfigWarning is reported once per block rather than + # exactly once; `recwarn` keeps the config-time ones, which escape an + # ensemble, out of this run's own warnings summary. + record = run_tests(test, spec=spec) + record.assert_outcomes(passed=1) + assert {w.category for w in record.warnings} == {pytest.PytestConfigWarning} + assert {str(w.message) for w in record.warnings} == { + "Failed to import filter module 'foobar': ignore::foobar.Foobar", + "Failed to import filter module 'bizbaz': ignore::bizbaz.Bizbaz", + } class TestDeprecationWarningsByDefault: """ - Note: all pytest runs are executed in a subprocess so we don't inherit warning filters - from pytest's own test suite + Note: the original pytest runs are all executed in a subprocess so we don't + inherit warning filters from pytest's own test suite. An ensemble does + inherit them, but the "always" filters a config installs for + (Pending)DeprecationWarning are prepended to whatever is in force, so the + default still wins over this suite's ``filterwarnings = error``. """ - def create_file(self, pytester: Pytester, mark="") -> None: - pytester.makepyfile( - f""" - import pytest, warnings - - warnings.warn(DeprecationWarning("collection")) + def create_sources(self, mark=None) -> tuple[object, Source]: + """The sources of ``create_file``. - {mark} - def test_foo(): - warnings.warn(PendingDeprecationWarning("test run")) + The module-level ``warnings.warn`` of the original runs while the + module is imported for collection; an ensemble module body is never + executed, so that warning is issued from a collection hook instead. """ - ) + + class CollectWarner: + def pytest_collection_modifyitems(self): + warnings.warn(DeprecationWarning("collection")) + + def test_foo(): + warnings.warn(PendingDeprecationWarning("test run")) + + return CollectWarner(), test_foo if mark is None else mark(test_foo) @pytest.mark.parametrize("customize_filters", [True, False]) - def test_shown_by_default(self, pytester: Pytester, customize_filters) -> None: + def test_shown_by_default(self, tmp_path: Path, customize_filters) -> None: """Show deprecation warnings by default, even if user has customized the warnings filters (#4013).""" - self.create_file(pytester) - if customize_filters: - pytester.makeini( - """ - [pytest] - filterwarnings = - once::UserWarning - """ - ) - result = pytester.runpytest_subprocess() - result.stdout.fnmatch_lines( - [ - f"*== {WARNINGS_SUMMARY_HEADER} ==*", - "*test_shown_by_default.py:3: DeprecationWarning: collection", - "*test_shown_by_default.py:7: PendingDeprecationWarning: test run", - "* 1 passed, 2 warnings*", - ] + collect_warner, test_foo = self.create_sources() + inicfg = {"filterwarnings": ["once::UserWarning"]} if customize_filters else {} + record = run_tests( + test_foo, + spec=ConfigSpec( + rootpath=tmp_path, + inicfg=inicfg, + extra_plugins=(collect_warner,), + ), ) - - def test_hidden_by_ini(self, pytester: Pytester) -> None: - self.create_file(pytester) - pytester.makeini( - """ - [pytest] - filterwarnings = - ignore::DeprecationWarning - ignore::PendingDeprecationWarning - """ + record.assert_outcomes(passed=1, warnings=2) + collection_warning, test_run_warning = record.warnings + assert collection_warning.category is DeprecationWarning + assert str(collection_warning.message) == "collection" + assert test_run_warning.category is PendingDeprecationWarning + assert str(test_run_warning.message) == "test run" + + def test_hidden_by_ini(self, tmp_path: Path) -> None: + collect_warner, test_foo = self.create_sources() + record = run_tests( + test_foo, + spec=ConfigSpec( + rootpath=tmp_path, + inicfg={ + "filterwarnings": [ + "ignore::DeprecationWarning", + "ignore::PendingDeprecationWarning", + ] + }, + extra_plugins=(collect_warner,), + ), ) - result = pytester.runpytest_subprocess() - assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + record.assert_outcomes(passed=1, warnings=0) + assert record.warnings == [] - def test_hidden_by_mark(self, pytester: Pytester) -> None: + def test_hidden_by_mark(self, tmp_path: Path) -> None: """Should hide the deprecation warning from the function, but the warning during collection should be displayed normally. """ - self.create_file( - pytester, - mark='@pytest.mark.filterwarnings("ignore::PendingDeprecationWarning")', + collect_warner, test_foo = self.create_sources( + mark=pytest.mark.filterwarnings("ignore::PendingDeprecationWarning") ) - result = pytester.runpytest_subprocess() - result.stdout.fnmatch_lines( - [ - f"*== {WARNINGS_SUMMARY_HEADER} ==*", - "*test_hidden_by_mark.py:3: DeprecationWarning: collection", - "* 1 passed, 1 warning*", - ] + record = run_tests( + test_foo, + spec=ConfigSpec(rootpath=tmp_path, extra_plugins=(collect_warner,)), ) - - def test_hidden_by_cmdline(self, pytester: Pytester) -> None: - self.create_file(pytester) - result = pytester.runpytest_subprocess( - "-W", - "ignore::DeprecationWarning", - "-W", - "ignore::PendingDeprecationWarning", + record.assert_outcomes(passed=1, warnings=1) + (warning,) = record.warnings + assert warning.category is DeprecationWarning + assert str(warning.message) == "collection" + + def test_hidden_by_cmdline(self, tmp_path: Path) -> None: + collect_warner, test_foo = self.create_sources() + record = run_tests( + test_foo, + spec=ConfigSpec( + rootpath=tmp_path, + args=( + "-W", + "ignore::DeprecationWarning", + "-W", + "ignore::PendingDeprecationWarning", + ), + extra_plugins=(collect_warner,), + ), ) - assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + record.assert_outcomes(passed=1, warnings=0) + assert record.warnings == [] + # ensemble: PYTHONWARNINGS is read by the interpreter at startup, so this + # needs a fresh process. def test_hidden_by_system(self, pytester: Pytester, monkeypatch) -> None: - self.create_file(pytester) + pytester.makepyfile( + """ + import pytest, warnings + + warnings.warn(DeprecationWarning("collection")) + + def test_foo(): + warnings.warn(PendingDeprecationWarning("test run")) + """ + ) monkeypatch.setenv("PYTHONWARNINGS", "once::UserWarning") result = pytester.runpytest_subprocess() assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() - def test_invalid_regex_in_filterwarning(self, pytester: Pytester) -> None: - self.create_file(pytester) - pytester.makeini( - """ - [pytest] - filterwarnings = - ignore::DeprecationWarning:* - """ + def test_invalid_regex_in_filterwarning(self, tmp_path: Path) -> None: + collect_warner, test_foo = self.create_sources() + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={"filterwarnings": ["ignore::DeprecationWarning:*"]}, + extra_plugins=(collect_warner,), ) - result = pytester.runpytest_subprocess() - assert result.ret == pytest.ExitCode.USAGE_ERROR - result.stderr.fnmatch_lines( - [ - "ERROR: while parsing the following warning configuration:", - "", - " ignore::DeprecationWarning:[*]", - "", - "This error occurred:", - "", - "Invalid regex '[*]': nothing to repeat at position 0", - ] + # The original asserted the rendered stderr of a usage error; here the + # UsageError itself is caught, with the same message. + with pytest.raises(UsageError) as excinfo: + run_tests(test_foo, spec=spec) + assert str(excinfo.value) == ( + "while parsing the following warning configuration:\n" + "\n" + " ignore::DeprecationWarning:*\n" + "\n" + "This error occurred:\n" + "\n" + "Invalid regex '*': nothing to repeat at position 0\n" ) +# ensemble: the source names ``pytest.PytestRemovedIn10Warning``, which does +# not exist yet; it only survives as text inside a makepyfile string. @pytest.mark.skip("not relevant until pytest 10.0") @pytest.mark.parametrize("change_default", [None, "ini", "cmdline"]) def test_removed_in_x_warning_as_error(pytester: Pytester, change_default) -> None: @@ -605,6 +653,8 @@ class TestAssertionWarnings: def assert_result_warns(result, msg) -> None: result.stdout.fnmatch_lines([f"*PytestAssertRewriteWarning: {msg}*"]) + # ensemble: the warning is issued by the assertion rewriter, which is not + # applied to ensemble sources. def test_tuple_warning(self, pytester: Pytester) -> None: pytester.makepyfile( """\ @@ -627,6 +677,8 @@ def test_warnings_checker_twice() -> None: warnings.warn("Message B", UserWarning) +# ensemble: the subject is how the summary groups warnings by rendered +# location, and every expected line quotes the example file's own path. @pytest.mark.filterwarnings("always::UserWarning") def test_group_warnings_by_message(pytester: Pytester) -> None: pytester.copy_example("warnings/test_group_warnings_by_message.py") @@ -658,6 +710,7 @@ def test_group_warnings_by_message(pytester: Pytester) -> None: ) +# ensemble: as above, and the grouping counts are per real test file. @pytest.mark.filterwarnings("always::UserWarning") def test_group_warnings_by_message_summary(pytester: Pytester) -> None: pytester.copy_example("warnings/test_group_warnings_by_message_summary") @@ -682,57 +735,62 @@ def test_group_warnings_by_message_summary(pytester: Pytester) -> None: ) -def test_pytest_configure_warning(pytester: Pytester, recwarn) -> None: +def test_pytest_configure_warning(tmp_path: Path, recwarn) -> None: """Issue 5115.""" - pytester.makeconftest( - """ - def pytest_configure(): - import warnings + class ConfigureWarner: + """Stands in for the conftest of the original.""" + + def pytest_configure(self): warnings.warn("from pytest_configure") - """ - ) - result = pytester.runpytest() - assert result.ret == 5 - assert "INTERNALERROR" not in result.stderr.str() + # A warning issued from ``pytest_configure`` is not recorded (the config + # catches those without recording), but it must not blow the run up + # either; the original's ``ret == 5`` was "no tests collected, no + # internal error". + record = run_tests( + spec=ConfigSpec(rootpath=tmp_path, extra_plugins=(ConfigureWarner(),)) + ) + record.assert_outcomes() warning = recwarn.pop() assert str(warning.message) == "from pytest_configure" @pytest.mark.parametrize("tryfirst", [True, False]) -def test_pytest_configure_warning_filter(pytester: Pytester, tryfirst: bool) -> None: +def test_pytest_configure_warning_filter(tmp_path: Path, tryfirst: bool) -> None: """Issue 10128. Parametrize over ``tryfirst`` to guard against hooks that run early from avoiding the filterwarnings configuration. """ - pytester.makeini( - """ - [pytest] - filterwarnings = - ignore::UserWarning - """ - ) - pytester.makeconftest( - f""" - import warnings - import pytest - @pytest.hookimpl(tryfirst={tryfirst}) - def pytest_configure(): + class ConfigureWarner: + @pytest.hookimpl(tryfirst=tryfirst) + def pytest_configure(self): warnings.warn("from pytest_configure", UserWarning) - """ - ) - pytester.makepyfile("def test_it(): pass") - - result = pytester.runpytest_subprocess() - result.assert_outcomes(passed=1) - result.stdout.no_fnmatch_line("*from pytest_configure*") - result.stderr.no_fnmatch_line("*from pytest_configure*") + def test_it(): + pass + + # If the ini filter were not in force around ``pytest_configure`` the + # warning would escape to this suite, which runs with + # ``filterwarnings = error``, and blow the run up - the in-process + # equivalent of the original's "nothing on stdout or stderr". + record = run_tests( + test_it, + spec=ConfigSpec( + rootpath=tmp_path, + inicfg={"filterwarnings": ["ignore::UserWarning"]}, + extra_plugins=(ConfigureWarner(),), + ), + ) + record.assert_outcomes(passed=1, warnings=0) +# ensemble: every test here needs a real importable plugin module, and the +# warning is caught by ``Config._capture_plugin_import_warnings`` from the +# plugin-loading phase of ``Config.parse`` - an ensemble imports the plugins +# named in its spec itself and never runs that phase. class TestPluginImportWarning: """filterwarnings apply to warnings emitted whilst importing plugins. @@ -806,6 +864,9 @@ def test_plugin_import_warning_from_pytest_plugins( result.stdout.fnmatch_lines("*DeprecationWarning: from plugin import") +# ensemble: the whole class asserts the file, line and function a warning is +# attributed to. Ensemble sources are anchored in this host file, and the +# conftest loading these exercise has no ensemble equivalent. class TestStackLevel: @pytest.fixture def capwarn(self, pytester: Pytester): @@ -927,6 +988,9 @@ def test_it(): ) +# ensemble: the warning comes from the ``testpaths`` glob expansion in +# ``Config._decide_args``, which an ensemble skips - its args are taken +# verbatim and its collection never walks the filesystem. def test_warning_on_testpaths_not_found(pytester: Pytester) -> None: # Check for warning when testpaths set, but not found by glob pytester.makeini( @@ -941,6 +1005,8 @@ def test_warning_on_testpaths_not_found(pytester: Pytester) -> None: ) +# ensemble: needs a fresh interpreter started with ``-Xdev`` and a +# PYTHONTRACEMALLOC environment. def test_resource_warning(pytester: Pytester, monkeypatch: pytest.MonkeyPatch) -> None: # Some platforms (notably PyPy) don't have tracemalloc. # We choose to explicitly not skip this in case tracemalloc is not @@ -995,158 +1061,191 @@ def test_resource_warning(tmp_path): result.stdout.fnmatch_lines([*expected_extra, "*1 passed*"]) +def run_for_exitstatus( + *sources: Source, + spec: ConfigSpec, +) -> tuple[RunRecord, int | ExitCode]: + """Run an ensemble and make the session-level exit status decision. + + ``--max-warnings`` is enforced by the terminal reporter in + ``pytest_sessionfinish``, which promotes ``session.exitstatus`` from + ``OK`` to ``MAX_WARNINGS_ERROR``. That attribute is normally set by + ``_pytest.main.wrap_session``, which an ensemble does not run, so the + same OK/TESTS_FAILED decision is made explicitly here - the promotion + itself, which is what these tests are about, is left to pytest. + """ + with Ensemble(*sources, spec=spec, capture_output=True) as ensemble: + record = ensemble.run() + session = ensemble.session + session.exitstatus = ( + ExitCode.TESTS_FAILED if session.testsfailed else ExitCode.OK + ) + # The summary line is only written on the way out of the block. + return dataclasses.replace(record, output=ensemble.output), session.exitstatus + + class TestMaxWarnings: """Tests for the --max-warnings feature.""" - PYFILE = """ - import warnings + @staticmethod + def sources() -> tuple[Source, Source]: + """The two warning-emitting tests of the original ``PYFILE``.""" + def test_one(): warnings.warn(UserWarning("warning one")) + def test_two(): warnings.warn(UserWarning("warning two")) - """ - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_not_set(self, pytester: Pytester) -> None: + return test_one, test_two + + @staticmethod + def spec( + tmp_path: Path, + *, + args: tuple[str, ...] = (), + inicfg: dict[str, object] | None = None, + ) -> ConfigSpec: + """A spec showing UserWarnings, as the enclosing marks did.""" + return ConfigSpec( + rootpath=tmp_path, + args=args, + inicfg={"filterwarnings": ["default::UserWarning"], **(inicfg or {})}, + ) + + def test_max_warnings_not_set(self, tmp_path: Path) -> None: """Without --max-warnings, warnings don't affect exit code.""" - pytester.makepyfile(self.PYFILE) - result = pytester.runpytest() - result.assert_outcomes(passed=2, warnings=2) - assert result.ret == ExitCode.OK + record, exitstatus = run_for_exitstatus( + *self.sources(), spec=self.spec(tmp_path) + ) + record.assert_outcomes(passed=2, warnings=2) + assert exitstatus == ExitCode.OK - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_not_exceeded(self, pytester: Pytester) -> None: + def test_max_warnings_not_exceeded(self, tmp_path: Path) -> None: """When warning count is below the threshold, exit code is OK.""" - pytester.makepyfile(self.PYFILE) - result = pytester.runpytest("--max-warnings", "10") - result.assert_outcomes(passed=2, warnings=2) - assert result.ret == ExitCode.OK + record, exitstatus = run_for_exitstatus( + *self.sources(), spec=self.spec(tmp_path, args=("--max-warnings", "10")) + ) + record.assert_outcomes(passed=2, warnings=2) + assert exitstatus == ExitCode.OK - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_exceeded(self, pytester: Pytester) -> None: + def test_max_warnings_exceeded(self, tmp_path: Path) -> None: """When warning count exceeds threshold, exit code is MAX_WARNINGS_ERROR.""" - pytester.makepyfile(self.PYFILE) - result = pytester.runpytest("--max-warnings", "1") - assert result.ret == ExitCode.MAX_WARNINGS_ERROR + _, exitstatus = run_for_exitstatus( + *self.sources(), spec=self.spec(tmp_path, args=("--max-warnings", "1")) + ) + assert exitstatus == ExitCode.MAX_WARNINGS_ERROR - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_equal_to_count(self, pytester: Pytester) -> None: + def test_max_warnings_equal_to_count(self, tmp_path: Path) -> None: """When warning count equals threshold exactly, exit code is OK.""" - pytester.makepyfile(self.PYFILE) - result = pytester.runpytest("--max-warnings", "2") - result.assert_outcomes(passed=2, warnings=2) - assert result.ret == ExitCode.OK + record, exitstatus = run_for_exitstatus( + *self.sources(), spec=self.spec(tmp_path, args=("--max-warnings", "2")) + ) + record.assert_outcomes(passed=2, warnings=2) + assert exitstatus == ExitCode.OK - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_zero(self, pytester: Pytester) -> None: + def test_max_warnings_zero(self, tmp_path: Path) -> None: """--max-warnings 0 means no warnings are allowed.""" - pytester.makepyfile(self.PYFILE) - result = pytester.runpytest("--max-warnings", "0") - assert result.ret == ExitCode.MAX_WARNINGS_ERROR + _, exitstatus = run_for_exitstatus( + *self.sources(), spec=self.spec(tmp_path, args=("--max-warnings", "0")) + ) + assert exitstatus == ExitCode.MAX_WARNINGS_ERROR - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_exceeded_message(self, pytester: Pytester) -> None: + def test_max_warnings_exceeded_message(self, tmp_path: Path) -> None: """Verify the output message when max warnings is exceeded.""" - pytester.makepyfile(self.PYFILE) - result = pytester.runpytest("--max-warnings", "1") - result.stdout.fnmatch_lines( + record, _ = run_for_exitstatus( + *self.sources(), spec=self.spec(tmp_path, args=("--max-warnings", "1")) + ) + record.stdout.fnmatch_lines( ["*Tests pass, but maximum allowed warnings exceeded: 2 > 1*"] ) - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_ini_option(self, pytester: Pytester) -> None: + def test_max_warnings_ini_option(self, tmp_path: Path) -> None: """max_warnings can be set via INI configuration.""" - pytester.makeini( - """ - [pytest] - max_warnings = 1 - """ + _, exitstatus = run_for_exitstatus( + *self.sources(), spec=self.spec(tmp_path, inicfg={"max_warnings": "1"}) ) - pytester.makepyfile(self.PYFILE) - result = pytester.runpytest() - assert result.ret == ExitCode.MAX_WARNINGS_ERROR + assert exitstatus == ExitCode.MAX_WARNINGS_ERROR - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_with_test_failure(self, pytester: Pytester) -> None: + def test_max_warnings_with_test_failure(self, tmp_path: Path) -> None: """When tests fail AND warnings exceed max, TESTS_FAILED takes priority.""" - pytester.makepyfile( - """ - import warnings - def test_fail(): - warnings.warn(UserWarning("a warning")) - assert False - """ + + def test_fail(): + warnings.warn(UserWarning("a warning")) + raise AssertionError + + record, exitstatus = run_for_exitstatus( + test_fail, spec=self.spec(tmp_path, args=("--max-warnings", "0")) ) - result = pytester.runpytest("--max-warnings", "0") - assert result.ret == ExitCode.TESTS_FAILED + record.assert_outcomes(failed=1, warnings=1) + assert exitstatus == ExitCode.TESTS_FAILED - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_with_filterwarnings_ignore(self, pytester: Pytester) -> None: + def test_max_warnings_with_filterwarnings_ignore(self, tmp_path: Path) -> None: """Filtered (ignored) warnings don't count toward max_warnings.""" - pytester.makepyfile( - """ - import warnings - def test_one(): - warnings.warn(UserWarning("counted")) - warnings.warn(RuntimeWarning("ignored")) - """ - ) - result = pytester.runpytest( - "--max-warnings", - "1", - "-W", - "ignore::RuntimeWarning", + + def test_one(): + warnings.warn(UserWarning("counted")) + warnings.warn(RuntimeWarning("ignored")) + + record, exitstatus = run_for_exitstatus( + test_one, + spec=self.spec( + tmp_path, + args=("--max-warnings", "1", "-W", "ignore::RuntimeWarning"), + ), ) - result.assert_outcomes(passed=1, warnings=1) - assert result.ret == ExitCode.OK + record.assert_outcomes(passed=1, warnings=1) + assert exitstatus == ExitCode.OK - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_with_filterwarnings_error(self, pytester: Pytester) -> None: + def test_max_warnings_with_filterwarnings_error(self, tmp_path: Path) -> None: """Warnings turned into errors via filterwarnings don't count as warnings.""" - pytester.makepyfile( - """ - import warnings - def test_one(): - warnings.warn(UserWarning("still a warning")) - def test_two(): - warnings.warn(RuntimeWarning("becomes an error")) - """ - ) - result = pytester.runpytest( - "--max-warnings", - "0", - "-W", - "error::RuntimeWarning", + + def test_one(): + warnings.warn(UserWarning("still a warning")) + + def test_two(): + warnings.warn(RuntimeWarning("becomes an error")) + + record, exitstatus = run_for_exitstatus( + test_one, + test_two, + spec=self.spec( + tmp_path, + args=("--max-warnings", "0", "-W", "error::RuntimeWarning"), + ), ) + record.assert_outcomes(passed=1, failed=1, warnings=1) # The RuntimeWarning becomes a test error, so TESTS_FAILED takes priority. - assert result.ret == ExitCode.TESTS_FAILED + assert exitstatus == ExitCode.TESTS_FAILED - @pytest.mark.filterwarnings("default::UserWarning") - def test_max_warnings_with_filterwarnings_ini_ignore( - self, pytester: Pytester - ) -> None: + def test_max_warnings_with_filterwarnings_ini_ignore(self, tmp_path: Path) -> None: """Warnings ignored via ini filterwarnings don't count toward max_warnings.""" - pytester.makeini( - """ - [pytest] - filterwarnings = - ignore::RuntimeWarning - max_warnings = 1 - """ - ) - pytester.makepyfile( - """ - import warnings - def test_one(): - warnings.warn(UserWarning("counted")) - warnings.warn(RuntimeWarning("ignored by ini")) - """ + + def test_one(): + warnings.warn(UserWarning("counted")) + warnings.warn(RuntimeWarning("ignored by ini")) + + record, exitstatus = run_for_exitstatus( + test_one, + spec=self.spec( + tmp_path, + inicfg={ + "filterwarnings": [ + "default::UserWarning", + "ignore::RuntimeWarning", + ], + "max_warnings": "1", + }, + ), ) - result = pytester.runpytest() - result.assert_outcomes(passed=1, warnings=1) - assert result.ret == ExitCode.OK + record.assert_outcomes(passed=1, warnings=1) + assert exitstatus == ExitCode.OK +# ensemble: the duplication regressed on is produced by ``Config.parse`` +# invoking the argument parser several times over the same args; an ensemble +# builds its namespace with a single ``parse_known_args`` call, so the same +# assertion there could not fail. def test_pythonwarnings_not_duplicated(pytester: Pytester) -> None: """Regression test for #13484: -W values should not be duplicated in known_args_namespace due to the arg parser being called multiple times.""" From 8e76bbe1b15223edc81a93ac04f0dd34d74b6182 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 20:52:55 +0200 Subject: [PATCH 10/30] testing: port test_subtests.py to _pytest.ensemble 27 of 31 pytester-based tests ported. This file taught us how subtest outcomes actually read: a failed subtest is plain "failed" and the top-level report is also failed, so one test with one failing subtest is failed=2; a passed subtest is "subtests passed", a key assert_outcomes() silently ignores, and one that only exists when verbosity_subtests > 0. Ported tests assert it explicitly. The --last-failed test was a silently green no-op before it was made to load the terminal plugin: subtests marks the top-level test failed as a side effect *inside* pytest_report_teststatus, so with nothing rendering, LFPlugin saw the report still passed and popped the nodeid the failing subtest had just added. cache/lastfailed was never written and the rerun was indistinguishable from a fresh run. 4 stay on pytester: three need capture around the whole item, and one is xdist over real files. --- testing/test_subtests.py | 1081 ++++++++++++++++++++------------------ 1 file changed, 575 insertions(+), 506 deletions(-) diff --git a/testing/test_subtests.py b/testing/test_subtests.py index 877e32b5204..ee01d141bf0 100644 --- a/testing/test_subtests.py +++ b/testing/test_subtests.py @@ -2,10 +2,20 @@ from enum import Enum import json +import logging +from pathlib import Path import sys +import types from typing import Literal +import unittest from _pytest._io.saferepr import saferepr +from _pytest.ensemble import build_module +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import run_tests +from _pytest.ensemble import Source +from _pytest.outcomes import Exit +from _pytest.reports import TestReport from _pytest.subtests import SubtestContext from _pytest.subtests import SubtestReport import pytest @@ -14,24 +24,55 @@ IS_PY311 = sys.version_info[:2] >= (3, 11) -def test_failures(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - def test_foo(subtests): - with subtests.test("foo subtest"): - assert False, "foo subtest failure" +def _subtests_spec(tmp_path: Path, *args: str, **inicfg: str) -> ConfigSpec: + """A :class:`ConfigSpec` for an ensemble exercising subtests. - def test_bar(subtests): - with subtests.test("bar subtest"): - assert False, "bar subtest failure" - assert False, "test_bar also failed" + ``subtests`` is not one of the ensemble default plugins, so it has to be + asked for explicitly. + """ + return ConfigSpec(rootpath=tmp_path, args=args, inicfg=inicfg).with_plugins( + "subtests" + ) - def test_zaz(subtests): - with subtests.test("zaz subtest"): - pass - """ + +def _rendering_spec(tmp_path: Path, *args: str, **inicfg: str) -> ConfigSpec: + """As :func:`_subtests_spec`, for ensembles whose *output* is the subject. + + Only usable together with ``capture_output=True``, which is what pulls in + the terminal plugin these settings belong to. The console output style is + bumped because the terminal reporter draws the progress percentages only + when it believes output is being captured - which an ensemble's terminal, + having no capture manager of its own, never is. + """ + return _subtests_spec( + tmp_path, + *args, + console_output_style="progress-even-when-capture-no", + **inicfg, ) + + +def _failure_sources() -> types.ModuleType: + """The module under test of ``test_failures``.""" + + def test_foo(subtests: pytest.Subtests) -> None: + with subtests.test("foo subtest"): + assert False, "foo subtest failure" + + def test_bar(subtests: pytest.Subtests) -> None: + with subtests.test("bar subtest"): + assert False, "bar subtest failure" + assert False, "test_bar also failed" + + def test_zaz(subtests: pytest.Subtests) -> None: + with subtests.test("zaz subtest"): + pass + + return build_module("test_failures", test_foo, test_bar, test_zaz) + + +def test_failures(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("COLUMNS", "120") summary_lines = [ "*=== FAILURES ===*", # @@ -53,17 +94,26 @@ def test_zaz(subtests): "SUBFAILED[[]bar subtest[]] test_*.py::test_bar - AssertionError*", "FAILED test_*.py::test_bar - AssertionError*", ] - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record = run_tests( + _failure_sources(), spec=_rendering_spec(tmp_path), capture_output=True + ) + record.stdout.fnmatch_lines( [ - "test_*.py uFuF. * [[]100%[]]", + # The original also matched a trailing "[100%]" here. The terminal + # reporter defers that final fill to ``pytest_runtestloop``, which + # an ensemble never calls - it drives the items directly. The + # per-test letters, which are what this test is about, are intact. + "test_*.py uFuF.", *summary_lines, "* 4 failed, 1 passed in *", ] ) + record.assert_outcomes(failed=4, passed=1) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + record = run_tests( + _failure_sources(), spec=_rendering_spec(tmp_path, "-v"), capture_output=True + ) + record.stdout.fnmatch_lines( [ "test_*.py::test_foo SUBFAILED[[]foo subtest[]] * [[] 33%[]]", "test_*.py::test_foo FAILED * [[] 33%[]]", @@ -75,14 +125,16 @@ def test_zaz(subtests): "* 4 failed, 1 passed, 1 subtests passed in *", ] ) - pytester.makeini( - """ - [pytest] - verbosity_subtests = 0 - """ + # "subtests passed" is a terminal category of its own, which + # assert_outcomes() (like RunResult.assert_outcomes) does not know about. + assert record.outcomes()["subtests passed"] == 1 + + record = run_tests( + _failure_sources(), + spec=_rendering_spec(tmp_path, "-v", verbosity_subtests="0"), + capture_output=True, ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "test_*.py::test_foo SUBFAILED[[]foo subtest[]] * [[] 33%[]]", "test_*.py::test_foo FAILED * [[] 33%[]]", @@ -93,32 +145,42 @@ def test_zaz(subtests): "* 4 failed, 1 passed in *", ] ) - result.stdout.no_fnmatch_line("test_*.py::test_zaz SUBPASSED[[]zaz subtest[]]*") + record.stdout.no_fnmatch_line("test_*.py::test_zaz SUBPASSED[[]zaz subtest[]]*") + assert "subtests passed" not in record.outcomes() -def test_passes(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - def test_foo(subtests): - with subtests.test("foo subtest"): - pass +def _passing_sources() -> types.ModuleType: + """The module under test of ``test_passes``.""" - def test_bar(subtests): - with subtests.test("bar subtest"): - pass - """ + def test_foo(subtests: pytest.Subtests) -> None: + with subtests.test("foo subtest"): + pass + + def test_bar(subtests: pytest.Subtests) -> None: + with subtests.test("bar subtest"): + pass + + return build_module("test_passes", test_foo, test_bar) + + +def test_passes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("COLUMNS", "120") + record = run_tests( + _passing_sources(), spec=_rendering_spec(tmp_path), capture_output=True ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ - "test_*.py .. * [[]100%[]]", + # see test_failures on the dropped "[100%]" + "test_*.py ..", "* 2 passed in *", ] ) + record.assert_outcomes(passed=2) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + record = run_tests( + _passing_sources(), spec=_rendering_spec(tmp_path, "-v"), capture_output=True + ) + record.stdout.fnmatch_lines( [ "*.py::test_foo SUBPASSED[[]foo subtest[]] * [[] 50%[]]", "*.py::test_foo PASSED * [[] 50%[]]", @@ -127,52 +189,63 @@ def test_bar(subtests): "* 2 passed, 2 subtests passed in *", ] ) + assert record.outcomes()["subtests passed"] == 2 - pytester.makeini( - """ - [pytest] - verbosity_subtests = 0 - """ + record = run_tests( + _passing_sources(), + spec=_rendering_spec(tmp_path, "-v", verbosity_subtests="0"), + capture_output=True, ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*.py::test_foo PASSED * [[] 50%[]]", "*.py::test_bar PASSED * [[]100%[]]", "* 2 passed in *", ] ) - result.stdout.no_fnmatch_line("*.py::test_foo SUBPASSED[[]foo subtest[]]*") - result.stdout.no_fnmatch_line("*.py::test_bar SUBPASSED[[]bar subtest[]]*") + record.stdout.no_fnmatch_line("*.py::test_foo SUBPASSED[[]foo subtest[]]*") + record.stdout.no_fnmatch_line("*.py::test_bar SUBPASSED[[]bar subtest[]]*") + + +def _skip_sources() -> types.ModuleType: + """The module under test of ``test_skip``.""" + def test_foo(subtests: pytest.Subtests) -> None: + with subtests.test("foo subtest"): + pytest.skip("skip foo subtest") -def test_skip(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + def test_bar(subtests: pytest.Subtests) -> None: + with subtests.test("bar subtest"): + pytest.skip("skip bar subtest") + pytest.skip("skip test_bar") + + return build_module("test_skip", test_foo, test_bar) + + +def test_skip(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - import pytest - def test_foo(subtests): - with subtests.test("foo subtest"): - pytest.skip("skip foo subtest") - - def test_bar(subtests): - with subtests.test("bar subtest"): - pytest.skip("skip bar subtest") - pytest.skip("skip test_bar") - """ + record = run_tests( + _skip_sources(), spec=_rendering_spec(tmp_path, "-ra"), capture_output=True ) - result = pytester.runpytest("-ra") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ - "test_*.py .s * [[]100%[]]", + # see test_failures on the dropped "[100%]" + "test_*.py .s", "*=== short test summary info ===*", - "SKIPPED [[]1[]] test_skip.py:9: skip test_bar", + # the original spelled out "test_skip.py:9" here; the location of + # an in-memory source is anchored in *this* file. + "SKIPPED [[]1[]] *: skip test_bar", "* 1 passed, 1 skipped in *", ] ) + record.assert_outcomes(passed=1, skipped=1) - result = pytester.runpytest("-v", "-ra") - result.stdout.fnmatch_lines( + record = run_tests( + _skip_sources(), + spec=_rendering_spec(tmp_path, "-v", "-ra"), + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "*.py::test_foo SUBSKIPPED[[]foo subtest[]] (skip foo subtest) * [[] 50%[]]", "*.py::test_foo PASSED * [[] 50%[]]", @@ -185,15 +258,14 @@ def test_bar(subtests): "* 1 passed, 3 skipped in *", ] ) + record.assert_outcomes(passed=1, skipped=3) - pytester.makeini( - """ - [pytest] - verbosity_subtests = 0 - """ + record = run_tests( + _skip_sources(), + spec=_rendering_spec(tmp_path, "-v", "-ra", verbosity_subtests="0"), + capture_output=True, ) - result = pytester.runpytest("-v", "-ra") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*.py::test_foo PASSED * [[] 50%[]]", "*.py::test_bar SKIPPED (skip test_bar) * [[]100%[]]", @@ -201,42 +273,52 @@ def test_bar(subtests): "* 1 passed, 1 skipped in *", ] ) - result.stdout.no_fnmatch_line("*.py::test_foo SUBPASSED[[]foo subtest[]]*") - result.stdout.no_fnmatch_line("*.py::test_bar SUBPASSED[[]bar subtest[]]*") - result.stdout.no_fnmatch_line( + record.stdout.no_fnmatch_line("*.py::test_foo SUBPASSED[[]foo subtest[]]*") + record.stdout.no_fnmatch_line("*.py::test_bar SUBPASSED[[]bar subtest[]]*") + record.stdout.no_fnmatch_line( "SUBSKIPPED[[]foo subtest[]] [[]1[]] *.py:*: skip foo subtest" ) - result.stdout.no_fnmatch_line( + record.stdout.no_fnmatch_line( "SUBSKIPPED[[]foo subtest[]] [[]1[]] *.py:*: skip test_bar" ) -def test_xfail(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: +def _xfail_sources() -> types.ModuleType: + """The module under test of ``test_xfail``.""" + + def test_foo(subtests: pytest.Subtests) -> None: + with subtests.test("foo subtest"): + pytest.xfail("xfail foo subtest") + + def test_bar(subtests: pytest.Subtests) -> None: + with subtests.test("bar subtest"): + pytest.xfail("xfail bar subtest") + pytest.xfail("xfail test_bar") + + return build_module("test_xfail", test_foo, test_bar) + + +def test_xfail(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - import pytest - def test_foo(subtests): - with subtests.test("foo subtest"): - pytest.xfail("xfail foo subtest") - - def test_bar(subtests): - with subtests.test("bar subtest"): - pytest.xfail("xfail bar subtest") - pytest.xfail("xfail test_bar") - """ + record = run_tests( + _xfail_sources(), spec=_rendering_spec(tmp_path, "-ra"), capture_output=True ) - result = pytester.runpytest("-ra") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ - "test_*.py .x * [[]100%[]]", + # see test_failures on the dropped "[100%]" + "test_*.py .x", "*=== short test summary info ===*", "* 1 passed, 1 xfailed in *", ] ) + record.assert_outcomes(passed=1, xfailed=1) - result = pytester.runpytest("-v", "-ra") - result.stdout.fnmatch_lines( + record = run_tests( + _xfail_sources(), + spec=_rendering_spec(tmp_path, "-v", "-ra"), + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "*.py::test_foo SUBXFAIL[[]foo subtest[]] (xfail foo subtest) * [[] 50%[]]", "*.py::test_foo PASSED * [[] 50%[]]", @@ -249,15 +331,14 @@ def test_bar(subtests): "* 1 passed, 3 xfailed in *", ] ) + record.assert_outcomes(passed=1, xfailed=3) - pytester.makeini( - """ - [pytest] - verbosity_subtests = 0 - """ + record = run_tests( + _xfail_sources(), + spec=_rendering_spec(tmp_path, "-v", "-ra", verbosity_subtests="0"), + capture_output=True, ) - result = pytester.runpytest("-v", "-ra") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*.py::test_foo PASSED * [[] 50%[]]", "*.py::test_bar XFAIL (xfail test_bar) * [[]100%[]]", @@ -265,45 +346,49 @@ def test_bar(subtests): "* 1 passed, 1 xfailed in *", ] ) - result.stdout.no_fnmatch_line( + record.stdout.no_fnmatch_line( "SUBXFAIL[[]foo subtest[]] *.py::test_foo - xfail foo subtest" ) - result.stdout.no_fnmatch_line( + record.stdout.no_fnmatch_line( "SUBXFAIL[[]bar subtest[]] *.py::test_bar - xfail bar subtest" ) -def test_typing_exported(pytester: pytest.Pytester) -> None: - pytester.makepyfile( - """ - from pytest import Subtests +def test_typing_exported(tmp_path: Path) -> None: + from pytest import Subtests - def test_typing_exported(subtests: Subtests) -> None: - assert isinstance(subtests, Subtests) - """ + def test_typing_exported(subtests: Subtests) -> None: + assert isinstance(subtests, Subtests) + + record = run_tests( + test_typing_exported, spec=_subtests_spec(tmp_path), name="test_typing_exported" ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 passed*"]) + record.assert_outcomes(passed=1) + + +def _parametrized_sources() -> types.ModuleType: + """The module under test of ``test_subtests_and_parametrization``.""" + + @pytest.mark.parametrize("x", [0, 1]) + def test_foo(subtests: pytest.Subtests, x: int) -> None: + for i in range(3): + with subtests.test("custom", i=i): + assert i % 2 == 0 + assert x == 0 + + return build_module("test_subtests_and_parametrization", test_foo) def test_subtests_and_parametrization( - pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - import pytest - - @pytest.mark.parametrize("x", [0, 1]) - def test_foo(subtests, x): - for i in range(3): - with subtests.test("custom", i=i): - assert i % 2 == 0 - assert x == 0 - """ + record = run_tests( + _parametrized_sources(), + spec=_rendering_spec(tmp_path, "-v"), + capture_output=True, ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*.py::test_foo[[]0[]] SUBFAILED[[]custom[]] (i=1) *[[] 50%[]]", "*.py::test_foo[[]0[]] FAILED *[[] 50%[]]", @@ -313,15 +398,15 @@ def test_foo(subtests, x): "* 4 failed, 4 subtests passed in *", ] ) + record.assert_outcomes(failed=4) + assert record.outcomes()["subtests passed"] == 4 - pytester.makeini( - """ - [pytest] - verbosity_subtests = 0 - """ + record = run_tests( + _parametrized_sources(), + spec=_rendering_spec(tmp_path, "-v", verbosity_subtests="0"), + capture_output=True, ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*.py::test_foo[[]0[]] SUBFAILED[[]custom[]] (i=1) *[[] 50%[]]", "*.py::test_foo[[]0[]] FAILED *[[] 50%[]]", @@ -331,51 +416,49 @@ def test_foo(subtests, x): "* 4 failed in *", ] ) + assert "subtests passed" not in record.outcomes() -def test_subtests_fail_top_level_test(pytester: pytest.Pytester) -> None: - pytester.makepyfile( - """ - import pytest +def test_subtests_fail_top_level_test(tmp_path: Path) -> None: + def test_foo(subtests: pytest.Subtests) -> None: + for i in range(3): + with subtests.test("custom", i=i): + assert i % 2 == 0 - def test_foo(subtests): - for i in range(3): - with subtests.test("custom", i=i): - assert i % 2 == 0 - """ + # ``-v`` is a terminal plugin option, which an ensemble only loads when it + # is asked to capture output; the ini has the same effect on the category. + record = run_tests( + test_foo, + spec=_subtests_spec(tmp_path, verbosity_subtests="1"), + name="test_subtests_fail_top_level_test", ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( - [ - "* 2 failed, 2 subtests passed in *", - ] + # the original read "* 2 failed, 2 subtests passed in *" off the summary + record.assert_outcomes(failed=2) + assert record.outcomes()["subtests passed"] == 2 + + +def test_subtests_do_not_overwrite_top_level_failure(tmp_path: Path) -> None: + def test_foo(subtests: pytest.Subtests) -> None: + for i in range(3): + with subtests.test("custom", i=i): + assert i % 2 == 0 + assert False, "top-level failure" + + record = run_tests( + test_foo, + spec=_subtests_spec(tmp_path, verbosity_subtests="1"), + name="test_subtests_do_not_overwrite_top_level_failure", ) + record.assert_outcomes(failed=2) + assert record.outcomes()["subtests passed"] == 2 + # the top level report keeps its own failure instead of being replaced by + # the "contains N failed subtests" one + call = record["test_foo"].call + assert call is not None + assert "AssertionError: top-level failure" in call.longreprtext -def test_subtests_do_not_overwrite_top_level_failure(pytester: pytest.Pytester) -> None: - pytester.makepyfile( - """ - import pytest - - def test_foo(subtests): - for i in range(3): - with subtests.test("custom", i=i): - assert i % 2 == 0 - assert False, "top-level failure" - """ - ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( - [ - "*AssertionError: top-level failure", - "* 2 failed, 2 subtests passed in *", - ] - ) - - -def test_msg_not_a_string( - pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_msg_not_a_string(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """ Using a non-string in subtests.test() should still show it in the terminal (#14195). @@ -383,19 +466,18 @@ def test_msg_not_a_string( was added for symmetry. """ monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - def test_int_msg(subtests): - with subtests.test(42): - assert False, "subtest failure" - def test_no_msg(subtests): - with subtests.test(): - assert False, "subtest failure" - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + def test_int_msg(subtests: pytest.Subtests) -> None: + with subtests.test(42): # type: ignore[arg-type] + assert False, "subtest failure" + + def test_no_msg(subtests: pytest.Subtests) -> None: + with subtests.test(): + assert False, "subtest failure" + + module = build_module("test_msg_not_a_string", test_int_msg, test_no_msg) + record = run_tests(module, spec=_rendering_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines( [ "SUBFAILED[[]42[]] test_msg_not_a_string.py::test_int_msg - AssertionError: subtest failure", "SUBFAILED() test_msg_not_a_string.py::test_no_msg - AssertionError: subtest failure", @@ -404,221 +486,197 @@ def test_no_msg(subtests): @pytest.mark.parametrize("flag", ["--last-failed", "--stepwise"]) -def test_subtests_last_failed_step_wise(pytester: pytest.Pytester, flag: str) -> None: +def test_subtests_last_failed_step_wise(tmp_path: Path, flag: str) -> None: """Check that --last-failed and --step-wise correctly rerun tests with failed subtests.""" - pytester.makepyfile( - """ - import pytest - def test_foo(subtests): - for i in range(3): - with subtests.test("custom", i=i): - assert i % 2 == 0 - """ + def test_foo(subtests: pytest.Subtests) -> None: + for i in range(3): + with subtests.test("custom", i=i): + assert i % 2 == 0 + + # Both flags read the cache the first run leaves behind in the shared + # rootpath, so the two runs have to agree on it. The terminal plugin is + # not optional here: what marks the top level test as failed is a *side + # effect* of ``pytest_report_teststatus``, and without a terminal nothing + # calls that hook while the run is in progress - so the last-failed cache + # would never learn about the failure and the flag below would be a no-op. + spec = _rendering_spec(tmp_path, "-v").with_plugins("cacheprovider", "stepwise") + name = "test_subtests_last_failed_step_wise" + record = run_tests(test_foo, spec=spec, name=name, capture_output=True) + record.stdout.fnmatch_lines(["* 2 failed, 2 subtests passed in *"]) + record.assert_outcomes(failed=2) + assert record.outcomes()["subtests passed"] == 2 + + record = run_tests( + test_foo, spec=spec.replace(args=("-v", flag)), name=name, capture_output=True ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( - [ - "* 2 failed, 2 subtests passed in *", - ] - ) - - result = pytester.runpytest("-v", flag) - result.stdout.fnmatch_lines( + # proof the flag had the previous run's state to act on at all + record.stdout.fnmatch_lines( [ + { + "--last-failed": "run-last-failure: rerun previous 1 failure", + # stepwise only records state when it was itself active, and + # the first run above was a plain one - same as in the original + "--stepwise": "stepwise: no previously failed tests, not skipping.", + }[flag], "* 2 failed, 2 subtests passed in *", ] ) + record.assert_outcomes(failed=2) + assert record.outcomes()["subtests passed"] == 2 class TestUnittestSubTest: """Test unittest.TestCase.subTest functionality.""" - def test_failures( - self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch - ) -> None: + def test_failures(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - from unittest import TestCase - class T(TestCase): - def test_foo(self): - with self.subTest("foo subtest"): - assert False, "foo subtest failure" + class T(unittest.TestCase): + def test_foo(self) -> None: + with self.subTest("foo subtest"): + assert False, "foo subtest failure" - def test_bar(self): - with self.subTest("bar subtest"): - assert False, "bar subtest failure" - assert False, "test_bar also failed" + def test_bar(self) -> None: + with self.subTest("bar subtest"): + assert False, "bar subtest failure" + assert False, "test_bar also failed" - def test_zaz(self): - with self.subTest("zaz subtest"): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - "* 3 failed, 2 passed in *", - ] - ) + def test_zaz(self) -> None: + with self.subTest("zaz subtest"): + pass - def test_passes( - self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch - ) -> None: + record = run_tests(T, spec=_subtests_spec(tmp_path)) + record.assert_outcomes(failed=3, passed=2) + + def test_passes(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - from unittest import TestCase - class T(TestCase): - def test_foo(self): - with self.subTest("foo subtest"): - pass + class T(unittest.TestCase): + def test_foo(self) -> None: + with self.subTest("foo subtest"): + pass - def test_bar(self): - with self.subTest("bar subtest"): - pass + def test_bar(self) -> None: + with self.subTest("bar subtest"): + pass - def test_zaz(self): - with self.subTest("zaz subtest"): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - "* 3 passed in *", - ] - ) + def test_zaz(self) -> None: + with self.subTest("zaz subtest"): + pass - def test_skip( - self, - pytester: pytest.Pytester, - ) -> None: - pytester.makepyfile( - """ - from unittest import TestCase, main + record = run_tests(T, spec=_subtests_spec(tmp_path)) + record.assert_outcomes(passed=3) - class T(TestCase): + def test_skip(self, tmp_path: Path) -> None: + class T(unittest.TestCase): + def test_foo(self) -> None: + for i in range(5): + with self.subTest(msg="custom", i=i): + if i % 2 == 0: + self.skipTest("even number") - def test_foo(self): - for i in range(5): - with self.subTest(msg="custom", i=i): - if i % 2 == 0: - self.skipTest('even number') - """ - ) # This output might change #13756. - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 1 passed in *"]) + record = run_tests(T, spec=_subtests_spec(tmp_path)) + record.assert_outcomes(passed=1) def test_non_subtest_skip( - self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - from unittest import TestCase, main - class T(TestCase): + class T(unittest.TestCase): + def test_foo(self) -> None: + with self.subTest(msg="subtest"): + assert False, "failed subtest" + self.skipTest("non-subtest skip") - def test_foo(self): - with self.subTest(msg="subtest"): - assert False, "failed subtest" - self.skipTest('non-subtest skip') - """ - ) # This output might change #13756. - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record = run_tests( + T, + spec=_rendering_spec(tmp_path), + name="test_non_subtest_skip", + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "SUBFAILED[[]subtest[]] test_non_subtest_skip.py::T::test_foo*", "* 1 failed, 1 skipped in *", ] ) + record.assert_outcomes(failed=1, skipped=1) + + def test_xfail(self, tmp_path: Path) -> None: + class T(unittest.TestCase): + @unittest.expectedFailure + def test_foo(self) -> None: + for i in range(5): + with self.subTest(msg="custom", i=i): + if i % 2 == 0: + raise pytest.xfail("even number") - def test_xfail( - self, - pytester: pytest.Pytester, - ) -> None: - pytester.makepyfile( - """ - import pytest - from unittest import expectedFailure, TestCase - - class T(TestCase): - @expectedFailure - def test_foo(self): - for i in range(5): - with self.subTest(msg="custom", i=i): - if i % 2 == 0: - raise pytest.xfail('even number') - - if __name__ == '__main__': - main() - """ - ) # This output might change #13756. - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 1 xfailed in *"]) + record = run_tests(T, spec=_subtests_spec(tmp_path)) + record.assert_outcomes(xfailed=1) def test_only_original_skip_is_called( - self, - pytester: pytest.Pytester, - monkeypatch: pytest.MonkeyPatch, + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Regression test for pytest-dev/pytest-subtests#173.""" monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - import unittest - from unittest import TestCase - @unittest.skip("skip this test") - class T(unittest.TestCase): - def test_foo(self): - assert 1 == 2 - """ - ) - result = pytester.runpytest("-v", "-rsf") - result.stdout.fnmatch_lines( - ["SKIPPED [1] test_only_original_skip_is_called.py:6: skip this test"] + @unittest.skip("skip this test") + class T(unittest.TestCase): + def test_foo(self) -> None: + # deliberately false; the class level skip must keep it from running + assert 1 == 2 # type: ignore[comparison-overlap] + + record = run_tests( + T, + spec=_rendering_spec(tmp_path, "-v", "-rsf"), + name="test_only_original_skip_is_called", + capture_output=True, ) + # the original spelled out "test_only_original_skip_is_called.py:6" + # here; the location of an in-memory source is anchored in *this* file. + record.stdout.fnmatch_lines(["SKIPPED [[]1[]] *: skip this test"]) + record.assert_outcomes(skipped=1) def test_skip_with_failure( - self, - pytester: pytest.Pytester, - monkeypatch: pytest.MonkeyPatch, + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - import pytest - from unittest import TestCase - - class T(TestCase): - def test_foo(self): - with self.subTest("subtest 1"): - self.skipTest(f"skip subtest 1") - with self.subTest("subtest 2"): - assert False, "fail subtest 2" - """ - ) - result = pytester.runpytest("-ra") - result.stdout.fnmatch_lines( + class T(unittest.TestCase): + def test_foo(self) -> None: + with self.subTest("subtest 1"): + self.skipTest("skip subtest 1") + # skipTest() is typed NoReturn, but subTest() swallows the skip + with self.subTest("subtest 2"): # type: ignore[unreachable] + assert False, "fail subtest 2" + + name = "test_skip_with_failure" + record = run_tests( + T, spec=_rendering_spec(tmp_path, "-ra"), name=name, capture_output=True + ) + record.stdout.fnmatch_lines( [ - "*.py u. * [[]100%[]]", + # see test_failures on the dropped "[100%]" + "*.py u.", "*=== short test summary info ===*", "SUBFAILED[[]subtest 2[]] *.py::T::test_foo - AssertionError: fail subtest 2", "* 1 failed, 1 passed in *", ] ) + record.assert_outcomes(failed=1, passed=1) - result = pytester.runpytest("-v", "-ra") - result.stdout.fnmatch_lines( + record = run_tests( + T, + spec=_rendering_spec(tmp_path, "-v", "-ra"), + name=name, + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "*.py::T::test_foo SUBSKIPPED[[]subtest 1[]] (skip subtest 1) * [[]100%[]]", "*.py::T::test_foo SUBFAILED[[]subtest 2[]] * [[]100%[]]", @@ -628,15 +686,15 @@ def test_foo(self): "* 1 failed, 1 passed, 1 skipped in *", ] ) + record.assert_outcomes(failed=1, passed=1, skipped=1) - pytester.makeini( - """ - [pytest] - verbosity_subtests = 0 - """ + record = run_tests( + T, + spec=_rendering_spec(tmp_path, "-v", "-ra", verbosity_subtests="0"), + name=name, + capture_output=True, ) - result = pytester.runpytest("-v", "-ra") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*.py::T::test_foo SUBFAILED[[]subtest 2[]] * [[]100%[]]", "*.py::T::test_foo PASSED * [[]100%[]]", @@ -645,34 +703,35 @@ def test_foo(self): r"* 1 failed, 1 passed in *", ] ) - result.stdout.no_fnmatch_line( + record.stdout.no_fnmatch_line( "*.py::T::test_foo SUBSKIPPED[[]subtest 1[]] (skip subtest 1) * [[]100%[]]" ) - result.stdout.no_fnmatch_line( + record.stdout.no_fnmatch_line( "SUBSKIPPED[[]subtest 1[]] [[]1[]] *.py:*: skip subtest 1" ) def test_msg_not_a_string( - self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Using a non-string in TestCase.subTest should still show it in the terminal (#14195).""" monkeypatch.setenv("COLUMNS", "120") - pytester.makepyfile( - """ - from unittest import TestCase - - class T(TestCase): - def test_int_msg(self): - with self.subTest(42): - assert False, "subtest failure" - def test_no_msg(self): - with self.subTest(): - assert False, "subtest failure" - """ + class T(unittest.TestCase): + def test_int_msg(self) -> None: + with self.subTest(42): + assert False, "subtest failure" + + def test_no_msg(self) -> None: + with self.subTest(): + assert False, "subtest failure" + + record = run_tests( + T, + spec=_rendering_spec(tmp_path), + name="test_msg_not_a_string", + capture_output=True, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "SUBFAILED[[]42[]] test_msg_not_a_string.py::T::test_int_msg - AssertionError: subtest failure", "SUBFAILED() test_msg_not_a_string.py::T::test_no_msg - AssertionError: subtest failure", @@ -680,6 +739,13 @@ def test_no_msg(self): ) +# ensemble: every test in this class needs capture *around the whole item* - +# the "__ test __" section holding the top level "start test"/"end test" +# output, ``-s``, and ``capsys``/``capfd``. An ensemble config never runs +# ``pytest_load_initial_conftests``, so its capture manager never starts +# global capturing: the subtests' own CaptureFixture still works, but the +# enclosing test's output is neither captured nor reported - it escapes to +# the *host's* stdout instead. class TestCapture: def create_file(self, pytester: pytest.Pytester) -> None: pytester.makepyfile( @@ -772,31 +838,31 @@ def test(subtests, {fixture}): class TestLogging: - def create_file(self, pytester: pytest.Pytester) -> None: - pytester.makepyfile( - """ - import logging + def create_module(self) -> types.ModuleType: + def test_foo(subtests: pytest.Subtests) -> None: + logging.info("before") + + with subtests.test("sub1"): + print("sub1 stdout") + logging.info("sub1 logging") + logging.debug("sub1 logging debug") - def test_foo(subtests): - logging.info("before") + with subtests.test("sub2"): + print("sub2 stdout") + logging.info("sub2 logging") + logging.debug("sub2 logging debug") + assert False - with subtests.test("sub1"): - print("sub1 stdout") - logging.info("sub1 logging") - logging.debug("sub1 logging debug") + return build_module("test_logging", test_foo) - with subtests.test("sub2"): - print("sub2 stdout") - logging.info("sub2 logging") - logging.debug("sub2 logging debug") - assert False - """ + def test_capturing_info(self, tmp_path: Path) -> None: + # the subtest sections need the capture plugin for stdout and the + # logging plugin for the log records; neither is an ensemble default. + spec = _rendering_spec(tmp_path, "--log-level=INFO").with_plugins( + "logging", "capture" ) - - def test_capturing_info(self, pytester: pytest.Pytester) -> None: - self.create_file(pytester) - result = pytester.runpytest("--log-level=INFO") - result.stdout.fnmatch_lines( + record = run_tests(self.create_module(), spec=spec, capture_output=True) + record.stdout.fnmatch_lines( [ "*___ test_foo [[]sub2[]] __*", "*-- Captured stdout call --*", @@ -808,13 +874,15 @@ def test_capturing_info(self, pytester: pytest.Pytester) -> None: "*== short test summary info ==*", ] ) - result.stdout.no_fnmatch_line("sub1 logging debug") - result.stdout.no_fnmatch_line("sub2 logging debug") + record.stdout.no_fnmatch_line("sub1 logging debug") + record.stdout.no_fnmatch_line("sub2 logging debug") - def test_capturing_debug(self, pytester: pytest.Pytester) -> None: - self.create_file(pytester) - result = pytester.runpytest("--log-level=DEBUG") - result.stdout.fnmatch_lines( + def test_capturing_debug(self, tmp_path: Path) -> None: + spec = _rendering_spec(tmp_path, "--log-level=DEBUG").with_plugins( + "logging", "capture" + ) + record = run_tests(self.create_module(), spec=spec, capture_output=True) + record.stdout.fnmatch_lines( [ "*___ test_foo [[]sub2[]] __*", "*-- Captured stdout call --*", @@ -829,55 +897,45 @@ def test_capturing_debug(self, pytester: pytest.Pytester) -> None: ] ) - def test_caplog(self, pytester: pytest.Pytester) -> None: - pytester.makepyfile( - """ - import logging + def test_caplog(self, tmp_path: Path) -> None: + def test(subtests: pytest.Subtests, caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.INFO) + logging.info("start test") - def test(subtests, caplog): - caplog.set_level(logging.INFO) - logging.info("start test") + with subtests.test("sub1"): + logging.info("inside %s", "subtest1") - with subtests.test("sub1"): - logging.info("inside %s", "subtest1") + assert len(caplog.records) == 2 + assert caplog.records[0].getMessage() == "start test" + assert caplog.records[1].getMessage() == "inside subtest1" - assert len(caplog.records) == 2 - assert caplog.records[0].getMessage() == "start test" - assert caplog.records[1].getMessage() == "inside subtest1" - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [ - "*1 passed*", - ] - ) + spec = _subtests_spec(tmp_path).with_plugins("logging") + record = run_tests(test, spec=spec, name="test_caplog") + record.assert_outcomes(passed=1) - def test_no_logging(self, pytester: pytest.Pytester) -> None: - pytester.makepyfile( - """ - import logging + def test_no_logging(self, tmp_path: Path) -> None: + def test(subtests: pytest.Subtests) -> None: + logging.info("start log line") - def test(subtests): - logging.info("start log line") + with subtests.test("sub passing"): + logging.info("inside %s", "passing log line") - with subtests.test("sub passing"): - logging.info("inside %s", "passing log line") + with subtests.test("sub failing"): + logging.info("inside %s", "failing log line") + assert False - with subtests.test("sub failing"): - logging.info("inside %s", "failing log line") - assert False + logging.info("end log line") - logging.info("end log line") - """ + # the original passed "-p no:logging"; an ensemble simply never loads + # the logging plugin in the first place. + record = run_tests( + test, + spec=_rendering_spec(tmp_path), + name="test_no_logging", + capture_output=True, ) - result = pytester.runpytest("-p no:logging") - result.stdout.fnmatch_lines( - [ - "*2 failed in*", - ] - ) - result.stdout.no_fnmatch_line("*root:test_no_logging.py*log line*") + record.assert_outcomes(failed=2) + record.stdout.no_fnmatch_line("*root:*log line*") class TestDebugging: @@ -902,34 +960,28 @@ def interaction(self, *_: object) -> None: def cleanup_calls(self) -> None: self._FakePdb.calls.clear() - def test_pdb_fixture( - self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch - ) -> None: - pytester.makepyfile( - """ - def test(subtests): - with subtests.test(): - assert 0 - """ - ) - self.runpytest_and_check_pdb(pytester, monkeypatch) + def test_pdb_fixture(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def test(subtests: pytest.Subtests) -> None: + with subtests.test(): + assert 0 + + self.run_and_check_pdb(test, tmp_path, monkeypatch) def test_pdb_unittest( - self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - pytester.makepyfile( - """ - from unittest import TestCase - class Test(TestCase): - def test(self): - with self.subTest(): - assert 0 - """ - ) - self.runpytest_and_check_pdb(pytester, monkeypatch) + class Test(unittest.TestCase): + def test(self) -> None: + with self.subTest(): + assert 0 + + self.run_and_check_pdb(Test, tmp_path, monkeypatch) - def runpytest_and_check_pdb( - self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + def run_and_check_pdb( + self, + source: Source, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: # Install the fake pdb implementation in _pytest.subtests so we can reference # it in the command line (any module would do). @@ -938,29 +990,35 @@ def runpytest_and_check_pdb( monkeypatch.setattr( _pytest.subtests, "_CustomPdb", self._FakePdb, raising=False ) - result = pytester.runpytest("--pdb", "--pdbcls=_pytest.subtests:_CustomPdb") + spec = _rendering_spec( + tmp_path, "--pdb", "--pdbcls=_pytest.subtests:_CustomPdb" + ).with_plugins("debugging") + record = run_tests(source, spec=spec, name="test_pdb", capture_output=True) # Ensure pytest entered in debugging mode when encountering the failing # assert. - result.stdout.fnmatch_lines("*entering PDB*") + record.stdout.fnmatch_lines("*entering PDB*") assert self._FakePdb.calls == ["init", "reset", "interaction"] -def test_exitfirst(pytester: pytest.Pytester) -> None: +def test_exitfirst(tmp_path: Path) -> None: """Validate that when passing --exitfirst the test exits after the first failed subtest.""" - pytester.makepyfile( - """ - def test_foo(subtests): - with subtests.test("sub1"): - assert False - with subtests.test("sub2"): - assert False - """ + def test_foo(subtests: pytest.Subtests) -> None: + with subtests.test("sub1"): + assert False + + with subtests.test("sub2"): + assert False + + record = run_tests( + test_foo, + spec=_rendering_spec(tmp_path, "--exitfirst"), + name="test_exitfirst", + capture_output=True, ) - result = pytester.runpytest("--exitfirst") - assert result.parseoutcomes()["failed"] == 2 - result.stdout.fnmatch_lines( + assert record.outcomes()["failed"] == 2 + record.stdout.fnmatch_lines( [ "SUBFAILED*[[]sub1[]] *.py::test_foo - assert False*", "FAILED *.py::test_foo - assert False", @@ -968,53 +1026,62 @@ def test_foo(subtests): ], consecutive=True, ) - result.stdout.no_fnmatch_line("*sub2*") # sub2 not executed. + record.stdout.no_fnmatch_line("*sub2*") # sub2 not executed. -def test_do_not_swallow_pytest_exit(pytester: pytest.Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def test(subtests): - with subtests.test(): - pytest.exit() +def test_do_not_swallow_pytest_exit(tmp_path: Path) -> None: + reports: list[TestReport] = [] - def test2(): pass - """ - ) - result = pytester.runpytest_subprocess() - result.stdout.fnmatch_lines( - [ - "* _pytest.outcomes.Exit *", - "* 1 failed in *", - ] - ) + class ReportRecorder: + def pytest_runtest_logreport(self, report: TestReport) -> None: + reports.append(report) + + def test(subtests: pytest.Subtests) -> None: + with subtests.test(): + pytest.exit() + + def test2() -> None: + pass + + module = build_module("test_do_not_swallow_pytest_exit", test, test2) + spec = _subtests_spec(tmp_path).with_plugins(ReportRecorder()) + # the original observed the Exit escaping as a subprocess traceback; here + # it simply has to come back out of the run. + with pytest.raises(Exit): + run_tests(module, spec=spec) + # the subtest was reported as failed, and ``test2`` never ran + assert [(report.when, report.outcome) for report in reports] == [ + ("setup", "passed"), + ("call", "failed"), + ] + assert isinstance(reports[-1], SubtestReport) + assert reports[-1].nodeid == "test_do_not_swallow_pytest_exit.py::test" -def test_nested(pytester: pytest.Pytester) -> None: +def test_nested(tmp_path: Path) -> None: """ Currently we do nothing special with nested subtests. This test only sediments how they work now, we might reconsider adding some kind of nesting support in the future. """ - pytester.makepyfile( - """ - import pytest - def test(subtests): - with subtests.test("a"): - with subtests.test("b"): - assert False, "b failed" - assert False, "a failed" - """ + + def test(subtests: pytest.Subtests) -> None: + with subtests.test("a"): + with subtests.test("b"): + assert False, "b failed" + assert False, "a failed" + + record = run_tests( + test, spec=_rendering_spec(tmp_path), name="test_nested", capture_output=True ) - result = pytester.runpytest_subprocess() - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "SUBFAILED[b] test_nested.py::test - AssertionError: b failed", "SUBFAILED[a] test_nested.py::test - AssertionError: a failed", "* 3 failed in *", ] ) + record.assert_outcomes(failed=3) class MyEnum(Enum): @@ -1048,6 +1115,8 @@ def test_serialization() -> None: ) +# ensemble: needs xdist, which reruns the tests in worker *subprocesses* over +# real files on disk (hence the syspathinsert). def test_serialization_xdist(pytester: pytest.Pytester) -> None: # pragma: no cover """Regression test for pytest-dev/pytest-xdist#1273.""" pytest.importorskip("xdist") From c949d725bba021dd925d1afe7045250a77d9ef47 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 20:52:55 +0200 Subject: [PATCH 11/30] testing: port test_session.py to _pytest.ensemble 10 of 24 pytester-based tests ported - a lower yield than the file's pytester count suggests, because much of it is bound to rootdir discovery and path-argument collection. --exitfirst and --maxfail were the interesting ones. run_items runs every item unconditionally; the early exit lives in pytest_runtestloop, which ensembles bypass, so a naive port would have run everything and asserted the opposite of the subject. They now drive the real hook and assert the exact surviving nodeid list plus the shouldfail message. The sticky-flag tests hit a related edge: RunRecord is built before pytest_sessionfinish, and the warnings those tests are about are emitted inside it, so assert_outcomes(warnings=1) would have been a silent false-green. Both use a pytest_warning_recorded impl on their plugin object instead. 14 stay on pytester: filesystem and path-argument collection (6), real module import (2), -p (2), rootdir discovery (2), pytest_collect_file (1), wrap_session cwd restore (1). --- testing/test_session.py | 482 +++++++++++++++++++++++++--------------- 1 file changed, 297 insertions(+), 185 deletions(-) diff --git a/testing/test_session.py b/testing/test_session.py index be1e66112d7..3184dc272cd 100644 --- a/testing/test_session.py +++ b/testing/test_session.py @@ -1,45 +1,74 @@ # mypy: allow-untyped-defs from __future__ import annotations +from pathlib import Path + from _pytest.config import ExitCode +from _pytest.ensemble import build_module +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import Ensemble +from _pytest.ensemble import run_tests +from _pytest.ensemble import RunRecord +from _pytest.main import Session from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester import pytest +def run_testloop(ensemble: Ensemble, expect: type[BaseException]) -> RunRecord: + """Drive the real ``pytest_runtestloop`` over an ensemble's items. + + ``run_items`` (and with it ``Ensemble.run``) runs every item it is given + unconditionally; the early exit on ``session.shouldfail``/``shouldstop`` + lives in ``pytest_runtestloop``, so tests about ``--exitfirst``, + ``--maxfail`` and friends have to go through that hook instead. The + reports are recorded on the config either way, so an empty final ``run`` + hands back everything the loop produced before it bailed out. + """ + ensemble.collect() + session = ensemble.session + with pytest.raises(expect): + session.config.hook.pytest_runtestloop(session=session) + return ensemble.run([]) + + class SessionTests: - def test_basic_testitem_events(self, pytester: Pytester) -> None: - tfile = pytester.makepyfile( - """ - def test_one(): - pass - def test_one_one(): - assert 0 - def test_other(): - raise ValueError(23) - class TestClass(object): - def test_two(self, someargs): - pass - """ - ) - reprec = pytester.inline_run(tfile) - passed, skipped, failed = reprec.listoutcomes() - assert len(skipped) == 0 - assert len(passed) == 1 - assert len(failed) == 3 + def test_basic_testitem_events(self, tmp_path: Path) -> None: + def test_one(): + pass - def end(x): - return x.nodeid.split("::")[-1] + def test_one_one(): + assert 0 - assert end(failed[0]) == "test_one_one" - assert end(failed[1]) == "test_other" - itemstarted = reprec.getcalls("pytest_itemcollected") - assert len(itemstarted) == 4 - # XXX check for failing funcarg setup - # colreports = reprec.getcalls("pytest_collectreport") - # assert len(colreports) == 4 - # assert colreports[1].report.failed + def test_other(): + raise ValueError(23) + + class TestClass: + def test_two(self, someargs): + pass + record = run_tests( + test_one, + test_one_one, + test_other, + TestClass, + rootpath=tmp_path, + name="test_basic_testitem_events", + ) + # HookRecorder.listoutcomes counted all three of these as failed; the + # missing ``someargs`` fixture fails at setup, which is a category of + # its own here. + record.assert_outcomes(passed=1, failed=2, errors=1) + # ... and one item collected per test, in definition order. + assert list(record.by_test) == [ + "test_basic_testitem_events.py::test_one", + "test_basic_testitem_events.py::test_one_one", + "test_basic_testitem_events.py::test_other", + "test_basic_testitem_events.py::TestClass::test_two", + ] + + # ensemble: the whole subject is a real module import failing on an import + # of another real module - the chokepoint EnsembleModule bypasses. def test_nested_import_error(self, pytester: Pytester) -> None: tfile = pytester.makepyfile( """ @@ -58,20 +87,20 @@ def test_this(): out = str(values[0].longrepr) assert out.find("does_not_work") != -1 - def test_raises_output(self, pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - import pytest - def test_raises_doesnt(): - with pytest.raises(ValueError): - int("3") - """ - ) - _passed, _skipped, failed = reprec.listoutcomes() - assert len(failed) == 1 - out = failed[0].longrepr.reprcrash.message # type: ignore[union-attr] + def test_raises_output(self, tmp_path: Path) -> None: + def test_raises_doesnt(): + with pytest.raises(ValueError): + int("3") + + record = run_tests(test_raises_doesnt, rootpath=tmp_path) + record.assert_outcomes(failed=1) + call = record["test_raises_doesnt"].call + assert call is not None + out = call.longrepr.reprcrash.message # type: ignore[union-attr] assert "DID NOT RAISE" in out + # ensemble: a module that is not python at all only exists as a file that + # fails to compile on import. def test_syntax_error_module(self, pytester: Pytester) -> None: reprec = pytester.inline_runsource("this is really not python") values = reprec.getfailedcollections() @@ -79,87 +108,102 @@ def test_syntax_error_module(self, pytester: Pytester) -> None: out = str(values[0].longrepr) assert out.find("not python") != -1 - def test_exit_first_problem(self, pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - def test_one(): assert 0 - def test_two(): assert 0 - """, - "--exitfirst", - ) - passed, skipped, failed = reprec.countoutcomes() - assert failed == 1 - assert passed == skipped == 0 + def test_exit_first_problem(self, tmp_path: Path) -> None: + def test_one(): + assert 0 - def test_maxfail(self, pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - def test_one(): assert 0 - def test_two(): assert 0 - def test_three(): assert 0 - """, - "--maxfail=2", - ) - passed, skipped, failed = reprec.countoutcomes() - assert failed == 2 - assert passed == skipped == 0 + def test_two(): + assert 0 - def test_broken_repr(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest + spec = ConfigSpec(rootpath=tmp_path, args=("--exitfirst",)) + with Ensemble(test_one, test_two, spec=spec, name="test_exitfirst") as ensemble: + record = run_testloop(ensemble, Session.Failed) + assert ensemble.session.shouldfail == "stopping after 1 failures" + record.assert_outcomes(failed=1) + # the second test never ran + assert list(record.by_test) == ["test_exitfirst.py::test_one"] - class reprexc(BaseException): - def __str__(self): - return "Ha Ha fooled you, I'm a broken repr()." + def test_maxfail(self, tmp_path: Path) -> None: + def test_one(): + assert 0 + + def test_two(): + assert 0 + + def test_three(): + assert 0 + + spec = ConfigSpec(rootpath=tmp_path, args=("--maxfail=2",)) + with Ensemble( + test_one, test_two, test_three, spec=spec, name="test_maxfail" + ) as ensemble: + record = run_testloop(ensemble, Session.Failed) + assert ensemble.session.shouldfail == "stopping after 2 failures" + record.assert_outcomes(failed=2) + # the third test never ran + assert list(record.by_test) == [ + "test_maxfail.py::test_one", + "test_maxfail.py::test_two", + ] - class BrokenRepr1(object): - foo=0 - def __repr__(self): - raise reprexc + def test_broken_repr(self, tmp_path: Path) -> None: + class reprexc(BaseException): + def __str__(self): + return "Ha Ha fooled you, I'm a broken repr()." - class TestBrokenClass(object): - def test_explicit_bad_repr(self): - t = BrokenRepr1() - with pytest.raises(BaseException, match="broken repr"): - repr(t) + class BrokenRepr1: + foo = 0 - def test_implicit_bad_repr1(self): - t = BrokenRepr1() - assert t.foo == 1 + def __repr__(self): + raise reprexc - """ - ) - reprec = pytester.inline_run(p) - passed, skipped, failed = reprec.listoutcomes() - assert (len(passed), len(skipped), len(failed)) == (1, 0, 1) - out = failed[0].longrepr.reprcrash.message # type: ignore[union-attr] - assert out.find("<[reprexc() raised in repr()] BrokenRepr1") != -1 + class TestBrokenClass: + def test_explicit_bad_repr(self): + t = BrokenRepr1() + with pytest.raises(BaseException, match="broken repr"): + repr(t) - def test_broken_repr_with_showlocals_verbose(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - class ObjWithErrorInRepr: - def __repr__(self): - raise NotImplementedError + def test_implicit_bad_repr1(self): + t = BrokenRepr1() + assert t.foo == 1 - def test_repr_error(): - x = ObjWithErrorInRepr() - assert x == "value" - """ - ) - reprec = pytester.inline_run("--showlocals", "-vv", p) - passed, skipped, failed = reprec.listoutcomes() - assert (len(passed), len(skipped), len(failed)) == (0, 0, 1) - entries = failed[0].longrepr.reprtraceback.reprentries # type: ignore[union-attr] + record = run_tests(TestBrokenClass, rootpath=tmp_path, name="test_broken_repr") + record.assert_outcomes(passed=1, failed=1) + call = record["test_implicit_bad_repr1"].call + assert call is not None + out = call.longrepr.reprcrash.message # type: ignore[union-attr] + assert out.find("<[reprexc() raised in repr()] BrokenRepr1") != -1 + + def test_broken_repr_with_showlocals_verbose(self, tmp_path: Path) -> None: + class ObjWithErrorInRepr: + def __repr__(self): + raise NotImplementedError + + def test_repr_error(): + x = ObjWithErrorInRepr() + assert x == "value" # type: ignore[comparison-overlap] + + # --showlocals and -vv are terminal options, so the terminal plugin + # has to be loaded; capture_output binds its stream to a private + # buffer instead of the outer stdout. + spec = ConfigSpec(rootpath=tmp_path, args=("--showlocals", "-vv")) + record = run_tests(test_repr_error, spec=spec, capture_output=True) + record.assert_outcomes(failed=1) + call = record["test_repr_error"].call + assert call is not None + entries = call.longrepr.reprtraceback.reprentries # type: ignore[union-attr] assert len(entries) == 1 repr_locals = entries[0].reprlocals assert repr_locals.lines - assert len(repr_locals.lines) == 1 - assert repr_locals.lines[0].startswith( + # ObjWithErrorInRepr is a closure cell here - a module global, and so + # not a local at all, in the original - hence the extra line. + assert len(repr_locals.lines) == 2 + assert repr_locals.lines[-1].startswith( "x = <[NotImplementedError() raised in repr()] ObjWithErrorInRepr" ) + # ensemble: pytest_collect_file never fires for an ensemble - its + # collection tree is preset, no path is ever walked. def test_skip_file_by_conftest(self, pytester: Pytester) -> None: pytester.makepyfile( conftest=""" @@ -182,32 +226,43 @@ def test_one(): pass class TestNewSession(SessionTests): - def test_order_of_execution(self, pytester: Pytester) -> None: - reprec = pytester.inline_runsource( - """ - values = [] - def test_1(): - values.append(1) - def test_2(): - values.append(2) - def test_3(): - assert values == [1,2] - class Testmygroup(object): - reslist = values - def test_1(self): - self.reslist.append(1) - def test_2(self): - self.reslist.append(2) - def test_3(self): - self.reslist.append(3) - def test_4(self): - assert self.reslist == [1,2,1,2,3] - """ + def test_order_of_execution(self, tmp_path: Path) -> None: + # a closure rather than a module global: ensemble sources keep this + # file's globals, which are emphatically not the ensemble's. + values: list[int] = [] + + def test_1(): + values.append(1) + + def test_2(): + values.append(2) + + def test_3(): + assert values == [1, 2] + + class Testmygroup: + reslist = values + + def test_1(self): + self.reslist.append(1) + + def test_2(self): + self.reslist.append(2) + + def test_3(self): + self.reslist.append(3) + + def test_4(self): + assert self.reslist == [1, 2, 1, 2, 3] + + record = run_tests( + test_1, test_2, test_3, Testmygroup, rootpath=tmp_path, name="test_order" ) - passed, skipped, failed = reprec.countoutcomes() - assert failed == skipped == 0 - assert passed == 7 + record.assert_outcomes(passed=7) + # ensemble: a directory argument, an __init__.py and a module that is not + # python - all filesystem, plus the collect event counting is about the + # tree an ensemble presets rather than walks. def test_collect_only_with_various_situations(self, pytester: Pytester) -> None: p = pytester.makepyfile( test_one=""" @@ -236,6 +291,8 @@ class TestY(TestX): colfail = [x for x in finished if x.failed] assert len(colfail) == 1 + # ensemble: import errors of real files, collected via a directory + # argument. def test_minus_x_import_error(self, pytester: Pytester) -> None: pytester.makepyfile(__init__="") pytester.makepyfile(test_one="xxxx", test_two="yyyy") @@ -244,6 +301,8 @@ def test_minus_x_import_error(self, pytester: Pytester) -> None: colfail = [x for x in finished if x.failed] assert len(colfail) == 1 + # ensemble: as above - import errors of real files behind a directory + # argument. def test_minus_x_overridden_by_maxfail(self, pytester: Pytester) -> None: pytester.makepyfile(__init__="") pytester.makepyfile(test_one="xxxx", test_two="yyyy", test_third="zzz") @@ -253,6 +312,11 @@ def test_minus_x_overridden_by_maxfail(self, pytester: Pytester) -> None: assert len(colfail) == 2 +# ensemble: ``-p`` is handled by PytestPluginManager.consider_preparse, which +# is part of Config._preparse; an ensemble config builds its plugin set from +# the spec instead and never runs it, so ``args=("-p", ...)`` lands in +# config.option.plugins and is then ignored - the test would pass without +# testing anything. def test_plugin_specify(pytester: Pytester) -> None: with pytest.raises(ImportError): pytester.parseconfig("-p", "nqweotexistent") @@ -261,6 +325,7 @@ def test_plugin_specify(pytester: Pytester) -> None: # ) +# ensemble: same as above - ``-p`` is never consumed by an ensemble config. def test_plugin_already_exists(pytester: Pytester) -> None: config = pytester.parseconfig("-p", "terminal") assert config.option.plugins == ["terminal"] @@ -268,6 +333,7 @@ def test_plugin_already_exists(pytester: Pytester) -> None: config._ensure_unconfigure() +# ensemble: --ignore excludes filesystem paths from a directory walk. def test_exclude(pytester: Pytester) -> None: hellodir = pytester.mkdir("hello") hellodir.joinpath("test_hello.py").write_text("x y syntaxerror", encoding="utf-8") @@ -279,6 +345,7 @@ def test_exclude(pytester: Pytester) -> None: result.stdout.fnmatch_lines(["*1 passed*"]) +# ensemble: as above, --ignore-glob is about the directory walk. def test_exclude_glob(pytester: Pytester) -> None: hellodir = pytester.mkdir("hello") hellodir.joinpath("test_hello.py").write_text("x y syntaxerror", encoding="utf-8") @@ -294,34 +361,42 @@ def test_exclude_glob(pytester: Pytester) -> None: result.stdout.fnmatch_lines(["*1 passed*"]) -def test_deselect(pytester: Pytester) -> None: - pytester.makepyfile( - test_a=""" - import pytest +def test_deselect(tmp_path: Path) -> None: + def test_a1(): + pass - def test_a1(): pass + @pytest.mark.parametrize("b", range(3)) + def test_a2(b): + pass - @pytest.mark.parametrize('b', range(3)) - def test_a2(b): pass + class TestClass: + def test_c1(self): + pass - class TestClass: - def test_c1(self): pass + def test_c2(self): + pass - def test_c2(self): pass - """ + module = build_module("test_a", test_a1, test_a2, TestClass) + spec = ConfigSpec( + rootpath=tmp_path, + args=( + "-v", + "--deselect=test_a.py::test_a2[1]", + "--deselect=test_a.py::test_a2[2]", + "--deselect=test_a.py::TestClass::test_c1", + ), ) - result = pytester.runpytest( - "-v", - "--deselect=test_a.py::test_a2[1]", - "--deselect=test_a.py::test_a2[2]", - "--deselect=test_a.py::TestClass::test_c1", - ) - assert result.ret == 0 - result.stdout.fnmatch_lines(["*3 passed, 3 deselected*"]) - for line in result.stdout.lines: + record = run_tests(module, spec=spec, capture_output=True) + record.assert_outcomes(passed=3, deselected=3) + # the verbose listing really is there, so the loop below has something to + # not find + record.stdout.fnmatch_lines(["test_a.py::test_a2[[]0[]] PASSED*"]) + for line in record.stdout.lines: assert not line.startswith(("test_a.py::test_a2[1]", "test_a.py::test_a2[2]")) +# ensemble: the cwd is restored by wrap_session, which an ensemble does not go +# through (and the subject of the assertion is the exit status). def test_sessionfinish_with_start(pytester: Pytester) -> None: pytester.makeconftest( """ @@ -340,6 +415,9 @@ def pytest_sessionfinish(): assert res.ret == ExitCode.NO_TESTS_COLLECTED +# ensemble: collection driven by path arguments (and --keep-duplicates over a +# directory given twice); an ensemble's collection tree is preset, so there +# are no initial arguments to deduplicate. def test_collection_args_do_not_duplicate_modules(pytester: Pytester) -> None: """Test that when multiple collection args are specified on the command line for the same module, only a single Module collector is created. @@ -390,6 +468,8 @@ def test_2(): pass ) +# ensemble: --rootdir feeds rootdir *discovery*, which an ensemble config +# skips entirely - its rootpath is handed to it. @pytest.mark.parametrize("path", ["root", "{relative}/root", "{environment}/root"]) def test_rootdir_option_arg( pytester: Pytester, monkeypatch: MonkeyPatch, path: str @@ -417,6 +497,7 @@ def test_one(): ) +# ensemble: as above, plus the assertion is on stderr of a real run. def test_rootdir_wrong_option_arg(pytester: Pytester) -> None: result = pytester.runpytest("--rootdir=wrong_dir") result.stderr.fnmatch_lines( @@ -424,61 +505,92 @@ def test_rootdir_wrong_option_arg(pytester: Pytester) -> None: ) -def test_shouldfail_is_sticky(pytester: Pytester) -> None: +def test_shouldfail_is_sticky(tmp_path: Path) -> None: """Test that session.shouldfail cannot be reset to False after being set. Issue #11706. """ - pytester.makeconftest( - """ - def pytest_sessionfinish(session): + recorded_warnings: list[str] = [] + + class ConftestPlugin: + def pytest_sessionfinish(self, session): assert session.shouldfail session.shouldfail = False assert session.shouldfail - """ - ) - pytester.makepyfile( - """ - import pytest - def test_foo(): - pytest.fail("This is a failing test") + # the RunRecord is built before pytest_sessionfinish runs, so the + # warning raised in there is recorded here instead. + def pytest_warning_recorded(self, warning_message): + recorded_warnings.append(str(warning_message.message)) - def test_bar(): pass - """ - ) + def test_foo(): + pytest.fail("This is a failing test") - result = pytester.runpytest("--maxfail=1", "-Wall") + def test_bar(): + pass - result.assert_outcomes(failed=1, warnings=1) - result.stdout.fnmatch_lines("*session.shouldfail cannot be unset*") + spec = ConfigSpec( + rootpath=tmp_path, + args=("--maxfail=1",), + # -Wall in the original; the host's ``filterwarnings = error`` would + # otherwise turn the warning into an exception out of sessionfinish. + inicfg={"filterwarnings": ["always"]}, + extra_plugins=(ConftestPlugin(),), + ) + with Ensemble(test_foo, test_bar, spec=spec, name="test_shouldfail") as ensemble: + record = run_testloop(ensemble, Session.Failed) + + record.assert_outcomes(failed=1) + assert recorded_warnings == [ + "session.shouldfail cannot be unset after it has been set; ignoring." + ] -def test_shouldstop_is_sticky(pytester: Pytester) -> None: +def test_shouldstop_is_sticky(tmp_path: Path) -> None: """Test that session.shouldstop cannot be reset to False after being set. Issue #11706. """ - pytester.makeconftest( - """ - def pytest_sessionfinish(session): + recorded_warnings: list[str] = [] + + class ConftestPlugin: + session: Session + + def pytest_sessionstart(self, session): + self.session = session + + # --stepwise sets shouldstop in the original; the stepwise plugin + # needs the cache and terminal plugins, and what is under test is the + # setter, so the flag is set the same way stepwise sets it. + def pytest_runtest_logreport(self, report): + if report.failed: + self.session.shouldstop = ( + "Test failed, continuing from this test next run." + ) + + def pytest_sessionfinish(self, session): assert session.shouldstop session.shouldstop = False assert session.shouldstop - """ - ) - pytester.makepyfile( - """ - import pytest - def test_foo(): - pytest.fail("This is a failing test") + def pytest_warning_recorded(self, warning_message): + recorded_warnings.append(str(warning_message.message)) - def test_bar(): pass - """ - ) + def test_foo(): + pytest.fail("This is a failing test") - result = pytester.runpytest("--stepwise", "-Wall") + def test_bar(): + pass + + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={"filterwarnings": ["always"]}, + extra_plugins=(ConftestPlugin(),), + ) + with Ensemble(test_foo, test_bar, spec=spec, name="test_shouldstop") as ensemble: + record = run_testloop(ensemble, Session.Interrupted) - result.assert_outcomes(failed=1, warnings=1) - result.stdout.fnmatch_lines("*session.shouldstop cannot be unset*") + record.assert_outcomes(failed=1) + assert recorded_warnings == [ + "session.shouldstop cannot be unset after it has been set; ignoring." + ] From 6b2869588652fa027892625cfd2a0e45132b513d Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 20:53:18 +0200 Subject: [PATCH 12/30] testing: port test_assertion.py to _pytest.ensemble 33 of 54 pytester-based tests ported; the 98 that already unit-test the assertion internals directly are untouched. A rewritten assertion renders essentially byte-identical to a real run - same >/E prefixes, same explanation, same Full diff, same short-summary echo. The traceback formatter dedents the source, so exact-indent patterns transfer unchanged however deeply the source function is nested in the host test body. The decisive finding is that an ensemble needs the `assertion` plugin loaded, and it is not in DEFAULT_PLUGINS. Without it util._reprcompare stays bound to whatever the *host* run installed: an explanation is still produced, so tests go green, but the ensemble's own -vv, ini values and pytest_assertrepr_compare implementations are silently ignored. Swapping "assertion" for "terminal" in the local spec helper makes 16 of the 33 ports fail, so they are load-bearing rather than vacuous. 21 stay on pytester: rewriting of files on import (7), assertions on the failing source's file:line (3), rewrite-time warnings (2), module-file semantics of the rewriter (2), and seven one-offs. --- testing/test_assertion.py | 1013 +++++++++++++++++++++++-------------- 1 file changed, 625 insertions(+), 388 deletions(-) diff --git a/testing/test_assertion.py b/testing/test_assertion.py index e473233b208..b06f1c0828a 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -5,6 +5,7 @@ from collections.abc import Mapping from collections.abc import MutableSequence import dataclasses +from pathlib import Path import sys import textwrap from typing import Any @@ -25,11 +26,31 @@ from _pytest.assertion.compare_text import _compare_eq_text from _pytest.assertion.compare_text import _notin_text from _pytest.config import Config as _Config +from _pytest.config import UsageError +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import run_tests from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester import pytest +def assertion_spec(rootpath: Path, *args: str, **inicfg: str) -> ConfigSpec: + """A nested config that renders assertion explanations like a real run. + + The ``assertion`` plugin is not among the ensemble defaults, and without + it ``util._reprcompare`` stays bound to whatever the *host* run installed: + an explanation is still produced (the sources in this file are rewritten + by the host at import time), but its verbosity, its ini settings and the + ``pytest_assertrepr_compare`` implementations consulted are the host's, + not the ensemble's. Loading the plugin is safe here because the rewrite + import hook is installed from ``Config._preparse``, which an ensemble + never runs. + """ + return ConfigSpec(rootpath=rootpath, args=args, inicfg=inicfg).with_plugins( + "assertion" + ) + + def mock_config( verbose: int = 0, assertion_override: int | None = None, @@ -124,6 +145,10 @@ def test_getini_unsupported_error(self): config.getini("--- NOT AN INI ---") +# ensemble: every test in this class is about the assertion *rewriting* of +# files on import - conftests, plugin modules, installed distributions - which +# an ensemble cannot reproduce: its sources are objects the host already +# rewrote, and it never installs an import hook of its own. class TestImportHookInstallation: @pytest.mark.parametrize("initial_conftest", [True, False]) @pytest.mark.parametrize("mode", ["plain", "rewrite"]) @@ -427,29 +452,34 @@ def test_register_assert_rewrite_checks_types(self) -> None: class TestBinReprIntegration: - def test_pytest_assertrepr_compare_called(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - values = [] - def pytest_assertrepr_compare(op, left, right): + def test_pytest_assertrepr_compare_called(self, tmp_path: Path) -> None: + values: list[tuple[str, object, object]] = [] + + class Conftest: + def pytest_assertrepr_compare(self, op, left, right): values.append((op, left, right)) - @pytest.fixture - def list(request): + @pytest.fixture(name="list") + def list_fixture(self, request): return values - """ - ) - pytester.makepyfile( - """ - def test_hello(): - assert 0 == 1 - def test_check(list): - assert list == [("==", 0, 1)] - """ + + def test_hello(): + assert 0 == 1 # type: ignore[comparison-overlap] + + def test_check(list): + assert list == [("==", 0, 1)] + + record = run_tests( + test_hello, + test_check, + spec=assertion_spec(tmp_path).with_plugins(Conftest()), ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines(["*test_hello*FAIL*", "*test_check*PASS*"]) + # The "-v" lines the original matched only carried the two outcomes; + # assert them as reports, plus what the hook actually saw. + record.assert_outcomes(failed=1, passed=1) + assert record["test_hello"].failed + assert record["test_check"].passed + assert values == [("==", 0, 1)] def callop( @@ -760,30 +790,29 @@ def test_iterable_quiet(self) -> None: ] def test_iterable_full_diff_ci( - self, monkeypatch: MonkeyPatch, pytester: Pytester + self, monkeypatch: MonkeyPatch, tmp_path: Path ) -> None: - pytester.makepyfile( - r""" - def test_full_diff(): - left = [0, 1] - right = [0, 2] - assert left == right - """ - ) + def test_full_diff(): + left = [0, 1] + right = [0, 2] + assert left == right + + def run(): + return run_tests( + test_full_diff, spec=assertion_spec(tmp_path), capture_output=True + ) + monkeypatch.setenv("CI", "true") - result = pytester.runpytest() - result.stdout.fnmatch_lines( + run().stdout.fnmatch_lines( ["E Full diff: (-: missing in left side, +: extra in left side)"] ) # Setting CI to empty string is same as having it undefined monkeypatch.setenv("CI", "") - result = pytester.runpytest() - result.stdout.fnmatch_lines(["E Use -v to get more diff"]) + run().stdout.fnmatch_lines(["E Use -v to get more diff"]) monkeypatch.delenv("CI", raising=False) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["E Use -v to get more diff"]) + run().stdout.fnmatch_lines(["E Use -v to get more diff"]) def test_list_different_lengths(self) -> None: expl = callequal([0, 1], [0, 1, 2]) @@ -1239,12 +1268,55 @@ def test_nfc_nfd_same_string(self) -> None: ] +# Mirrors ``example_scripts/dataclasses/test_compare_recursive_dataclasses.py``. +# These live at module level rather than inside the test because the expected +# output contains their reprs, and a dataclass repr is built from +# ``__qualname__`` - a class nested in a test function would render as +# ``TestX.test_y..S(...)``. +@dataclasses.dataclass +class S: + a: int + b: str + + +@dataclasses.dataclass +class C: + c: S + d: S + + +@dataclasses.dataclass +class C2: + e: C + f: S + + +@dataclasses.dataclass +class C3: + g: S + h: C2 + i: str + j: str + + class TestAssert_reprcompare_dataclass: - def test_dataclasses(self, pytester: Pytester) -> None: - p = pytester.copy_example("dataclasses/test_compare_dataclasses.py") - result = pytester.runpytest(p) - result.assert_outcomes(failed=1, passed=0) - result.stdout.fnmatch_lines( + def test_dataclasses(self, tmp_path: Path) -> None: + def test_dataclasses() -> None: + @dataclasses.dataclass + class SimpleDataObject: + field_a: int = dataclasses.field() + field_b: str = dataclasses.field() + + left = SimpleDataObject(1, "b") + right = SimpleDataObject(1, "c") + + assert left == right + + record = run_tests( + test_dataclasses, spec=assertion_spec(tmp_path), capture_output=True + ) + record.assert_outcomes(failed=1, passed=0) + record.stdout.fnmatch_lines( [ "E Omitting 1 identical items, use -vv to show", "E Differing attributes:", @@ -1258,11 +1330,30 @@ def test_dataclasses(self, pytester: Pytester) -> None: consecutive=True, ) - def test_recursive_dataclasses(self, pytester: Pytester) -> None: - p = pytester.copy_example("dataclasses/test_compare_recursive_dataclasses.py") - result = pytester.runpytest(p) - result.assert_outcomes(failed=1, passed=0) - result.stdout.fnmatch_lines( + def test_recursive_dataclasses(self, tmp_path: Path) -> None: + def test_recursive_dataclasses(): + left = C3( + S(10, "ten"), + C2(C(S(1, "one"), S(2, "two")), S(2, "three")), + "equal", + "left", + ) + right = C3( + S(20, "xxx"), + C2(C(S(1, "one"), S(2, "yyy")), S(3, "three")), + "equal", + "right", + ) + + assert left == right + + record = run_tests( + test_recursive_dataclasses, + spec=assertion_spec(tmp_path), + capture_output=True, + ) + record.assert_outcomes(failed=1, passed=0) + record.stdout.fnmatch_lines( [ "E Omitting 1 identical items, use -vv to show", "E Differing attributes:", @@ -1276,11 +1367,30 @@ def test_recursive_dataclasses(self, pytester: Pytester) -> None: consecutive=True, ) - def test_recursive_dataclasses_verbose(self, pytester: Pytester) -> None: - p = pytester.copy_example("dataclasses/test_compare_recursive_dataclasses.py") - result = pytester.runpytest(p, "-vv") - result.assert_outcomes(failed=1, passed=0) - result.stdout.fnmatch_lines( + def test_recursive_dataclasses_verbose(self, tmp_path: Path) -> None: + def test_recursive_dataclasses(): + left = C3( + S(10, "ten"), + C2(C(S(1, "one"), S(2, "two")), S(2, "three")), + "equal", + "left", + ) + right = C3( + S(20, "xxx"), + C2(C(S(1, "one"), S(2, "yyy")), S(3, "three")), + "equal", + "right", + ) + + assert left == right + + record = run_tests( + test_recursive_dataclasses, + spec=assertion_spec(tmp_path, "-vv"), + capture_output=True, + ) + record.assert_outcomes(failed=1, passed=0) + record.stdout.fnmatch_lines( [ "E Matching attributes:", "E ['i']", @@ -1306,11 +1416,25 @@ def test_recursive_dataclasses_verbose(self, pytester: Pytester) -> None: consecutive=True, ) - def test_dataclasses_verbose(self, pytester: Pytester) -> None: - p = pytester.copy_example("dataclasses/test_compare_dataclasses_verbose.py") - result = pytester.runpytest(p, "-vv") - result.assert_outcomes(failed=1, passed=0) - result.stdout.fnmatch_lines( + def test_dataclasses_verbose(self, tmp_path: Path) -> None: + def test_dataclasses_verbose() -> None: + @dataclasses.dataclass + class SimpleDataObject: + field_a: int = dataclasses.field() + field_b: str = dataclasses.field() + + left = SimpleDataObject(1, "b") + right = SimpleDataObject(1, "c") + + assert left == right + + record = run_tests( + test_dataclasses_verbose, + spec=assertion_spec(tmp_path, "-vv"), + capture_output=True, + ) + record.assert_outcomes(failed=1, passed=0) + record.stdout.fnmatch_lines( [ "*Matching attributes:*", "*['field_a']*", @@ -1319,37 +1443,91 @@ def test_dataclasses_verbose(self, pytester: Pytester) -> None: ] ) - def test_dataclasses_with_attribute_comparison_off( - self, pytester: Pytester - ) -> None: - p = pytester.copy_example( - "dataclasses/test_compare_dataclasses_field_comparison_off.py" - ) - result = pytester.runpytest(p, "-vv") - result.assert_outcomes(failed=0, passed=1) + def test_dataclasses_with_attribute_comparison_off(self, tmp_path: Path) -> None: + def test_dataclasses_with_attribute_comparison_off() -> None: + @dataclasses.dataclass + class SimpleDataObject: + field_a: int = dataclasses.field() + field_b: str = dataclasses.field(compare=False) + + left = SimpleDataObject(1, "b") + right = SimpleDataObject(1, "c") + + assert left == right - def test_comparing_two_different_data_classes(self, pytester: Pytester) -> None: - p = pytester.copy_example( - "dataclasses/test_compare_two_different_dataclasses.py" + record = run_tests( + test_dataclasses_with_attribute_comparison_off, + spec=assertion_spec(tmp_path, "-vv"), + # "-vv" is a terminal plugin option, which capturing loads. + capture_output=True, ) - result = pytester.runpytest(p, "-vv") - result.assert_outcomes(failed=0, passed=1) + record.assert_outcomes(failed=0, passed=1) - def test_data_classes_with_custom_eq(self, pytester: Pytester) -> None: - p = pytester.copy_example( - "dataclasses/test_compare_dataclasses_with_custom_eq.py" + def test_comparing_two_different_data_classes(self, tmp_path: Path) -> None: + def test_comparing_two_different_data_classes() -> None: + @dataclasses.dataclass + class SimpleDataObjectOne: + field_a: int = dataclasses.field() + field_b: str = dataclasses.field() + + @dataclasses.dataclass + class SimpleDataObjectTwo: + field_a: int = dataclasses.field() + field_b: str = dataclasses.field() + + left = SimpleDataObjectOne(1, "b") + right = SimpleDataObjectTwo(1, "c") + + assert left != right # type: ignore[comparison-overlap] + + record = run_tests( + test_comparing_two_different_data_classes, + spec=assertion_spec(tmp_path, "-vv"), + capture_output=True, ) + record.assert_outcomes(failed=0, passed=1) + + def test_data_classes_with_custom_eq(self, tmp_path: Path) -> None: + def test_dataclasses() -> None: + @dataclasses.dataclass + class SimpleDataObject: + field_a: int = dataclasses.field() + field_b: str = dataclasses.field() + + def __eq__(self, o: object, /) -> bool: + return super().__eq__(o) + + left = SimpleDataObject(1, "b") + right = SimpleDataObject(1, "c") + + assert left == right + # issue 9362 - result = pytester.runpytest(p, "-vv") - result.assert_outcomes(failed=1, passed=0) - result.stdout.no_re_match_line(".*Differing attributes.*") + record = run_tests( + test_dataclasses, + spec=assertion_spec(tmp_path, "-vv"), + capture_output=True, + ) + record.assert_outcomes(failed=1, passed=0) + record.stdout.no_re_match_line(".*Differing attributes.*") + + def test_data_classes_with_initvar(self, tmp_path: Path) -> None: + @dataclasses.dataclass + class Foo: + init_only: dataclasses.InitVar[int] + real_attr: int + + def test_demonstrate(): + assert Foo(1, 2) == Foo(1, 3) - def test_data_classes_with_initvar(self, pytester: Pytester) -> None: - p = pytester.copy_example("dataclasses/test_compare_initvar.py") # issue 9820 - result = pytester.runpytest(p, "-vv") - result.assert_outcomes(failed=1, passed=0) - result.stdout.no_re_match_line(".*AttributeError.*") + record = run_tests( + test_demonstrate, + spec=assertion_spec(tmp_path, "-vv"), + capture_output=True, + ) + record.assert_outcomes(failed=1, passed=0) + record.stdout.no_re_match_line(".*AttributeError.*") class TestAssert_reprcompare_attrsclass: @@ -1536,17 +1714,15 @@ class NT2(NamedTuple): class TestFormatExplanation: - def test_special_chars_full(self, pytester: Pytester) -> None: + def test_special_chars_full(self, tmp_path: Path) -> None: # Issue 453, for the bug this would raise IndexError - pytester.makepyfile( - """ - def test_foo(): - assert '\\n}' == '' - """ - ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines(["*AssertionError*"]) + def test_foo(): + assert "\n}" == "" # type: ignore[comparison-overlap] + + record = run_tests(test_foo, spec=assertion_spec(tmp_path), capture_output=True) + # ``ret == 1`` is TESTS_FAILED; assert the failure itself. + record.assert_outcomes(failed=1) + record.stdout.fnmatch_lines(["*AssertionError*"]) def test_fmt_simple(self) -> None: expl = "assert foo" @@ -1740,25 +1916,29 @@ def test_truncates_at_1_line_when_first_line_is_GT_max_chars(self) -> None: last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") - def test_full_output_truncated(self, monkeypatch, pytester: Pytester) -> None: - """Test against full runpytest() output.""" + def test_full_output_truncated(self, monkeypatch, tmp_path: Path) -> None: + """Test against the full rendered ensemble output.""" line_count = 7 line_len = 100 - pytester.makepyfile( - rf""" - def test_many_lines(): - a = list([str(i)[0] * {line_len} for i in range({line_count})]) - b = a[::2] - a = '\n'.join(map(str, a)) - b = '\n'.join(map(str, b)) - assert a == b - """ - ) + + def test_many_lines(): + lines_a = [str(i)[0] * line_len for i in range(line_count)] + lines_b = lines_a[::2] + a = "\n".join(map(str, lines_a)) + b = "\n".join(map(str, lines_b)) + assert a == b + + def run(*args: str): + return run_tests( + test_many_lines, + spec=assertion_spec(tmp_path, *args), + capture_output=True, + ) + monkeypatch.delenv("CI", raising=False) - result = pytester.runpytest() # without -vv, truncate the message showing a few diff lines only - result.stdout.fnmatch_lines( + run().stdout.fnmatch_lines( [ "*+ 1*", "*+ 3*", @@ -1766,13 +1946,11 @@ def test_many_lines(): ] ) - result = pytester.runpytest("-vv") - result.stdout.fnmatch_lines(["* 6*"]) + run("-vv").stdout.fnmatch_lines(["* 6*"]) # Setting CI to empty string is same as having it undefined monkeypatch.setenv("CI", "") - result = pytester.runpytest() - result.stdout.fnmatch_lines( + run().stdout.fnmatch_lines( [ "*+ 1*", "*+ 3*", @@ -1781,8 +1959,7 @@ def test_many_lines(): ) monkeypatch.setenv("CI", "1") - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 6*"]) + run().stdout.fnmatch_lines(["* 6*"]) @pytest.mark.parametrize( ["truncation_lines", "truncation_chars", "expected_lines_hidden"], @@ -1801,20 +1978,17 @@ def test_many_lines(): def test_truncation_with_ini( self, monkeypatch, - pytester: Pytester, + tmp_path: Path, truncation_lines: int | None, truncation_chars: int | None, expected_lines_hidden: int, ) -> None: - pytester.makepyfile( - """\ - string_a = "123456789\\n23456789\\n3" - string_b = "123456789\\n23456789\\n4" - - def test(): - assert string_a == string_b - """ - ) + # Module-level globals in the original; ensemble sources share the + # *host* module's globals, so these have to be locals. + def test(): + string_a = "123456789\n23456789\n3" + string_b = "123456789\n23456789\n4" + assert string_a == string_b # This test produces 6 lines of diff output or 79 characters # So the effect should be when threshold is < 4 lines (considering 2 additional lines for explanation) @@ -1822,20 +1996,21 @@ def test(): monkeypatch.delenv("CI", raising=False) - ini = "[pytest]\n" + inicfg = {} if truncation_lines is not None: - ini += f"truncation_limit_lines = {truncation_lines}\n" + inicfg["truncation_limit_lines"] = str(truncation_lines) if truncation_chars is not None: - ini += f"truncation_limit_chars = {truncation_chars}\n" - pytester.makeini(ini) + inicfg["truncation_limit_chars"] = str(truncation_chars) - result = pytester.runpytest() + record = run_tests( + test, spec=assertion_spec(tmp_path, **inicfg), capture_output=True + ) if expected_lines_hidden != 0: - result.stdout.fnmatch_lines(["*Full output truncated*"]) + record.stdout.fnmatch_lines(["*Full output truncated*"]) else: - result.stdout.no_fnmatch_line("*truncated*") - result.stdout.fnmatch_lines( + record.stdout.no_fnmatch_line("*truncated*") + record.stdout.fnmatch_lines( [ "*- 4*", "*+ 3*", @@ -1883,6 +2058,9 @@ def test(): ), ], ) + # ensemble: the subject is how a ``pyproject.toml`` section is parsed into + # ini values, and an ensemble's ``inicfg`` is authoritative - no config + # file is ever read. def test_truncation_limits_accept_int_and_string( self, monkeypatch, pytester: Pytester, config: str ) -> None: @@ -2156,6 +2334,8 @@ def test_no_terminalreporter_uses_plaintext_highlighter(self) -> None: assert not any("\x1b[" in line for line in result) +# ensemble: the subject is the rewriter compiling a *module* whose last line is +# a comment; an ensemble source is a function object the host already compiled. def test_python25_compile_issue257(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -2174,6 +2354,9 @@ def test_rewritten(): ) +# ensemble: an ensemble source's globals are this (already rewritten) module's +# globals, so "@py_builtins" would be found no matter what - a green test that +# asserts nothing. def test_rewritten(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -2243,32 +2426,62 @@ def test_reprcompare_whitespaces() -> None: class TestSetAssertions: @pytest.mark.parametrize("op", [">=", ">", "<=", "<", "=="]) - def test_set_extra_item(self, op, pytester: Pytester) -> None: - pytester.makepyfile( - f""" + def test_set_extra_item(self, op, tmp_path: Path) -> None: + # One source per operator: the rendered ``assert`` line is what is + # asserted on, so the operator has to be in the source text itself. + if op == ">=": + def test_hello(): x = set("hello x") y = set("hello y") - assert x {op} y - """ - ) + assert x >= y - result = pytester.runpytest() - result.stdout.fnmatch_lines( + elif op == ">": + + def test_hello(): + x = set("hello x") + y = set("hello y") + assert x > y + + elif op == "<=": + + def test_hello(): + x = set("hello x") + y = set("hello y") + assert x <= y + + elif op == "<": + + def test_hello(): + x = set("hello x") + y = set("hello y") + assert x < y + + else: + + def test_hello(): + x = set("hello x") + y = set("hello y") + assert x == y + + record = run_tests( + test_hello, spec=assertion_spec(tmp_path), capture_output=True + ) + record.stdout.fnmatch_lines( [ "*def test_hello():*", f"*assert x {op} y*", ] ) if op in [">=", ">", "=="]: - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*E*Extra items in the right set:*", "*E*'y'", ] ) if op in ["<=", "<", "=="]: - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*E*Extra items in the left set:*", "*E*'x'", @@ -2276,18 +2489,32 @@ def test_hello(): ) @pytest.mark.parametrize("op", [">", "<", "!="]) - def test_set_proper_superset_equal(self, pytester: Pytester, op) -> None: - pytester.makepyfile( - f""" + def test_set_proper_superset_equal(self, tmp_path: Path, op) -> None: + if op == ">": + def test_hello(): x = set([1, 2, 3]) y = x.copy() - assert x {op} y - """ - ) + assert x > y - result = pytester.runpytest() - result.stdout.fnmatch_lines( + elif op == "<": + + def test_hello(): + x = set([1, 2, 3]) + y = x.copy() + assert x < y + + else: + + def test_hello(): + x = set([1, 2, 3]) + y = x.copy() + assert x != y + + record = run_tests( + test_hello, spec=assertion_spec(tmp_path), capture_output=True + ) + record.stdout.fnmatch_lines( [ "*def test_hello():*", f"*assert x {op} y*", @@ -2295,18 +2522,17 @@ def test_hello(): ] ) - def test_pytest_assertrepr_compare_integration(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_hello(): - x = set(range(100)) - y = x.copy() - y.remove(50) - assert x == y - """ + def test_pytest_assertrepr_compare_integration(self, tmp_path: Path) -> None: + def test_hello(): + x = set(range(100)) + y = x.copy() + y.remove(50) + assert x == y + + record = run_tests( + test_hello, spec=assertion_spec(tmp_path), capture_output=True ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*def test_hello():*", "*assert x == y*", @@ -2317,29 +2543,27 @@ def test_hello(): ) @pytest.mark.parametrize("op", [">=", "<="]) - def test_dict_items_view_subset(self, op, pytester: Pytester) -> None: + def test_dict_items_view_subset(self, op, tmp_path: Path) -> None: """dict.items() supports set-like comparisons; assert diff should show the missing items.""" if op == ">=": - pytester.makepyfile( - """ - def test_hello(): - x = {"a": 1, "b": 2} - y = {"a": 1, "b": 2, "c": 3} - assert x.items() >= y.items() - """ - ) + + def test_hello(): + x = {"a": 1, "b": 2} + y = {"a": 1, "b": 2, "c": 3} + assert x.items() >= y.items() + else: - pytester.makepyfile( - """ - def test_hello(): - x = {"a": 1, "b": 2, "c": 3} - y = {"a": 1, "b": 2} - assert x.items() <= y.items() - """ - ) - result = pytester.runpytest() + + def test_hello(): + x = {"a": 1, "b": 2, "c": 3} + y = {"a": 1, "b": 2} + assert x.items() <= y.items() + + record = run_tests( + test_hello, spec=assertion_spec(tmp_path), capture_output=True + ) side = "right" if op == ">=" else "left" - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*def test_hello():*", f"*assert x.items() {op} y.items()*", @@ -2349,6 +2573,8 @@ def test_hello(): ) +# ensemble: three sibling directories, each with its own conftest; conftest +# visibility is per-directory and an ensemble has no directory tree. def test_assertrepr_loaded_per_dir(pytester: Pytester) -> None: pytester.makepyfile(test_base=["def test_base(): assert 1 == 2"]) a = pytester.mkdir("a") @@ -2375,6 +2601,8 @@ def test_assertrepr_loaded_per_dir(pytester: Pytester) -> None: ) +# ensemble: the subject is that "--assert=plain" turns the *rewriting* off, and +# ensemble sources are rewritten by the host whatever the nested config says. def test_assertion_options(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -2389,18 +2617,23 @@ def test_hello(): result.stdout.no_fnmatch_line("*3 == 4*") -def test_triple_quoted_string_issue113(pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_hello(): - assert "" == ''' - '''""" +def test_triple_quoted_string_issue113(tmp_path: Path) -> None: + def test_hello(): + assert ( + "" # type: ignore[comparison-overlap] + == """ +""" + ) + + record = run_tests( + test_hello, spec=assertion_spec(tmp_path, "--fulltrace"), capture_output=True ) - result = pytester.runpytest("--fulltrace") - result.stdout.fnmatch_lines(["*1 failed*"]) - result.stdout.no_fnmatch_line("*SyntaxError*") + record.stdout.fnmatch_lines(["*1 failed*"]) + record.stdout.no_fnmatch_line("*SyntaxError*") +# ensemble: every expected line is a "file:line" of the failing source, which +# is host-anchored (this file, at its own absolute line numbers). def test_traceback_failure(pytester: Pytester) -> None: p1 = pytester.makepyfile( """ @@ -2456,6 +2689,8 @@ def test_onefails(): ) +# ensemble: multiprocessing needs an importable module-level target function, +# so the test subject has to live in a real file. def test_exception_handling_no_traceback(pytester: Pytester) -> None: """Handle chain exceptions in tasks submitted by the multiprocess module (#1984).""" p1 = pytester.makepyfile( @@ -2513,6 +2748,7 @@ def test_multitask_job(): ), ], ) +# ensemble: needs a subprocess started with "python -OO". def test_warn_missing(pytester: Pytester, cmdline_args, warning_output) -> None: pytester.makepyfile("") @@ -2520,6 +2756,8 @@ def test_warn_missing(pytester: Pytester, cmdline_args, warning_output) -> None: result.stdout.fnmatch_lines(warning_output) +# ensemble: the subject is collecting a file from disk under a custom +# "python_files"; ensemble collection is preset, never path-driven. def test_recursion_source_decode(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -2541,34 +2779,27 @@ def test_something(): ) -def test_AssertionError_message(pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_hello(): - x,y = 1,2 - assert 0, (x,y) - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( +def test_AssertionError_message(tmp_path: Path) -> None: + def test_hello(): + x, y = 1, 2 + assert 0, (x, y) + + record = run_tests(test_hello, spec=assertion_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines( """ *def test_hello* - *assert 0, (x,y)* + *assert 0, (x, y)* *AssertionError: (1, 2)* """ ) -def test_diff_newline_at_end(pytester: Pytester) -> None: - pytester.makepyfile( - r""" - def test_diff(): - assert 'asdf' == 'asdf\n' - """ - ) +def test_diff_newline_at_end(tmp_path: Path) -> None: + def test_diff(): + assert "asdf" == "asdf\n" # type: ignore[comparison-overlap] - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record = run_tests(test_diff, spec=assertion_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines( r""" *assert 'asdf' == 'asdf\n' * - asdf @@ -2578,6 +2809,10 @@ def test_diff(): ) +# ensemble: the "assertion is always true" warning is emitted by the *rewriter* +# while compiling a module, with that module's own file:line - both of which +# are the host's here (and the host's "filterwarnings = error" would turn the +# warning into an import error of this very file). @pytest.mark.filterwarnings("default") def test_assert_tuple_warning(pytester: Pytester) -> None: msg = "assertion is always true" @@ -2601,6 +2836,8 @@ def test_tuple(): assert msg not in result.stdout.str() +# ensemble: same as above - the absence of a rewrite-time warning can only be +# observed for a module the run under test rewrote itself. def test_assert_indirect_tuple_no_warning(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -2614,44 +2851,44 @@ def test_tuple(): assert "WR1" not in output -def test_assert_with_unicode(pytester: Pytester) -> None: - pytester.makepyfile( - """\ - def test_unicode(): - assert '유니코드' == 'Unicode' - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*AssertionError*"]) +def test_assert_with_unicode(tmp_path: Path) -> None: + def test_unicode(): + assert "유니코드" == "Unicode" # type: ignore[comparison-overlap] + record = run_tests(test_unicode, spec=assertion_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines(["*AssertionError*"]) -def test_raise_unprintable_assertion_error(pytester: Pytester) -> None: - pytester.makepyfile( - r""" - def test_raise_assertion_error(): - raise AssertionError('\xff') - """ + +def test_raise_unprintable_assertion_error(tmp_path: Path) -> None: + def test_raise_assertion_error(): + raise AssertionError("\xff") + + record = run_tests( + test_raise_assertion_error, spec=assertion_spec(tmp_path), capture_output=True ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - [r"> raise AssertionError('\xff')", "E AssertionError: *"] + # The rendered source line is this file's, dedented by the traceback + # formatter - only the quoting follows this file's formatting. + record.stdout.fnmatch_lines( + [r'> raise AssertionError("\xff")', "E AssertionError: *"] ) -def test_raise_assertion_error_raising_repr(pytester: Pytester) -> None: - pytester.makepyfile( - """ - class RaisingRepr(object): - def __repr__(self): - raise Exception() - def test_raising_repr(): - raise AssertionError(RaisingRepr()) - """ +def test_raise_assertion_error_raising_repr(tmp_path: Path) -> None: + class RaisingRepr: + def __repr__(self): + raise Exception() + + def test_raising_repr(): + raise AssertionError(RaisingRepr()) + + record = run_tests( + test_raising_repr, spec=assertion_spec(tmp_path), capture_output=True ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["E AssertionError: "]) + record.stdout.fnmatch_lines(["E AssertionError: "]) +# ensemble: the failure happens while *importing* the test module, and an +# ensemble module object is never imported. def test_issue_1944(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -2681,75 +2918,75 @@ def raise_exit(obj): callequal(1, 1) -def test_plugin_hook_returning_none_is_skipped(pytester: Pytester) -> None: +def test_plugin_hook_returning_none_is_skipped(tmp_path: Path) -> None: """A ``pytest_assertrepr_compare`` impl returning ``None`` is skipped so the next impl (or the built-in) can produce the explanation.""" - pytester.makeconftest( - """ - def pytest_assertrepr_compare(op, left, right): + + class Conftest: + def pytest_assertrepr_compare(self, op, left, right): # Always defer to the next plugin / the built-in. return None - """ - ) - pytester.makepyfile( - """ - def test_diff(): - assert {1, 2} == {1, 3} - """ + + def test_diff(): + assert {1, 2} == {1, 3} + + record = run_tests( + test_diff, + spec=assertion_spec(tmp_path).with_plugins(Conftest()), + capture_output=True, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( ["*Extra items in the left set:*", "*Extra items in the right set:*"] ) -def test_plugin_hook_returning_empty_iterator_is_skipped(pytester: Pytester) -> None: +def test_plugin_hook_returning_empty_iterator_is_skipped(tmp_path: Path) -> None: """A plugin returning a truthy but ultimately empty iterable is skipped after materialisation.""" - pytester.makeconftest( - """ - def pytest_assertrepr_compare(op, left, right): + + class Conftest: + def pytest_assertrepr_compare(self, op, left, right): return iter([]) - """ - ) - pytester.makepyfile( - """ - def test_diff(): - assert {1, 2} == {1, 3} - """ + + def test_diff(): + assert {1, 2} == {1, 3} + + record = run_tests( + test_diff, + spec=assertion_spec(tmp_path).with_plugins(Conftest()), + capture_output=True, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( ["*Extra items in the left set:*", "*Extra items in the right set:*"] ) def test_callbinrepr_falls_through_when_all_hooks_return_none( - pytester: Pytester, + tmp_path: Path, ) -> None: """When no ``pytest_assertrepr_compare`` impl produces an explanation, the plain assert rewrite is shown.""" - pytester.makepyfile( - """ - def test_trivial(): - assert 1 == 2 - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*assert 1 == 2*"]) - result.assert_outcomes(failed=1) + def test_trivial(): + assert 1 == 2 # type: ignore[comparison-overlap] + + record = run_tests(test_trivial, spec=assertion_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines(["*assert 1 == 2*"]) + record.assert_outcomes(failed=1) -def test_callbinrepr_plain_assert_mode(pytester: Pytester) -> None: + +def test_callbinrepr_plain_assert_mode(tmp_path: Path) -> None: """In ``--assert=plain`` mode the comparison explanation is still produced.""" - pytester.makepyfile( - """ - def test_diff(): - assert {1, 2} == {1, 3} - """ + + def test_diff(): + assert {1, 2} == {1, 3} + + record = run_tests( + test_diff, + spec=assertion_spec(tmp_path, "--assert=plain"), + capture_output=True, ) - result = pytester.runpytest("--assert=plain") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( ["*Extra items in the left set:*", "*Extra items in the right set:*"] ) @@ -2773,19 +3010,22 @@ def raise_value_error(obj): assert any("ValueError" in line or "synthetic" in line for line in expl) -def test_assertion_location_with_coverage(pytester: Pytester) -> None: +def test_assertion_location_with_coverage(tmp_path: Path) -> None: """This used to report the wrong location when run with coverage (#5754).""" - p = pytester.makepyfile( - """ - def test(): - assert False, 1 - assert False, 2 - """ - ) - result = pytester.runpytest(str(p)) - result.stdout.fnmatch_lines( + # The messages are names rather than the original's int literals: as real + # code the source is linted, and ``assert False, 1`` is RUF040. What the + # test is about - the reported location being the *first* assert - is + # unaffected. + first, second = 1, 2 + + def test(): + assert False, first + assert False, second + + record = run_tests(test, spec=assertion_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines( [ - "> assert False, 1", + "> assert False, first", "E AssertionError: 1", "E assert False", "*= 1 failed in*", @@ -2810,13 +3050,10 @@ def test_reprcompare_verbose_long() -> None: @pytest.mark.parametrize("enable_colors", [True, False]) @pytest.mark.parametrize( - ("test_code", "expected_lines"), + ("case", "expected_lines"), ( ( - """ - def test(): - assert [0, 1] == [0, 2] - """, + "list", [ "{bold}{red}E At index 1 diff: {reset}{number}1{hl-reset}{endline} != {reset}{number}2*", "{bold}{red}E {reset}{light-red}- 2,{hl-reset}{endline}{reset}", @@ -2824,12 +3061,7 @@ def test(): ], ), ( - """ - def test(): - assert {f"number-is-{i}": i for i in range(1, 6)} == { - f"number-is-{i}": i for i in range(5) - } - """, + "dict", [ "{bold}{red}E Common items:{reset}", "{bold}{red}E {reset}{{{str}'{hl-reset}{str}number-is-1{hl-reset}{str}'{hl-reset}: {number}1*", @@ -2843,10 +3075,7 @@ def test(): ], ), ( - """ - def test(): - assert "abcd" == "abce" - """, + "text", [ "{bold}{red}E {reset}{light-red}- abce{hl-reset}{endline}{reset}", "{bold}{red}E {light-green}+ abcd{hl-reset}{endline}{reset}", @@ -2855,11 +3084,31 @@ def test(): ), ) def test_comparisons_handle_colors( - pytester: Pytester, color_mapping, enable_colors, test_code, expected_lines + tmp_path: Path, color_mapping, enable_colors, case, expected_lines ) -> None: - p = pytester.makepyfile(test_code) - result = pytester.runpytest( - f"--color={'yes' if enable_colors else 'no'}", "-vv", str(p) + if case == "list": + + def test(): + assert [0, 1] == [0, 2] + + elif case == "dict": + + def test(): + assert {f"number-is-{i}": i for i in range(1, 6)} == { + f"number-is-{i}": i for i in range(5) + } + + else: + + def test(): + assert "abcd" == "abce" # type: ignore[comparison-overlap] + + record = run_tests( + test, + spec=assertion_spec( + tmp_path, f"--color={'yes' if enable_colors else 'no'}", "-vv" + ), + capture_output=True, ) formatter = ( color_mapping.format_for_fnmatch @@ -2867,45 +3116,44 @@ def test_comparisons_handle_colors( else color_mapping.strip_colors ) - result.stdout.fnmatch_lines(formatter(expected_lines), consecutive=False) + record.stdout.fnmatch_lines(formatter(expected_lines), consecutive=False) -def test_fine_grained_assertion_verbosity(pytester: Pytester): +def test_fine_grained_assertion_verbosity(tmp_path: Path): long_text = "Lorem ipsum dolor sit amet " * 10 - p = pytester.makepyfile( - f""" - def test_ok(): - pass - - def test_words_fail(): - fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] - fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] - assert fruits1 == fruits2 + def test_ok(): + pass + def test_words_fail(): + fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] + fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] + assert fruits1 == fruits2 - def test_numbers_fail(): - number_to_text1 = {{str(x): x for x in range(5)}} - number_to_text2 = {{str(x * 10): x * 10 for x in range(5)}} - assert number_to_text1 == number_to_text2 + def test_numbers_fail(): + number_to_text1 = {str(x): x for x in range(5)} + number_to_text2 = {str(x * 10): x * 10 for x in range(5)} + assert number_to_text1 == number_to_text2 + def test_long_text_fail(): + assert "hello world" in long_text - def test_long_text_fail(): - long_text = "{long_text}" - assert "hello world" in long_text - """ - ) - pytester.makeini( - """ - [pytest] - verbosity_assertions = 2 - """ + record = run_tests( + test_ok, + test_words_fail, + test_numbers_fail, + test_long_text_fail, + spec=assertion_spec(tmp_path, verbosity_assertions="2"), + capture_output=True, ) - result = pytester.runpytest(p) - result.stdout.fnmatch_lines( + # Replaces the original's ".FFF [100%]" progress line: the ensemble drives + # the runtest protocol directly, so the terminal reporter never writes the + # end-of-file progress fill. + record.assert_outcomes(passed=1, failed=3) + + record.stdout.fnmatch_lines( [ - f"{p.name} .FFF [100%]", "E At index 2 diff: 'grapes' != 'orange'", "E Full diff: (-: missing in left side, +: extra in left side)", "E [", @@ -2940,27 +3188,22 @@ def test_long_text_fail(): def test_assertion_text_diff_style_block_for_multiline_strings( - pytester: Pytester, + tmp_path: Path, ) -> None: - pytester.makepyfile( - r""" + # Module-level globals in the original; an ensemble source's globals are + # this module's, so these are locals. + def test_text_diff(): actual = "alpha\n beta\n" expected = "alpha\n beta" + assert actual == expected - def test_text_diff(): - assert actual == expected - """ + record = run_tests( + test_text_diff, + spec=assertion_spec(tmp_path, "-vv", assertion_text_diff_style="block"), + capture_output=True, ) - pytester.makeini( - """ - [pytest] - assertion_text_diff_style = block - """ - ) - - result = pytester.runpytest("-vv") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "E Left:", "E alpha", @@ -2971,28 +3214,22 @@ def test_text_diff(): "E beta", ] ) - result.stdout.no_fnmatch_line("*? -*") + record.stdout.no_fnmatch_line("*? -*") def test_assertion_text_diff_style_block_for_single_line_strings( - pytester: Pytester, + tmp_path: Path, ) -> None: - pytester.makepyfile( - """ - def test_text_diff(): - assert "spam" == "eggs" - """ - ) - pytester.makeini( - """ - [pytest] - assertion_text_diff_style = block - """ - ) + def test_text_diff(): + assert "spam" == "eggs" # type: ignore[comparison-overlap] - result = pytester.runpytest("-vv") + record = run_tests( + test_text_diff, + spec=assertion_spec(tmp_path, "-vv", assertion_text_diff_style="block"), + capture_output=True, + ) - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "E Left:", "E spam", @@ -3000,34 +3237,32 @@ def test_text_diff(): "E eggs", ] ) - result.stdout.no_fnmatch_line("*- eggs*") + record.stdout.no_fnmatch_line("*- eggs*") -def test_assertion_text_diff_style_invalid(pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_ok(): - pass - """ - ) - pytester.makeini( - """ - [pytest] - assertion_text_diff_style = side-by-side - """ - ) - - result = pytester.runpytest() +def test_assertion_text_diff_style_invalid(tmp_path: Path) -> None: + def test_ok(): + pass - assert result.ret == pytest.ExitCode.USAGE_ERROR - result.stderr.fnmatch_lines( - [ - "*ERROR: *: config option 'assertion_text_diff_style' expects one of " - "'ndiff' | 'block', got 'side-by-side'" - ] - ) + # The ini value is validated from ``pytest_configure``; the ensemble sees + # the ``UsageError`` a command line run renders to stderr and turns into + # ``ExitCode.USAGE_ERROR``. + with pytest.raises( + UsageError, + match=( + "config option 'assertion_text_diff_style' expects one of " + r"'ndiff' \| 'block', got 'side-by-side'" + ), + ): + run_tests( + test_ok, + spec=assertion_spec(tmp_path, assertion_text_diff_style="side-by-side"), + ) +# ensemble: asserts the failing source's "file:line", which is host-anchored: +# it points at this file at its own absolute line number, not at the ensemble's +# synthetic module. def test_full_output_vvv(pytester: Pytester) -> None: pytester.makepyfile( r""" @@ -3061,6 +3296,8 @@ def test_vvv(): result.stdout.no_fnmatch_line(expected_non_vvv_arg_line) +# ensemble: same as above - the closing "test_order.py:*: AssertionError" line +# is host-anchored. def test_dict_extra_items_preserve_insertion_order(pytester: Pytester) -> None: """Assertion output of dict diff shows keys in insertion order (#13503).""" pytester.makepyfile( From 9a8dda7e53c181816b3f858ef6bb3a96f7bc399a Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 20:53:18 +0200 Subject: [PATCH 13/30] testing: port test_terminal.py to _pytest.ensemble 39 of 146 pytester-based tests ported - deliberately conservative. This file's subject is rendering, so a port buys speed and hermeticity but no assertion strength, and relaxing a pattern to make one pass would destroy what the test is for. Everything left carries an `# ensemble:` reason. What blocks the rest, measured rather than assumed: - the progress column, ~40 tests, and it is not the item-location problem. TerminalReporter._determine_show_progress_info() returns False whenever getoption("capture", "no") == "no", and an ensemble does not load the capture plugin, so the column is off at every verbosity whatever console_output_style says. The final [100%] additionally comes from a pytest_runtestloop hookwrapper that run_items never reaches. - --collect-only rendering, ~16, a pytest_cmdline_main concern. - captured-output sections, 10. - host-anchored file:line, 9, plus 4 asserting the `<- file>` suffix. Note _locationline only appends `<- ...` at verbosity >= 2, so plain -v progress lines already match upstream verbatim - which is what carried test_verbose_reporting and TestClassicOutputStyle.test_verbose. One hazard recorded for future ports: assertion-explanation verbosity is host-global, because _pytest.assertion.util._reprcompare is bound at host configure time. -vv inside an ensemble does not un-truncate an assertion. --- testing/test_terminal.py | 1136 +++++++++++++++++++++++++------------- 1 file changed, 739 insertions(+), 397 deletions(-) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 30208084ab2..6a0360f1b49 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -20,6 +20,12 @@ import _pytest.config from _pytest.config import Config from _pytest.config import ExitCode +from _pytest.ensemble import build_module +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import Ensemble +from _pytest.ensemble import run_tests +from _pytest.mark.structures import Mark +from _pytest.mark.structures import MarkDecorator from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester from _pytest.reports import BaseReport @@ -37,6 +43,16 @@ import pytest +def unregistered_mark(name: str, *args: object, **kwargs: object) -> MarkDecorator: + """Build a mark decorator without consulting the host configuration. + + ``pytest.mark.`` resolves against the *host* config at decoration + time, and this suite runs with strict markers, so a deliberately + unregistered mark applied to an ensemble source has to be built directly. + """ + return MarkDecorator(Mark(name, args, kwargs, _ispytest=True), _ispytest=True) + + class DistInfo(NamedTuple): project_name: str version: int @@ -86,21 +102,28 @@ def test_plugin_nameversion(input, expected): class TestTerminal: - def test_pass_skip_fail(self, pytester: Pytester, option) -> None: - pytester.makepyfile( - """ - import pytest - def test_ok(): - pass - def test_skip(): - pytest.skip("xx") - def test_func(): - assert 0 - """ + def test_pass_skip_fail(self, tmp_path: Path, option) -> None: + def test_ok(): + pass + + def test_skip(): + pytest.skip("xx") + + def test_func(): + assert 0 + + spec = ConfigSpec(rootpath=tmp_path, args=tuple(option.args)) + record = run_tests( + test_ok, + test_skip, + test_func, + spec=spec, + name="test_pass_skip_fail", + capture_output=True, ) - result = pytester.runpytest(*option.args) + record.assert_outcomes(passed=1, skipped=1, failed=1) if option.verbosity > 0: - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*test_pass_skip_fail.py::test_ok PASS*", "*test_pass_skip_fail.py::test_skip SKIP*", @@ -108,13 +131,17 @@ def test_func(): ] ) elif option.verbosity == 0: - result.stdout.fnmatch_lines(["*test_pass_skip_fail.py .sF*"]) + record.stdout.fnmatch_lines(["*test_pass_skip_fail.py .sF*"]) else: - result.stdout.fnmatch_lines([".sF*"]) - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines([".sF*"]) + record.stdout.fnmatch_lines( [" def test_func():", "> assert 0", "E assert 0"] ) + # ensemble: `console_output_style=times` needs the progress column, which + # only appears once capturing is active and the run goes through + # `pytest_runtestloop` - an ensemble has neither, so the code path this + # regression test exercises would never run. def test_console_output_style_times_with_skipped_and_passed( self, pytester: Pytester ) -> None: @@ -140,34 +167,57 @@ def test_hello(): combined = "\n".join(result.stdout.lines + result.stderr.lines) assert "INTERNALERROR" not in combined - def test_internalerror(self, pytester: Pytester, linecomp) -> None: - modcol = pytester.getmodulecol("def test_one(): pass") - rep = TerminalReporter(modcol.config, file=linecomp.stringio) - with pytest.raises(ValueError) as excinfo: - raise ValueError("hello") - rep.pytest_internalerror(excinfo.getrepr()) + def test_internalerror(self, tmp_path: Path, linecomp) -> None: + def test_one(): + pass + + with Ensemble(test_one, rootpath=tmp_path, capture_output=True) as ensemble: + rep = TerminalReporter(ensemble.config, file=linecomp.stringio) + with pytest.raises(ValueError) as excinfo: + raise ValueError("hello") + rep.pytest_internalerror(excinfo.getrepr()) linecomp.assert_contains_lines(["INTERNALERROR> *ValueError*hello*"]) - def test_writeline(self, pytester: Pytester, linecomp) -> None: - modcol = pytester.getmodulecol("def test_one(): pass") - rep = TerminalReporter(modcol.config, file=linecomp.stringio) - rep.write_fspath_result(modcol.nodeid, ".") - rep.write_line("hello world") - lines = linecomp.stringio.getvalue().split("\n") - assert not lines[0] - assert lines[1].endswith(modcol.name + " .") - assert lines[2] == "hello world" - - def test_show_runtest_logstart(self, pytester: Pytester, linecomp) -> None: - item = pytester.getitem("def test_func(): pass") - tr = TerminalReporter(item.config, file=linecomp.stringio) - item.config.pluginmanager.register(tr) - location = item.reportinfo() - tr.config.hook.pytest_runtest_logstart( - nodeid=item.nodeid, location=location, fspath=str(item.path) - ) + def test_writeline(self, tmp_path: Path, linecomp) -> None: + def test_one(): + pass + + with Ensemble( + test_one, rootpath=tmp_path, name="test_writeline", capture_output=True + ) as ensemble: + (item,) = ensemble.collect() + modcol = item.parent + assert modcol is not None + rep = TerminalReporter(ensemble.config, file=linecomp.stringio) + rep.write_fspath_result(modcol.nodeid, ".") + rep.write_line("hello world") + lines = linecomp.stringio.getvalue().split("\n") + assert not lines[0] + assert lines[1].endswith(modcol.name + " .") + assert lines[2] == "hello world" + + def test_show_runtest_logstart(self, tmp_path: Path, linecomp) -> None: + def test_func(): + pass + + with Ensemble( + test_func, + rootpath=tmp_path, + name="test_show_runtest_logstart", + capture_output=True, + ) as ensemble: + (item,) = ensemble.collect() + tr = TerminalReporter(item.config, file=linecomp.stringio) + item.config.pluginmanager.register(tr) + location = item.reportinfo() + tr.config.hook.pytest_runtest_logstart( + nodeid=item.nodeid, location=location, fspath=str(item.path) + ) + item.config.pluginmanager.unregister(tr) linecomp.assert_contains_lines(["*test_show_runtest_logstart.py*"]) + # ensemble: drives a real pytest through a pty to watch output appear + # before the test finishes. def test_runtest_location_shown_before_test_starts( self, pytester: Pytester ) -> None: @@ -183,6 +233,7 @@ def test_1(): child.sendeof() child.kill(15) + # ensemble: drives a real pytest through a pty. def test_report_collect_after_half_a_second( self, pytester: Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -211,6 +262,10 @@ def test_1(): rest = child.read().decode("utf8") assert "= \x1b[32m\x1b[1m2 passed\x1b[0m\x1b[32m in" in rest + # ensemble: asserts the `<- test_p1.py` suffix, which is derived from the + # item's reportinfo. Ensemble items report the *host* file, so every + # ensemble item would carry that suffix at -vv and none of these lines + # would match. def test_itemreport_subclasses_show_subclassed_file( self, pytester: Pytester ) -> None: @@ -264,6 +319,9 @@ class TestMore(BaseTests): pass ] ) + # ensemble: asserts `no_fnmatch_line("* <- *")`, which an ensemble can + # never satisfy - the item's reportinfo is the host file, so -vv always + # renders a `<- .../test_terminal.py` suffix. def test_itemreport_directclasses_not_shown_as_subclasses( self, pytester: Pytester ) -> None: @@ -283,6 +341,8 @@ def test_method(self): result.stdout.fnmatch_lines(["*a123/test_hello123.py*PASS*"]) result.stdout.no_fnmatch_line("* <- *") + # ensemble: KeyboardInterrupt reporting is done by `wrap_session`, which + # an ensemble never enters. @pytest.mark.parametrize("fulltrace", ("", "--fulltrace")) def test_keyboard_interrupt(self, pytester: Pytester, fulltrace) -> None: pytester.makepyfile( @@ -315,6 +375,8 @@ def test_interrupt_me(): ) result.stdout.fnmatch_lines(["*KeyboardInterrupt*"]) + # ensemble: asserts the exit code of an interrupted session; interrupt + # handling lives in `wrap_session`. def test_keyboard_in_sessionstart(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -333,27 +395,30 @@ def test_foobar(): assert result.ret == 2 result.stdout.fnmatch_lines(["*KeyboardInterrupt*"]) - def test_collect_single_item(self, pytester: Pytester) -> None: + def test_collect_single_item(self, tmp_path: Path) -> None: """Use singular 'item' when reporting a single test item""" - pytester.makepyfile( - """ - def test_foobar(): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["collected 1 item"]) - - def test_rewrite(self, pytester: Pytester, monkeypatch) -> None: - config = pytester.parseconfig() - f = StringIO() - monkeypatch.setattr(f, "isatty", lambda *args: True) - tr = TerminalReporter(config, f) - tr._tw.fullwidth = 10 - tr.write("hello") - tr.rewrite("hey", erase=True) - assert f.getvalue() == "hello" + "\r" + "hey" + (6 * " ") + def test_foobar(): + pass + + record = run_tests(test_foobar, rootpath=tmp_path, capture_output=True) + record.assert_outcomes(passed=1) + record.stdout.fnmatch_lines(["collected 1 item"]) + + def test_rewrite(self, tmp_path: Path, monkeypatch) -> None: + with Ensemble(rootpath=tmp_path, capture_output=True) as ensemble: + config = ensemble.config + f = StringIO() + monkeypatch.setattr(f, "isatty", lambda *args: True) + tr = TerminalReporter(config, f) + tr._tw.fullwidth = 10 + tr.write("hello") + tr.rewrite("hey", erase=True) + assert f.getvalue() == "hello" + "\r" + "hey" + (6 * " ") + + # ensemble: `assert not result.stderr.lines` has no in-process equivalent - + # an ensemble renders to a single buffer and has no stderr of its own - so + # porting would drop half of what this test checks. @pytest.mark.parametrize("category", ["foo", "failed", "error", "passed"]) def test_report_teststatus_explicit_markup( self, monkeypatch: MonkeyPatch, pytester: Pytester, color_mapping, category: str @@ -380,6 +445,9 @@ def test_foobar(): color_mapping.format_for_fnmatch(["*{red}FOO{reset}*"]) ) + # ensemble: the -vv half asserts on lines that, for an ensemble item, gain + # a `<- .../test_terminal.py` suffix from the host-anchored reportinfo, + # which also shifts where the skip reason gets wrapped. def test_verbose_skip_reason(self, pytester: Pytester) -> None: pytester.makepyfile( """ @@ -471,16 +539,21 @@ def test_long_xfail(): ) @pytest.mark.parametrize("isatty", [True, False]) - def test_isatty(self, pytester: Pytester, monkeypatch, isatty: bool) -> None: - config = pytester.parseconfig() - f = StringIO() - monkeypatch.setattr(f, "isatty", lambda: isatty) - tr = TerminalReporter(config, f) - assert tr.isatty() == isatty - # It was incorrectly implemented as a boolean so we still support using it as one. - assert bool(tr.isatty) == isatty - - + def test_isatty(self, tmp_path: Path, monkeypatch, isatty: bool) -> None: + with Ensemble(rootpath=tmp_path, capture_output=True) as ensemble: + config = ensemble.config + f = StringIO() + monkeypatch.setattr(f, "isatty", lambda: isatty) + tr = TerminalReporter(config, f) + assert tr.isatty() == isatty + # It was incorrectly implemented as a boolean so we still support using it as one. + assert bool(tr.isatty) == isatty + + +# ensemble: every test below asserts the rendering of `--collect-only`, which +# is served from `pytest_cmdline_main`. An ensemble never reaches that path: +# it still runs the collected items, and its collection tree renders as +# `` with no enclosing ``. class TestCollectonly: def test_collectonly_basic(self, pytester: Pytester) -> None: pytester.makepyfile( @@ -652,6 +725,9 @@ def test_bar(): pass ) +# ensemble: every test below asserts on a `Captured stdout` section. Capture +# is process-global state an ensemble deliberately does not install, so no +# captured-output section is ever rendered. class TestFixtureReporting: def test_setup_fixture_error(self, pytester: Pytester) -> None: pytester.makepyfile( @@ -753,84 +829,101 @@ def teardown_function(function): class TestTerminalFunctional: - def test_deselected(self, pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - def test_one(): - pass - def test_two(): - pass - def test_three(): - pass - """ + def test_deselected(self, tmp_path: Path) -> None: + def test_one(): + pass + + def test_two(): + pass + + def test_three(): + pass + + spec = ConfigSpec(rootpath=tmp_path, args=("-k", "test_t")) + record = run_tests( + test_one, + test_two, + test_three, + spec=spec, + name="test_deselected", + capture_output=True, ) - result = pytester.runpytest("-k", "test_t", testpath) - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( ["collected 3 items / 1 deselected / 2 selected", "*test_deselected.py ..*"] ) - assert result.ret == 0 - - def test_deselected_with_hook_wrapper(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest + # Stronger than the original `ret == 0`. + record.assert_outcomes(passed=2, deselected=1) + def test_deselected_with_hook_wrapper(self, tmp_path: Path) -> None: + class DeselectLastPlugin: @pytest.hookimpl(wrapper=True) - def pytest_collection_modifyitems(config, items): + def pytest_collection_modifyitems(self, config, items): yield deselected = items.pop() config.hook.pytest_deselected(items=[deselected]) - """ - ) - testpath = pytester.makepyfile( - """ - def test_one(): - pass - def test_two(): - pass - def test_three(): - pass - """ + + def test_one(): + pass + + def test_two(): + pass + + def test_three(): + pass + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(DeselectLastPlugin(),)) + record = run_tests( + test_one, + test_two, + test_three, + spec=spec, + name="test_deselected_hook", + capture_output=True, ) - result = pytester.runpytest(testpath) - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 3 items / 1 deselected / 2 selected", "*= 2 passed, 1 deselected in*", ] ) - assert result.ret == 0 + record.assert_outcomes(passed=2, deselected=1) def test_show_deselected_items_using_markexpr_before_test_execution( - self, pytester: Pytester + self, tmp_path: Path ) -> None: - pytester.makepyfile( - test_show_deselected=""" - import pytest + @unregistered_mark("foo") + def test_foobar(): + pass - @pytest.mark.foo - def test_foobar(): - pass + @unregistered_mark("bar") + def test_bar(): + pass - @pytest.mark.bar - def test_bar(): - pass + def test_pass(): + pass - def test_pass(): - pass - """ + spec = ConfigSpec(rootpath=tmp_path, args=("-m", "not foo")) + record = run_tests( + test_foobar, + test_bar, + test_pass, + spec=spec, + name="test_show_deselected", + capture_output=True, ) - result = pytester.runpytest("-m", "not foo") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 3 items / 1 deselected / 2 selected", "*test_show_deselected.py ..*", "*= 2 passed, 1 deselected in * =*", ] ) - result.stdout.no_fnmatch_line("*= 1 deselected =*") - assert result.ret == 0 + record.stdout.no_fnmatch_line("*= 1 deselected =*") + record.assert_outcomes(passed=2, deselected=1) + # ensemble: the `! Interrupted: ... !` line and the interrupted exit code + # come from `wrap_session`, and the collection error needs a module that + # blows up on import. def test_selected_count_with_error(self, pytester: Pytester) -> None: pytester.makepyfile( test_selected_count_3=""" @@ -858,41 +951,48 @@ def test_bar(): ) assert result.ret == ExitCode.INTERRUPTED - def test_no_skip_summary_if_failure(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def test_ok(): - pass - def test_fail(): - assert 0 - def test_skip(): - pytest.skip("dontshow") - """ + def test_no_skip_summary_if_failure(self, tmp_path: Path) -> None: + def test_ok(): + pass + + def test_fail(): + assert 0 + + def test_skip(): + pytest.skip("dontshow") + + record = run_tests( + test_ok, + test_fail, + test_skip, + rootpath=tmp_path, + name="test_no_skip_summary_if_failure", + capture_output=True, ) - result = pytester.runpytest() - assert result.stdout.str().find("skip test summary") == -1 - assert result.ret == 1 + assert record.output.find("skip test summary") == -1 + # Stronger than the original `ret == 1`. + record.assert_outcomes(passed=1, failed=1, skipped=1) - def test_passes(self, pytester: Pytester) -> None: - p1 = pytester.makepyfile( - """ - def test_passes(): + def test_passes(self, tmp_path: Path) -> None: + def test_passes(): + pass + + class TestClass: + def test_method(self): pass - class TestClass(object): - def test_method(self): - pass - """ + + record = run_tests( + test_passes, + TestClass, + rootpath=tmp_path, + name="test_passes", + capture_output=True, ) - old = p1.parent - pytester.chdir() - try: - result = pytester.runpytest() - finally: - os.chdir(old) - result.stdout.fnmatch_lines(["test_passes.py ..*", "* 2 pass*"]) - assert result.ret == 0 + record.stdout.fnmatch_lines(["test_passes.py ..*", "* 2 pass*"]) + record.assert_outcomes(passed=2) + # ensemble: the header block differs - an ensemble has no `plugins:` line + # and its rootdir is a throwaway tmp path. def test_header_trailer_info( self, monkeypatch: MonkeyPatch, pytester: Pytester, request ) -> None: @@ -917,6 +1017,8 @@ def test_passes(): if request.config.pluginmanager.list_plugin_distinfo(): result.stdout.fnmatch_lines(["plugins: *"]) + # ensemble: an ensemble never renders a `plugins:` line, so the + # `no_fnmatch_line("plugins: *")` half would hold for the wrong reason. def test_no_header_trailer_info( self, monkeypatch: MonkeyPatch, pytester: Pytester, request ) -> None: @@ -935,6 +1037,8 @@ def test_passes(): if request.config.pluginmanager.list_plugin_distinfo(): result.stdout.no_fnmatch_line("plugins: *") + # ensemble: asserts `configfile:`/`testpaths:` header lines; ensemble + # configs are built from data and never read a config file. def test_header(self, pytester: Pytester) -> None: pytester.path.joinpath("tests").mkdir() pytester.path.joinpath("gui").mkdir() @@ -964,6 +1068,7 @@ def test_header(self, pytester: Pytester) -> None: result = pytester.runpytest("tests") result.stdout.fnmatch_lines(["rootdir: *test_header0", "configfile: tox.ini"]) + # ensemble: asserts `configfile:`/`testpaths:` header lines. def test_header_absolute_testpath( self, pytester: Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -985,6 +1090,8 @@ def test_header_absolute_testpath( ] ) + # ensemble: asserts on `inifile:`/`testpaths:` header lines, which an + # ensemble never renders in the first place. def test_no_header(self, pytester: Pytester) -> None: pytester.path.joinpath("tests").mkdir() pytester.path.joinpath("gui").mkdir() @@ -1005,42 +1112,49 @@ def test_no_header(self, pytester: Pytester) -> None: result = pytester.runpytest("tests", "--no-header") result.stdout.no_fnmatch_line("rootdir: *test_header0, inifile: tox.ini") - def test_no_summary(self, pytester: Pytester) -> None: - p1 = pytester.makepyfile( - """ - def test_no_summary(): - assert false - """ + def test_no_summary(self, tmp_path: Path) -> None: + def test_no_summary(): + # Deliberately undefined, as in the original: the test only cares + # that the test fails and that no FAILURES section is rendered. + assert false # type: ignore[name-defined] # noqa: F821 + + spec = ConfigSpec(rootpath=tmp_path, args=("--no-summary",)) + record = run_tests( + test_no_summary, spec=spec, name="test_no_summary", capture_output=True ) - result = pytester.runpytest(p1, "--no-summary") - result.stdout.no_fnmatch_line("*= FAILURES =*") + record.stdout.no_fnmatch_line("*= FAILURES =*") + record.assert_outcomes(failed=1) - def test_no_summary_still_runs_terminal_summary_hook( - self, pytester: Pytester - ) -> None: + def test_no_summary_still_runs_terminal_summary_hook(self, tmp_path: Path) -> None: """--no-summary must not skip pytest_terminal_summary for plugins (#14724).""" - pytester.makeconftest( - """ - def pytest_terminal_summary(terminalreporter, exitstatus, config): + + class SummaryPlugin: + def pytest_terminal_summary(self, terminalreporter, exitstatus, config): terminalreporter.write_line("PLUGIN_TERMINAL_SUMMARY_RAN") - """ + + def test_ok(): + assert True + + spec = ConfigSpec( + rootpath=tmp_path, + args=("--no-summary",), + extra_plugins=(SummaryPlugin(),), ) - p1 = pytester.makepyfile("def test_ok(): assert True") - result = pytester.runpytest(p1, "--no-summary") - result.stdout.fnmatch_lines(["PLUGIN_TERMINAL_SUMMARY_RAN"]) - result.stdout.no_fnmatch_line("*= FAILURES =*") + record = run_tests(test_ok, spec=spec, capture_output=True) + record.stdout.fnmatch_lines(["PLUGIN_TERMINAL_SUMMARY_RAN"]) + record.stdout.no_fnmatch_line("*= FAILURES =*") - def test_showlocals(self, pytester: Pytester) -> None: - p1 = pytester.makepyfile( - """ - def test_showlocals(): - x = 3 - y = "x" * 5000 - assert 0 - """ + def test_showlocals(self, tmp_path: Path) -> None: + def test_showlocals(): + x = 3 # noqa: F841 + y = "x" * 5000 # noqa: F841 + assert 0 + + spec = ConfigSpec(rootpath=tmp_path, args=("-l",)) + record = run_tests( + test_showlocals, spec=spec, name="test_showlocals", capture_output=True ) - result = pytester.runpytest(p1, "-l") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ # "_ _ * Locals *", "x* = 3", @@ -1048,22 +1162,29 @@ def test_showlocals(): ] ) - def test_noshowlocals_addopts_override(self, pytester: Pytester) -> None: - pytester.makeini("[pytest]\naddopts=--showlocals") - p1 = pytester.makepyfile( - """ - def test_noshowlocals(): - x = 3 - y = "x" * 5000 - assert 0 - """ - ) + def test_noshowlocals_addopts_override(self, tmp_path: Path) -> None: + def test_noshowlocals(): + x = 3 # noqa: F841 + y = "x" * 5000 # noqa: F841 + assert 0 # Override global --showlocals for py.test via arg - result = pytester.runpytest(p1, "--no-showlocals") - result.stdout.no_fnmatch_line("x* = 3") - result.stdout.no_fnmatch_line("y* = 'xxxxxx*") - + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={"addopts": "--showlocals"}, + args=("--no-showlocals",), + ) + record = run_tests( + test_noshowlocals, + spec=spec, + name="test_noshowlocals", + capture_output=True, + ) + record.stdout.no_fnmatch_line("x* = 3") + record.stdout.no_fnmatch_line("y* = 'xxxxxx*") + + # ensemble: `--tb=short` renders the crash location as `:`, + # and an ensemble item's file is the host `test_terminal.py`. def test_showlocals_short(self, pytester: Pytester) -> None: p1 = pytester.makepyfile( """ @@ -1099,19 +1220,39 @@ def test_skip(self): """ ) - def test_verbose_reporting(self, verbose_testfile, pytester: Pytester) -> None: - result = pytester.runpytest( - verbose_testfile, "-v", "-Walways::pytest.PytestWarning" + def test_verbose_reporting(self, tmp_path: Path) -> None: + def test_fail(): + raise ValueError + + def test_pass(): + pass + + class TestClass: + def test_skip(self): + pytest.skip("hello") + + spec = ConfigSpec( + rootpath=tmp_path, args=("-v", "-Walways::pytest.PytestWarning") ) - result.stdout.fnmatch_lines( + record = run_tests( + test_fail, + test_pass, + TestClass, + spec=spec, + name="test_verbose_reporting", + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "*test_verbose_reporting.py::test_fail *FAIL*", "*test_verbose_reporting.py::test_pass *PASS*", "*test_verbose_reporting.py::TestClass::test_skip *SKIP*", ] ) - assert result.ret == 1 + # Stronger than the original `ret == 1`. + record.assert_outcomes(passed=1, failed=1, skipped=1) + # ensemble: runs the ensemble's tests through xdist workers. def test_verbose_reporting_xdist( self, verbose_testfile, @@ -1131,58 +1272,70 @@ def test_verbose_reporting_xdist( ) assert result.ret == 1 - def test_quiet_reporting(self, pytester: Pytester) -> None: - p1 = pytester.makepyfile("def test_pass(): pass") - result = pytester.runpytest(p1, "-q") - s = result.stdout.str() + def test_quiet_reporting(self, tmp_path: Path) -> None: + def test_pass(): + pass + + spec = ConfigSpec(rootpath=tmp_path, args=("-q",)) + record = run_tests( + test_pass, spec=spec, name="test_quiet_reporting", capture_output=True + ) + s = record.output assert "test session starts" not in s - assert p1.name not in s + assert "test_quiet_reporting.py" not in s assert "===" not in s assert "passed" in s - def test_more_quiet_reporting(self, pytester: Pytester) -> None: - p1 = pytester.makepyfile("def test_pass(): pass") - result = pytester.runpytest(p1, "-qq") - s = result.stdout.str() + def test_more_quiet_reporting(self, tmp_path: Path) -> None: + def test_pass(): + pass + + spec = ConfigSpec(rootpath=tmp_path, args=("-qq",)) + record = run_tests( + test_pass, spec=spec, name="test_more_quiet_reporting", capture_output=True + ) + s = record.output assert "test session starts" not in s - assert p1.name not in s + assert "test_more_quiet_reporting.py" not in s assert "===" not in s assert "passed" not in s @pytest.mark.parametrize( "params", [(), ("--collect-only",)], ids=["no-params", "collect-only"] ) - def test_report_collectionfinish_hook(self, pytester: Pytester, params) -> None: - pytester.makeconftest( - """ - def pytest_report_collectionfinish(config, start_path, items): - return [f'hello from hook: {len(items)} items'] - """ - ) - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize('i', range(3)) - def test(i): - pass - """ + def test_report_collectionfinish_hook(self, tmp_path: Path, params) -> None: + class CollectionFinishPlugin: + def pytest_report_collectionfinish(self, config, start_path, items): + return [f"hello from hook: {len(items)} items"] + + @pytest.mark.parametrize("i", range(3)) + def test(i): + pass + + spec = ConfigSpec( + rootpath=tmp_path, + args=params, + extra_plugins=(CollectionFinishPlugin(),), ) - result = pytester.runpytest(*params) - result.stdout.fnmatch_lines(["collected 3 items", "hello from hook: 3 items"]) + record = run_tests(test, spec=spec, capture_output=True) + record.stdout.fnmatch_lines(["collected 3 items", "hello from hook: 3 items"]) - def test_summary_f_alias(self, pytester: Pytester) -> None: + def test_summary_f_alias(self, tmp_path: Path) -> None: """Test that 'f' and 'F' report chars are aliases and don't show up twice in the summary (#6334)""" - pytester.makepyfile( - """ - def test(): - assert False - """ + + def test(): + assert False + + spec = ConfigSpec(rootpath=tmp_path, args=("-rfF",)) + record = run_tests( + test, spec=spec, name="test_summary_f_alias", capture_output=True ) - result = pytester.runpytest("-rfF") expected = "FAILED test_summary_f_alias.py::test - assert False" - result.stdout.fnmatch_lines([expected]) - assert result.stdout.lines.count(expected) == 1 + record.stdout.fnmatch_lines([expected]) + assert record.output.splitlines().count(expected) == 1 + # ensemble: the folded skip line is `:`, and an ensemble + # item's file:line is the host `test_terminal.py`. def test_summary_s_alias(self, pytester: Pytester) -> None: """Test that 's' and 'S' report chars are aliases and don't show up twice in the summary""" pytester.makepyfile( @@ -1199,6 +1352,7 @@ def test(): result.stdout.fnmatch_lines([expected]) assert result.stdout.lines.count(expected) == 1 + # ensemble: the folded skip line is `:`, host-anchored. def test_summary_s_folded(self, pytester: Pytester) -> None: """Test that skipped tests are correctly folded""" pytester.makepyfile( @@ -1216,26 +1370,26 @@ def test(param): result.stdout.fnmatch_lines([expected]) assert result.stdout.lines.count(expected) == 1 - def test_summary_s_unfolded(self, pytester: Pytester) -> None: + def test_summary_s_unfolded(self, tmp_path: Path) -> None: """Test that skipped tests are not folded if --no-fold-skipped is set""" - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize("param", [True, False]) - @pytest.mark.skip("Some reason") - def test(param): - pass - """ + @pytest.mark.parametrize("param", [True, False]) + @pytest.mark.skip("Some reason") + def test(param): + pass + + spec = ConfigSpec(rootpath=tmp_path, args=("-rs", "--no-fold-skipped")) + record = run_tests( + test, spec=spec, name="test_summary_s_unfolded", capture_output=True ) - result = pytester.runpytest("-rs", "--no-fold-skipped") expected = [ "SKIPPED test_summary_s_unfolded.py::test[True] - Skipped: Some reason", "SKIPPED test_summary_s_unfolded.py::test[False] - Skipped: Some reason", ] - result.stdout.fnmatch_lines(expected) - assert result.stdout.lines.count(expected[0]) == 1 - assert result.stdout.lines.count(expected[1]) == 1 + record.stdout.fnmatch_lines(expected) + lines = record.output.splitlines() + assert lines.count(expected[0]) == 1 + assert lines.count(expected[1]) == 1 @pytest.mark.parametrize( @@ -1247,18 +1401,31 @@ def test(param): ids=("on CI", "not on CI"), ) def test_fail_extra_reporting( - pytester: Pytester, monkeypatch, use_ci: bool, expected_message: str + tmp_path: Path, monkeypatch, use_ci: bool, expected_message: str ) -> None: if use_ci: monkeypatch.setenv("CI", "true") else: monkeypatch.delenv("CI", raising=False) monkeypatch.setenv("COLUMNS", "80") - pytester.makepyfile("def test_this(): assert 0, 'this_failed' * 100") - result = pytester.runpytest("-rN") - result.stdout.no_fnmatch_line("*short test summary*") - result = pytester.runpytest() - result.stdout.fnmatch_lines( + + def test_this(): + assert 0, "this_failed" * 100 + + record = run_tests( + test_this, + spec=ConfigSpec(rootpath=tmp_path, args=("-rN",)), + name="test_fail_extra_reporting", + capture_output=True, + ) + record.stdout.no_fnmatch_line("*short test summary*") + record = run_tests( + test_this, + rootpath=tmp_path, + name="test_fail_extra_reporting", + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "*test summary*", f"FAILED test_fail_extra_reporting.py::test_this {expected_message}", @@ -1266,26 +1433,51 @@ def test_fail_extra_reporting( ) -def test_fail_reporting_on_pass(pytester: Pytester) -> None: - pytester.makepyfile("def test_this(): assert 1") - result = pytester.runpytest("-rf") - result.stdout.no_fnmatch_line("*short test summary*") +def test_fail_reporting_on_pass(tmp_path: Path) -> None: + def test_this(): + assert 1 + record = run_tests( + test_this, + spec=ConfigSpec(rootpath=tmp_path, args=("-rf",)), + capture_output=True, + ) + record.stdout.no_fnmatch_line("*short test summary*") -def test_pass_extra_reporting(pytester: Pytester) -> None: - pytester.makepyfile("def test_this(): assert 1") - result = pytester.runpytest() - result.stdout.no_fnmatch_line("*short test summary*") - result = pytester.runpytest("-rp") - result.stdout.fnmatch_lines(["*test summary*", "PASS*test_pass_extra_reporting*"]) + +def test_pass_extra_reporting(tmp_path: Path) -> None: + def test_this(): + assert 1 + + record = run_tests( + test_this, + rootpath=tmp_path, + name="test_pass_extra_reporting", + capture_output=True, + ) + record.stdout.no_fnmatch_line("*short test summary*") + record = run_tests( + test_this, + spec=ConfigSpec(rootpath=tmp_path, args=("-rp",)), + name="test_pass_extra_reporting", + capture_output=True, + ) + record.stdout.fnmatch_lines(["*test summary*", "PASS*test_pass_extra_reporting*"]) -def test_pass_reporting_on_fail(pytester: Pytester) -> None: - pytester.makepyfile("def test_this(): assert 0") - result = pytester.runpytest("-rp") - result.stdout.no_fnmatch_line("*short test summary*") +def test_pass_reporting_on_fail(tmp_path: Path) -> None: + def test_this(): + assert 0 + + record = run_tests( + test_this, + spec=ConfigSpec(rootpath=tmp_path, args=("-rp",)), + capture_output=True, + ) + record.stdout.no_fnmatch_line("*short test summary*") +# ensemble: asserts on `Captured stdout` sections, which need capture. def test_pass_output_reporting(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1326,6 +1518,9 @@ def test_pass_no_output(): ) +# ensemble: asserts the ` [100%]` progress column (only written from +# `pytest_runtestloop`, which an ensemble does not run) and the +# `test_color_yes.py:5:` crash lines, which are host-anchored. def test_color_yes(pytester: Pytester, color_mapping) -> None: p1 = pytester.makepyfile( """ @@ -1385,34 +1580,41 @@ def test_this(): ) -def test_color_no(pytester: Pytester) -> None: - pytester.makepyfile("def test_this(): assert 1") - result = pytester.runpytest("--color=no") - assert "test session starts" in result.stdout.str() - result.stdout.no_fnmatch_line("*\x1b[1m*") +def test_color_no(tmp_path: Path) -> None: + def test_this(): + assert 1 + + record = run_tests( + test_this, + spec=ConfigSpec(rootpath=tmp_path, args=("--color=no",)), + capture_output=True, + ) + assert "test session starts" in record.output + record.stdout.no_fnmatch_line("*\x1b[1m*") @pytest.mark.parametrize("verbose", [True, False]) -def test_color_yes_collection_on_non_atty(pytester: Pytester, verbose) -> None: +def test_color_yes_collection_on_non_atty(tmp_path: Path, verbose) -> None: """#1397: Skip collect progress report when working on non-terminals.""" - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize('i', range(10)) - def test_this(i): - assert 1 - """ - ) + + @pytest.mark.parametrize("i", range(10)) + def test_this(i): + assert 1 + args = ["--color=yes"] if verbose: args.append("-vv") - result = pytester.runpytest(*args) - assert "test session starts" in result.stdout.str() - assert "\x1b[1m" in result.stdout.str() - result.stdout.no_fnmatch_line("*collecting 10 items*") + record = run_tests( + test_this, + spec=ConfigSpec(rootpath=tmp_path, args=tuple(args)), + capture_output=True, + ) + assert "test session starts" in record.output + assert "\x1b[1m" in record.output + record.stdout.no_fnmatch_line("*collecting 10 items*") if verbose: - assert "collecting ..." in result.stdout.str() - assert "collected 10 items" in result.stdout.str() + assert "collecting ..." in record.output + assert "collected 10 items" in record.output def test_getreportopt() -> None: @@ -1474,25 +1676,27 @@ class Option: assert getreportopt(config) == "fE" -def test_terminalreporter_reportopt_addopts(pytester: Pytester) -> None: - pytester.makeini("[pytest]\naddopts=-rs") - pytester.makepyfile( - """ - import pytest +def test_terminalreporter_reportopt_addopts(tmp_path: Path) -> None: + @pytest.fixture + def tr(request): + tr = request.config.pluginmanager.getplugin("terminalreporter") + return tr - @pytest.fixture - def tr(request): - tr = request.config.pluginmanager.getplugin("terminalreporter") - return tr - def test_opt(tr): - assert tr.hasopt('skipped') - assert not tr.hasopt('qwe') - """ + def test_opt(tr): + assert tr.hasopt("skipped") + assert not tr.hasopt("qwe") + + spec = ConfigSpec(rootpath=tmp_path, inicfg={"addopts": "-rs"}) + record = run_tests( + build_module("test_reportopt_addopts", tr=tr, test_opt=test_opt), + spec=spec, + capture_output=True, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 passed*"]) + record.stdout.fnmatch_lines(["*1 passed*"]) + record.assert_outcomes(passed=1) +# ensemble: asserts `*:8*`; an ensemble item's file:line is host-anchored. def test_tbstyle_short(pytester: Pytester) -> None: p = pytester.makepyfile( """ @@ -1517,6 +1721,8 @@ def test_opt(arg): assert "assert x" in s +# ensemble: asserts the NO_TESTS_COLLECTED exit code, which comes from +# `wrap_session`. def test_traceconfig(pytester: Pytester) -> None: result = pytester.runpytest("--traceconfig") result.stdout.fnmatch_lines(["*active plugins*"]) @@ -1527,6 +1733,8 @@ class TestGenericReporting: """Test class which can be subclassed with a different option provider to run e.g. distributed tests.""" + # ensemble: needs a module that raises ImportError at import time; + # ensemble sources are already-imported objects. def test_collect_fail(self, pytester: Pytester, option) -> None: pytester.makepyfile("import xyz\n") result = pytester.runpytest(*option.args) @@ -1534,6 +1742,8 @@ def test_collect_fail(self, pytester: Pytester, option) -> None: ["ImportError while importing*", "*No module named *xyz*", "*1 error*"] ) + # ensemble: `! stopping after N failures !` is emitted from the run loop + # in `_pytest.main`, which an ensemble bypasses. def test_maxfailures(self, pytester: Pytester, option) -> None: pytester.makepyfile( """ @@ -1555,6 +1765,7 @@ def test_3(): ] ) + # ensemble: `! session_interrupted !` is emitted from the run loop. def test_maxfailures_with_interrupted(self, pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1574,25 +1785,23 @@ def test(request): ] ) - def test_tb_option(self, pytester: Pytester, option) -> None: - pytester.makepyfile( - """ - import pytest - def g(): - raise IndexError - def test_func(): - print(6*7) - g() # --calling-- - """ - ) + def test_tb_option(self, tmp_path: Path, option) -> None: + def g(): + raise IndexError + + def test_func(): + print(6 * 7) + g() # --calling-- + + module = build_module("test_tb_option", g=g, test_func=test_func) for tbopt in ["long", "short", "no"]: print(f"testing --tb={tbopt}...") - result = pytester.runpytest("-rN", f"--tb={tbopt}") - s = result.stdout.str() + spec = ConfigSpec(rootpath=tmp_path, args=("-rN", f"--tb={tbopt}")) + s = run_tests(module, spec=spec, capture_output=True).output if tbopt == "long": - assert "print(6*7)" in s + assert "print(6 * 7)" in s else: - assert "print(6*7)" not in s + assert "print(6 * 7)" not in s if tbopt != "no": assert "--calling--" in s assert "IndexError" in s @@ -1601,6 +1810,7 @@ def test_func(): assert "--calling--" not in s assert "IndexError" not in s + # ensemble: asserts a `Captured stdout call` section, which needs capture. def test_tb_line_show_capture(self, pytester: Pytester, option) -> None: output_to_capture = "help! let me out!" pytester.makepyfile( @@ -1614,6 +1824,7 @@ def test_fail(): result = pytester.runpytest("--tb=line") result.stdout.fnmatch_lines(["*- Captured stdout call -*", output_to_capture]) + # ensemble: asserts `:` crash lines, host-anchored. def test_tb_crashline(self, pytester: Pytester, option) -> None: p = pytester.makepyfile( """ @@ -1635,6 +1846,7 @@ def test_func2(): s = result.stdout.str() assert "def test_func2" not in s + # ensemble: asserts a `:` crash line, host-anchored. def test_tb_crashline_pytrace_false(self, pytester: Pytester, option) -> None: p = pytester.makepyfile( """ @@ -1648,6 +1860,8 @@ def test_func1(): bn = p.name result.stdout.fnmatch_lines([f"*{bn}:3: Failed: test_func1"]) + # ensemble: the point is that a subdirectory conftest contributes header + # lines; ensembles have no directory tree to scope conftests to. def test_pytest_report_header(self, pytester: Pytester, option) -> None: pytester.makeconftest( """ @@ -1667,6 +1881,7 @@ def pytest_report_header(config, start_path): result = pytester.runpytest("a") result.stdout.fnmatch_lines(["*hello: 42*", "line1", str(pytester.path)]) + # ensemble: `--show-capture` is entirely about captured output sections. def test_show_capture(self, pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1718,6 +1933,7 @@ def test_one(): assert "!This is stderr!" not in stdout assert "!This is a warning log msg!" not in stdout + # ensemble: `--show-capture` is entirely about captured output sections. def test_show_capture_with_teardown_logs(self, pytester: Pytester) -> None: """Ensure that the capturing of teardown logs honor --show-capture setting""" pytester.makepyfile( @@ -1759,6 +1975,7 @@ def test_func(): assert "!log!" not in result +# ensemble: uses the `capfd` fixture, which needs capture. @pytest.mark.xfail("not hasattr(os, 'dup')") def test_fdopen_kept_alive_issue124(pytester: Pytester) -> None: pytester.makepyfile( @@ -1778,6 +1995,8 @@ def test_close_kept_alive_file(): result.stdout.fnmatch_lines(["*2 passed*"]) +# ensemble: asserts the file name in a native traceback frame, which for an +# ensemble source is the host `test_terminal.py`. def test_tbstyle_native_setup_error(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1796,6 +2015,8 @@ def test_error_fixture(setup_error_fixture): ) +# ensemble: asserts `exitstatus: 5` (NO_TESTS_COLLECTED); the exit status an +# ensemble hands to `pytest_sessionfinish` is not computed by `wrap_session`. def test_terminal_summary(pytester: Pytester) -> None: pytester.makeconftest( """ @@ -1816,6 +2037,8 @@ def pytest_terminal_summary(terminalreporter, exitstatus): ) +# ensemble: asserts `*conftest.py:3:*internal warning`, i.e. the file and line +# of the warning's origin, which for an ensemble plugin object is the host file. @pytest.mark.filterwarnings("default::UserWarning") def test_terminal_summary_warnings_are_displayed(pytester: Pytester) -> None: """Test that warnings emitted during pytest_terminal_summary are displayed. @@ -1853,18 +2076,27 @@ def test_failure(): assert stdout.count("=== warnings summary ") == 2 -@pytest.mark.filterwarnings("default::UserWarning") -def test_terminal_summary_warnings_header_once(pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_failure(): - import warnings - warnings.warn("warning_from_" + "test") - assert 0 - """ +def test_terminal_summary_warnings_header_once(tmp_path: Path) -> None: + def test_failure(): + import warnings + + warnings.warn("warning_from_" + "test") + assert 0 + + # The host suite runs with `filterwarnings = error`; the ensemble needs + # the warning to be shown rather than raised. + spec = ConfigSpec( + rootpath=tmp_path, + args=("-ra",), + inicfg={"filterwarnings": ["default::UserWarning"]}, ) - result = pytester.runpytest("-ra") - result.stdout.fnmatch_lines( + record = run_tests( + test_failure, + spec=spec, + name="test_terminal_summary_warnings_header_once", + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "*= warnings summary =*", "*warning_from_test*", @@ -1872,25 +2104,32 @@ def test_failure(): "*== 1 failed, 1 warning in *", ] ) - result.stdout.no_fnmatch_line("*None*") - stdout = result.stdout.str() + record.stdout.no_fnmatch_line("*None*") + stdout = record.output assert stdout.count("warning_from_test") == 1 assert stdout.count("=== warnings summary ") == 1 -@pytest.mark.filterwarnings("default") -def test_terminal_no_summary_warnings_header_once(pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_failure(): - import warnings - warnings.warn("warning_from_" + "test") - assert 0 - """ +def test_terminal_no_summary_warnings_header_once(tmp_path: Path) -> None: + def test_failure(): + import warnings + + warnings.warn("warning_from_" + "test") + assert 0 + + spec = ConfigSpec( + rootpath=tmp_path, + args=("--no-summary",), + inicfg={"filterwarnings": ["default"]}, ) - result = pytester.runpytest("--no-summary") - result.stdout.no_fnmatch_line("*= warnings summary =*") - result.stdout.no_fnmatch_line("*= short test summary info =*") + record = run_tests( + test_failure, + spec=spec, + name="test_terminal_no_summary_warnings_header_once", + capture_output=True, + ) + record.stdout.no_fnmatch_line("*= warnings summary =*") + record.stdout.no_fnmatch_line("*= short test summary info =*") @pytest.fixture(scope="session") @@ -2077,22 +2316,44 @@ class TestClassicOutputStyle: """Ensure classic output style works as expected (#3883)""" @pytest.fixture - def test_files(self, pytester: Pytester) -> None: - pytester.makepyfile( - **{ - "test_one.py": "def test_one(): pass", - "test_two.py": "def test_two(): assert 0", - "sub/test_three.py": """ - def test_three_1(): pass - def test_three_2(): assert 0 - def test_three_3(): pass - """, - } + def test_files(self) -> tuple[object, ...]: + def test_one(): + pass + + def test_two(): + assert 0 + + def test_three_1(): + pass + + def test_three_2(): + assert 0 + + def test_three_3(): + pass + + # Collected in the order a filesystem walk would produce them. + return ( + build_module( + "sub/test_three", + test_three_1=test_three_1, + test_three_2=test_three_2, + test_three_3=test_three_3, + ), + build_module("test_one", test_one=test_one), + build_module("test_two", test_two=test_two), ) - def test_normal_verbosity(self, pytester: Pytester, test_files) -> None: - result = pytester.runpytest("-o", "console_output_style=classic") - result.stdout.fnmatch_lines( + @staticmethod + def _run(tmp_path: Path, test_files, *args: str): + spec = ConfigSpec( + rootpath=tmp_path, args=("-o", "console_output_style=classic", *args) + ) + return run_tests(*test_files, spec=spec, capture_output=True) + + def test_normal_verbosity(self, tmp_path: Path, test_files) -> None: + record = self._run(tmp_path, test_files) + record.stdout.fnmatch_lines( [ f"sub{os.sep}test_three.py .F.", "test_one.py .", @@ -2100,10 +2361,11 @@ def test_normal_verbosity(self, pytester: Pytester, test_files) -> None: "*2 failed, 3 passed in*", ] ) + record.assert_outcomes(passed=3, failed=2) - def test_verbose(self, pytester: Pytester, test_files) -> None: - result = pytester.runpytest("-o", "console_output_style=classic", "-v") - result.stdout.fnmatch_lines( + def test_verbose(self, tmp_path: Path, test_files) -> None: + record = self._run(tmp_path, test_files, "-v") + record.stdout.fnmatch_lines( [ f"sub{os.sep}test_three.py::test_three_1 PASSED", f"sub{os.sep}test_three.py::test_three_2 FAILED", @@ -2113,12 +2375,15 @@ def test_verbose(self, pytester: Pytester, test_files) -> None: "*2 failed, 3 passed in*", ] ) + record.assert_outcomes(passed=3, failed=2) - def test_quiet(self, pytester: Pytester, test_files) -> None: - result = pytester.runpytest("-o", "console_output_style=classic", "-q") - result.stdout.fnmatch_lines([".F..F", "*2 failed, 3 passed in*"]) + def test_quiet(self, tmp_path: Path, test_files) -> None: + record = self._run(tmp_path, test_files, "-q") + record.stdout.fnmatch_lines([".F..F", "*2 failed, 3 passed in*"]) + record.assert_outcomes(passed=3, failed=2) +# ensemble: asserts a usage error written to stderr by the command line parser. def test_console_output_style_invalid(pytester: Pytester) -> None: """An invalid console_output_style fails with a clean usage error.""" result = pytester.runpytest("-o", "console_output_style=fancy") @@ -2132,6 +2397,12 @@ def test_console_output_style_invalid(pytester: Pytester) -> None: ) +# ensemble: every test below asserts on the progress column (` [ 50%]`, +# ` [10/20]`, a duration). The reporter only shows it when capturing is +# active - process-global state an ensemble does not install - and the +# final `[100%]` is written from a `pytest_runtestloop` wrapper, which an +# ensemble never runs. `test_zero_tests_collected` is left for the same +# reason: with the progress column off, the division it guards never happens. class TestProgressOutputStyle: @pytest.fixture def many_tests_files(self, pytester: Pytester) -> None: @@ -2452,6 +2723,8 @@ def test_capture_no_progress_enabled( ) +# ensemble: every test below asserts on the progress column; see +# TestProgressOutputStyle. class TestProgressWithTeardown: """Ensure we show the correct percentages for tests that fail during teardown (#3088)""" @@ -2660,6 +2933,10 @@ def markup(self, word: str, **markup: str): check("🉐🉐🉐🉐🉐\n2nd line", 80, "FAILED nodeid::🉐::withunicode - 🉐🉐🉐🉐🉐") +# ensemble: the assertion explanation is produced by the *host* assertion +# plugin (sources in this file are rewritten by the host, and +# `_pytest.assertion.util` keeps its verbosity in a module global bound at +# host configure time), so `-vv` inside an ensemble does not un-truncate it. def test_short_summary_with_verbose( monkeypatch: MonkeyPatch, pytester: Pytester ) -> None: @@ -2697,6 +2974,8 @@ def test(): ) +# ensemble: assertion verbosity is host-global; see +# test_short_summary_with_verbose. def test_full_sequence_print_with_vv( monkeypatch: MonkeyPatch, pytester: Pytester ) -> None: @@ -2726,6 +3005,8 @@ def test_len_dict(): ) +# ensemble: assertion verbosity is host-global; see +# test_short_summary_with_verbose. def test_force_short_summary(monkeypatch: MonkeyPatch, pytester: Pytester) -> None: monkeypatch.setattr(_pytest.terminal, "running_on_ci", lambda: False) @@ -2781,6 +3062,8 @@ def test_format_node_duration(seconds: float, expected: str) -> None: assert format_node_duration(seconds) == expected +# ensemble: asserts `! Interrupted: 1 error during collection !`, produced by +# `wrap_session`, and needs a module that fails at import. def test_collecterror(pytester: Pytester) -> None: p1 = pytester.makepyfile("raise SyntaxError()") result = pytester.runpytest("-ra", str(p1)) @@ -2798,12 +3081,15 @@ def test_collecterror(pytester: Pytester) -> None: ) +# ensemble: needs a module that fails at import. def test_no_summary_collecterror(pytester: Pytester) -> None: p1 = pytester.makepyfile("raise SyntaxError()") result = pytester.runpytest("-ra", "--no-summary", str(p1)) result.stdout.no_fnmatch_line("*= ERRORS =*") +# ensemble: asserts the `<- ` suffix at -vv; ensemble items always +# carry a `<- .../test_terminal.py` suffix instead. def test_via_exec(pytester: Pytester) -> None: p1 = pytester.makepyfile("exec('def test_via_exec(): pass')") result = pytester.runpytest(str(p1), "-vv") @@ -2813,6 +3099,11 @@ def test_via_exec(pytester: Pytester) -> None: class TestCodeHighlight: + # ensemble: the rendering asserted here is of the source line + # `assert 1 == 10`, character for character. As real code in this file that + # line needs a `# type: ignore[comparison-overlap]` (the suite runs mypy + # with strict_equality), and the trailing comment becomes part of the + # highlighted line - so the port would have to weaken the pattern. def test_code_highlight_simple(self, pytester: Pytester, color_mapping) -> None: pytester.makepyfile( """ @@ -2831,6 +3122,9 @@ def test_foo(): ) ) + # ensemble: the source under test is a `print('''...'''); assert 0` + # one-liner whose exact layout is the point; as real code in this file the + # formatter would rewrite it and the expected highlighting with it. def test_code_highlight_continuation( self, pytester: Pytester, color_mapping ) -> None: @@ -2854,6 +3148,7 @@ def test_foo(): ) ) + # ensemble: see test_code_highlight_simple. def test_code_highlight_custom_theme( self, pytester: Pytester, color_mapping, monkeypatch: MonkeyPatch ) -> None: @@ -2876,6 +3171,7 @@ def test_foo(): ) ) + # ensemble: asserts a startup error written to stderr by a subprocess. def test_code_highlight_invalid_theme( self, pytester: Pytester, color_mapping, monkeypatch: MonkeyPatch ) -> None: @@ -2892,6 +3188,7 @@ def test_foo(): "Hint: See available pygments styles with `pygmentize -L styles`." ) + # ensemble: asserts a startup error written to stderr by a subprocess. def test_code_highlight_invalid_theme_mode( self, pytester: Pytester, color_mapping, monkeypatch: MonkeyPatch ) -> None: @@ -2933,6 +3230,8 @@ def test_format_trimmed() -> None: assert _format_trimmed(" ({}) ", msg, len(msg) + 3) == " (unconditional ...) " +# ensemble: asserts a `configfile:` header line; ensemble configs never read +# a config file. def test_warning_when_init_trumps_pyproject_toml( pytester: Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -2954,6 +3253,7 @@ def test_warning_when_init_trumps_pyproject_toml( ) +# ensemble: asserts a `configfile:` header line. def test_warning_when_init_trumps_multiple_files( pytester: Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -2986,6 +3286,7 @@ def test_warning_when_init_trumps_multiple_files( ) +# ensemble: asserts a `configfile:` header line. def test_no_warning_when_init_but_pyproject_toml_has_no_entry( pytester: Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -3007,6 +3308,7 @@ def test_no_warning_when_init_but_pyproject_toml_has_no_entry( ) +# ensemble: asserts a `configfile:` header line. def test_no_warning_on_terminal_with_a_single_config_file( pytester: Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -3027,6 +3329,10 @@ def test_no_warning_on_terminal_with_a_single_config_file( ) +# ensemble: every test below matches whole blocks of consecutive lines whose +# exact column layout is the point - the trailing ` [100%]` progress column +# (never rendered by an ensemble, see TestProgressOutputStyle) is part of that +# layout, and half of them assert on `--collect-only` rendering. class TestFineGrainedTestCase: DEFAULT_FILE_CONTENTS = """ import pytest @@ -3260,26 +3566,42 @@ def _initialize_files( return p -def test_summary_xfail_reason(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest +def test_summary_xfail_reason(tmp_path: Path) -> None: + @pytest.mark.xfail + def test_xfail(): + assert False - @pytest.mark.xfail - def test_xfail(): - assert False + @pytest.mark.xfail(reason="foo") + def test_xfail_reason(): + assert False - @pytest.mark.xfail(reason="foo") - def test_xfail_reason(): - assert False - """ + record = run_tests( + test_xfail, + test_xfail_reason, + spec=ConfigSpec(rootpath=tmp_path, args=("-rx",)), + name="test_summary_xfail_reason", + capture_output=True, ) - result = pytester.runpytest("-rx") expect1 = "XFAIL test_summary_xfail_reason.py::test_xfail" expect2 = "XFAIL test_summary_xfail_reason.py::test_xfail_reason - foo" - result.stdout.fnmatch_lines([expect1, expect2]) - assert result.stdout.lines.count(expect1) == 1 - assert result.stdout.lines.count(expect2) == 1 + record.stdout.fnmatch_lines([expect1, expect2]) + lines = record.output.splitlines() + assert lines.count(expect1) == 1 + assert lines.count(expect2) == 1 + + +@pytest.fixture() +def xfail_testsources() -> tuple[object, ...]: + def test_fail(): + a, b = 1, 2 + assert a == b + + @pytest.mark.xfail + def test_xfail(): + c, d = 3, 4 + assert c == d + + return (test_fail, test_xfail) @pytest.fixture() @@ -3300,11 +3622,13 @@ def test_xfail(): ) -def test_xfail_tb_default(xfail_testfile, pytester: Pytester) -> None: - result = pytester.runpytest(xfail_testfile) +def test_xfail_tb_default(xfail_testsources, tmp_path: Path) -> None: + record = run_tests( + *xfail_testsources, rootpath=tmp_path, name="test_xfail_tb", capture_output=True + ) # test_fail, show traceback - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*= FAILURES =*", "*_ test_fail _*", @@ -3316,14 +3640,20 @@ def test_xfail_tb_default(xfail_testfile, pytester: Pytester) -> None: ) # test_xfail, don't show traceback - result.stdout.no_fnmatch_line("*= XFAILURES =*") + record.stdout.no_fnmatch_line("*= XFAILURES =*") + record.assert_outcomes(failed=1, xfailed=1) -def test_xfail_tb_true(xfail_testfile, pytester: Pytester) -> None: - result = pytester.runpytest(xfail_testfile, "--xfail-tb") +def test_xfail_tb_true(xfail_testsources, tmp_path: Path) -> None: + record = run_tests( + *xfail_testsources, + spec=ConfigSpec(rootpath=tmp_path, args=("--xfail-tb",)), + name="test_xfail_tb", + capture_output=True, + ) # both test_fail and test_xfail, show traceback - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*= FAILURES =*", "*_ test_fail _*", @@ -3340,8 +3670,11 @@ def test_xfail_tb_true(xfail_testfile, pytester: Pytester) -> None: "*short test summary info*", ] ) + record.assert_outcomes(failed=1, xfailed=1) +# ensemble: `--tb=line` renders `:: `, and an ensemble +# item's file:line is the host `test_terminal.py`. def test_xfail_tb_line(xfail_testfile, pytester: Pytester) -> None: result = pytester.runpytest(xfail_testfile, "--xfail-tb", "--tb=line") @@ -3357,28 +3690,29 @@ def test_xfail_tb_line(xfail_testfile, pytester: Pytester) -> None: ) -def test_summary_xpass_reason(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest +def test_summary_xpass_reason(tmp_path: Path) -> None: + @pytest.mark.xfail + def test_pass(): ... - @pytest.mark.xfail - def test_pass(): - ... + @pytest.mark.xfail(reason="foo") + def test_reason(): ... - @pytest.mark.xfail(reason="foo") - def test_reason(): - ... - """ + record = run_tests( + test_pass, + test_reason, + spec=ConfigSpec(rootpath=tmp_path, args=("-rX",)), + name="test_summary_xpass_reason", + capture_output=True, ) - result = pytester.runpytest("-rX") expect1 = "XPASS test_summary_xpass_reason.py::test_pass" expect2 = "XPASS test_summary_xpass_reason.py::test_reason - foo" - result.stdout.fnmatch_lines([expect1, expect2]) - assert result.stdout.lines.count(expect1) == 1 - assert result.stdout.lines.count(expect2) == 1 + record.stdout.fnmatch_lines([expect1, expect2]) + lines = record.output.splitlines() + assert lines.count(expect1) == 1 + assert lines.count(expect2) == 1 +# ensemble: asserts a `Captured stdout call` section, which needs capture. def test_xpass_output(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -3403,6 +3737,8 @@ def test_pass(): class TestNodeIDHandling: + # ensemble: the point is how nodeids are rendered relative to a rootdir + # that differs from the invocation dir, which needs a real directory tree. def test_nodeid_handling_windows_paths(self, pytester: Pytester, tmp_path) -> None: """Test the correct handling of Windows-style paths with backslashes.""" pytester.makeini("[pytest]") # Change `config.rootpath` @@ -3455,6 +3791,9 @@ def write_raw(content: str, *, flush: bool = False) -> None: tr._progress_nodeids_reported = set() return tr + # ensemble: the plugin decides whether to register from the terminal + # reporter's file being a tty; an ensemble's file is a private buffer, so + # monkeypatching `sys.stdout.isatty` would not reach it. @pytest.mark.skipif(sys.platform != "win32", reason="#13896") def test_plugin_registration_enabled_by_default( self, pytester: pytest.Pytester, monkeypatch: MonkeyPatch @@ -3469,6 +3808,7 @@ def test_plugin_registration_enabled_by_default( plugin = config.pluginmanager.get_plugin("terminalprogress") assert plugin is not None + # ensemble: see test_plugin_registration_enabled_by_default. def test_plugin_registred_on_all_platforms_when_explicitly_requested( self, pytester: pytest.Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -3479,6 +3819,7 @@ def test_plugin_registred_on_all_platforms_when_explicitly_requested( plugin = config.pluginmanager.get_plugin("terminalprogress") assert plugin is not None + # ensemble: see test_plugin_registration_enabled_by_default. def test_disabled_for_non_tty( self, pytester: pytest.Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -3488,6 +3829,7 @@ def test_disabled_for_non_tty( plugin = config.pluginmanager.get_plugin("terminalprogress-plugin") assert plugin is None + # ensemble: see test_plugin_registration_enabled_by_default. def test_disabled_for_dumb_terminal( self, pytester: pytest.Pytester, monkeypatch: MonkeyPatch ) -> None: From 53b898900d34231dd0ff24ef8db7aefddaf2a7aa Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 13 Aug 2026 21:38:56 +0200 Subject: [PATCH 14/30] testing: assert the ensemble's own config warnings, not the escaped ones Follow-on to the base branch containing configure-time warnings. test_pytest_configure_warning had been catching the warning through the host's recwarn fixture - it was depending on the leak. It now asserts on record.warnings, and declares the ini filter that makes the warning a warning rather than an error: absent one, an ensemble inherits this suite's filterwarnings = error. --- testing/test_warnings.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/testing/test_warnings.py b/testing/test_warnings.py index f1ce4e0f8e5..fc341bb1495 100644 --- a/testing/test_warnings.py +++ b/testing/test_warnings.py @@ -735,7 +735,7 @@ def test_group_warnings_by_message_summary(pytester: Pytester) -> None: ) -def test_pytest_configure_warning(tmp_path: Path, recwarn) -> None: +def test_pytest_configure_warning(tmp_path: Path) -> None: """Issue 5115.""" class ConfigureWarner: @@ -744,16 +744,21 @@ class ConfigureWarner: def pytest_configure(self): warnings.warn("from pytest_configure") - # A warning issued from ``pytest_configure`` is not recorded (the config - # catches those without recording), but it must not blow the run up - # either; the original's ``ret == 5`` was "no tests collected, no - # internal error". + # A warning issued from ``pytest_configure`` must not blow the run up - + # the original's ``ret == 5`` was "no tests collected, no internal + # error" - and it must be reported by the ensemble rather than escaping + # into whatever is running it. The ini filter is what makes it a warning + # rather than an error: absent one, the ensemble inherits this suite's + # ``filterwarnings = error``. record = run_tests( - spec=ConfigSpec(rootpath=tmp_path, extra_plugins=(ConfigureWarner(),)) + spec=ConfigSpec( + rootpath=tmp_path, + extra_plugins=(ConfigureWarner(),), + inicfg={"filterwarnings": ["always"]}, + ) ) record.assert_outcomes() - warning = recwarn.pop() - assert str(warning.message) == "from pytest_configure" + assert [str(w.message) for w in record.warnings] == ["from pytest_configure"] @pytest.mark.parametrize("tryfirst", [True, False]) From c300c0281fbdf6658dcb5d33fb11a3be02e9d303 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 07:31:07 +0200 Subject: [PATCH 15/30] testing: restore the [100%] progress assertions in test_subtests These five patterns had to drop their trailing "[100%]" when the file was ported, because the terminal reporter defers that final fill to pytest_runtestloop and an ensemble drove the items directly. The ensemble now goes through that hook, so the fill is rendered and the patterns match upstream again - one less place where a port asserted less than the original. --- testing/test_subtests.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/testing/test_subtests.py b/testing/test_subtests.py index ee01d141bf0..7737c81fd6d 100644 --- a/testing/test_subtests.py +++ b/testing/test_subtests.py @@ -99,11 +99,7 @@ def test_failures(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: ) record.stdout.fnmatch_lines( [ - # The original also matched a trailing "[100%]" here. The terminal - # reporter defers that final fill to ``pytest_runtestloop``, which - # an ensemble never calls - it drives the items directly. The - # per-test letters, which are what this test is about, are intact. - "test_*.py uFuF.", + "test_*.py uFuF.*[[]100%[]]", *summary_lines, "* 4 failed, 1 passed in *", ] @@ -170,8 +166,7 @@ def test_passes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: ) record.stdout.fnmatch_lines( [ - # see test_failures on the dropped "[100%]" - "test_*.py ..", + "test_*.py ..*[[]100%[]]", "* 2 passed in *", ] ) @@ -229,8 +224,7 @@ def test_skip(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: ) record.stdout.fnmatch_lines( [ - # see test_failures on the dropped "[100%]" - "test_*.py .s", + "test_*.py .s*[[]100%[]]", "*=== short test summary info ===*", # the original spelled out "test_skip.py:9" here; the location of # an in-memory source is anchored in *this* file. @@ -305,8 +299,7 @@ def test_xfail(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: ) record.stdout.fnmatch_lines( [ - # see test_failures on the dropped "[100%]" - "test_*.py .x", + "test_*.py .x*[[]100%[]]", "*=== short test summary info ===*", "* 1 passed, 1 xfailed in *", ] @@ -662,7 +655,7 @@ def test_foo(self) -> None: record.stdout.fnmatch_lines( [ # see test_failures on the dropped "[100%]" - "*.py u.", + "*.py u.*[[]100%[]]", "*=== short test summary info ===*", "SUBFAILED[[]subtest 2[]] *.py::T::test_foo - AssertionError: fail subtest 2", "* 1 failed, 1 passed in *", From 0511dacd393d9d8ab6ce143809609142c973aaf1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 07:43:12 +0200 Subject: [PATCH 16/30] testing: run the example scripts as themselves The example-driven tests were either left on pytester or ported by inlining their source - and inlining orphaned the example files while moving every reported location into the host test file. module_from_path() plus EnsembleModule keeping a real __file__ makes the third option work: run the example from where it lives. Eight tests in fixtures.py move off inlined copies and back onto the real scripts, and four more in test_unittest.py come off pytester entirely. The locations are the point. test_funcarg_lookupfails can now assert "file *test_funcarg_lookupfails.py, line 12" which named this file when the source was inlined, so the pattern could not be carried over at all. Same for test_fixture_named_request's "*test_fixture_named_request.py:8" and the nodeid assertions. test_extend_fixture_conftest_module is the shape worth noting: the example's conftest.py sits at the example's own root, which is exactly what an ensemble plugin object stands for, so both files now run as themselves. Every example script under fixtures/ and unittest/ is referenced again - the eight that inlining had orphaned are back in use. --- testing/python/fixtures.py | 193 ++++++++++++++----------------------- testing/test_unittest.py | 61 ++++++------ 2 files changed, 103 insertions(+), 151 deletions(-) diff --git a/testing/python/fixtures.py b/testing/python/fixtures.py index facf41d0c2e..fc014d9daf5 100644 --- a/testing/python/fixtures.py +++ b/testing/python/fixtures.py @@ -13,6 +13,7 @@ from _pytest.ensemble import collect_tests from _pytest.ensemble import ConfigSpec from _pytest.ensemble import Ensemble +from _pytest.ensemble import module_from_path from _pytest.ensemble import run_tests from _pytest.fixtures import deduplicate_names from _pytest.fixtures import ParamValueKey @@ -21,6 +22,7 @@ from _pytest.mark.structures import MarkDecorator from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import get_public_names +from _pytest.pytester import LineMatcher from _pytest.pytester import Pytester from _pytest.python import Function import pytest @@ -36,6 +38,11 @@ def unregistered_mark(name: str, *args: object, **kwargs: object) -> MarkDecorat return MarkDecorator(Mark(name, args, kwargs, _ispytest=True), _ispytest=True) +#: The example scripts, run as themselves rather than copied somewhere first. +EXAMPLES = Path(__file__).parent.parent / "example_scripts" +FILL_FIXTURES = EXAMPLES / "fixtures/fill_fixtures" + + def test_getfuncargnames_functions(): """Test getfuncargnames for normal functions""" @@ -151,59 +158,42 @@ class T: @pytest.mark.pytester_example_path("fixtures/fill_fixtures") class TestFillFixtures: - def test_funcarg_lookupfails(self, tmp_path: Path) -> None: - @pytest.fixture - def xyzsomething(request): - return 42 - - def test_func(some): - pass - + def test_funcarg_lookupfails(self) -> None: + example = FILL_FIXTURES / "test_funcarg_lookupfails.py" record = run_tests( - xyzsomething, test_func, rootpath=tmp_path, capture_output=True + module_from_path(example), rootpath=example.parent, capture_output=True ) # A fixture missing at setup is an error, not a failure. record.assert_outcomes(errors=1) record.stdout.fnmatch_lines( [ + # The example is reported at its own path and line, not at + # this file's - that is what running it as itself buys. + "file *test_funcarg_lookupfails.py, line 12", "*def test_func(some)*", "*fixture*some*not found*", "*xyzsomething*", ] ) - def test_detect_recursive_dependency_error(self, tmp_path: Path) -> None: - @pytest.fixture - def fix1(fix2): - return 1 - - @pytest.fixture - def fix2(fix1): - return 1 - - def test(fix1): - pass - - record = run_tests(fix1, fix2, test, rootpath=tmp_path, capture_output=True) + def test_detect_recursive_dependency_error(self) -> None: + example = FILL_FIXTURES / "test_detect_recursive_dependency_error.py" + record = run_tests( + module_from_path(example), rootpath=example.parent, capture_output=True + ) + record.assert_outcomes(errors=1) record.stdout.fnmatch_lines( ["*recursive dependency involving fixture 'fix1' detected*"] ) - def test_funcarg_basic(self, tmp_path: Path) -> None: - @pytest.fixture - def some(request): - return request.function.__name__ - - @pytest.fixture - def other(request): - return 42 - - def test_func(some, other): - pass - - with Ensemble(some, other, test_func, rootpath=tmp_path) as ensemble: + def test_funcarg_basic(self) -> None: + example = FILL_FIXTURES / "test_funcarg_basic.py" + with Ensemble(module_from_path(example), rootpath=example.parent) as ensemble: (item,) = ensemble.collect() assert isinstance(item, Function) + # The example is collected where it lives, so it keeps its identity. + assert item.nodeid == "test_funcarg_basic.py::test_func" + assert item.path == example # Execute's item's setup, which fills fixtures. item.session._setupstate.setup(item) del item.funcargs["request"] @@ -211,35 +201,24 @@ def test_func(some, other): assert item.funcargs["some"] == "test_func" assert item.funcargs["other"] == 42 - def test_funcarg_lookup_modulelevel(self, tmp_path: Path) -> None: - @pytest.fixture - def something(request): - return request.function.__name__ - - class TestClass: - def test_method(self, something): - assert something == "test_method" - - def test_func(something): - assert something == "test_func" - - record = run_tests(something, TestClass, test_func, rootpath=tmp_path) + def test_funcarg_lookup_modulelevel(self) -> None: + example = FILL_FIXTURES / "test_funcarg_lookup_modulelevel.py" + record = run_tests(module_from_path(example), rootpath=example.parent) record.assert_outcomes(passed=2) + assert sorted(record.by_test) == [ + "test_funcarg_lookup_modulelevel.py::TestClass::test_method", + "test_funcarg_lookup_modulelevel.py::test_func", + ] - def test_funcarg_lookup_classlevel(self, tmp_path: Path) -> None: - class TestClass: - @pytest.fixture - def something(self, request): - return request.instance - - def test_method(self, something): - assert something is self - - record = run_tests(TestClass, rootpath=tmp_path) + def test_funcarg_lookup_classlevel(self) -> None: + example = FILL_FIXTURES / "test_funcarg_lookup_classlevel.py" + record = run_tests(module_from_path(example), rootpath=example.parent) record.assert_outcomes(passed=1) # ensemble: conftest visibility is per-directory, and ensembles have no - # directory tree below the rootdir to scope conftests to. + # directory tree below the rootdir to scope conftests to. Running the + # example modules in place does not help: the conftests beside them are + # exactly what is under test, and ensembles never load conftest files. def test_conftest_funcargs_only_available_in_subdir( self, pytester: Pytester ) -> None: @@ -247,46 +226,29 @@ def test_conftest_funcargs_only_available_in_subdir( result = pytester.runpytest("-v") result.assert_outcomes(passed=2) - def test_extend_fixture_module_class(self, tmp_path: Path) -> None: - @pytest.fixture - def spam(): - return "spam" - - class TestSpam: - @pytest.fixture - def spam(self, spam): - return spam * 2 - - def test_spam(self, spam): - assert spam == "spamspam" - - record = run_tests(spam, TestSpam, rootpath=tmp_path) + def test_extend_fixture_module_class(self) -> None: + example = FILL_FIXTURES / "test_extend_fixture_module_class.py" + record = run_tests(module_from_path(example), rootpath=example.parent) record.assert_outcomes(passed=1) - - def test_extend_fixture_conftest_module(self, tmp_path: Path) -> None: - # The rootdir conftest is reproduced as a plugin object; the second - # run of the original (passing the test file directly) only covered - # conftest collection for an explicit file argument, which an - # ensemble has no equivalent of. - class ConftestPlugin: - @pytest.fixture - def spam(self): - return "spam" - - @pytest.fixture - def spam(spam): - return spam * 2 - - def test_spam(spam): - assert spam == "spamspam" - - spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) - record = run_tests( - build_module("test_extend", spam=spam, test_spam=test_spam), spec=spec - ) + assert record["test_extend_fixture_module_class.py::TestSpam::test_spam"].passed + + def test_extend_fixture_conftest_module(self) -> None: + # The example's conftest sits at its root, which is exactly what an + # ensemble plugin object stands for - so the conftest and the test + # module can both be run as themselves. The second run of the + # original (passing the test file directly) only covered conftest + # collection for an explicit file argument, which an ensemble has no + # equivalent of. + example_dir = FILL_FIXTURES / "test_extend_fixture_conftest_module" + conftest = module_from_path(example_dir / "conftest.py") + example = example_dir / "test_extend_fixture_conftest_module.py" + spec = ConfigSpec(rootpath=example_dir, extra_plugins=(conftest,)) + record = run_tests(module_from_path(example), spec=spec) record.assert_outcomes(passed=1) + assert record["test_extend_fixture_conftest_module.py::test_spam"].passed - # ensemble: two conftests at different directory levels. + # ensemble: two conftests at different directory levels; running the + # example module in place would not load either of them. def test_extend_fixture_conftest_conftest(self, pytester: Pytester) -> None: p = pytester.copy_example() result = pytester.runpytest() @@ -1185,26 +1147,12 @@ def test_function(request, farg): record = run_tests(arg1, farg, sarg, test_function, rootpath=tmp_path) record.assert_outcomes(passed=1) - def test_request_fixturenames_dynamic_fixture(self, tmp_path: Path) -> None: + def test_request_fixturenames_dynamic_fixture(self) -> None: """Regression test for #3057""" - - @pytest.fixture - def dynamic(): - pass - - @pytest.fixture - def a(request): - request.getfixturevalue("dynamic") - - @pytest.fixture - def b(a): - pass - - def test(b, request): - assert request.fixturenames == ["b", "a", "request", "dynamic"] - - record = run_tests(dynamic, a, b, test, rootpath=tmp_path) + example = EXAMPLES / "fixtures/test_getfixturevalue_dynamic.py" + record = run_tests(module_from_path(example), rootpath=example.parent) record.assert_outcomes(passed=1) + assert record["test_getfixturevalue_dynamic.py::test"].passed def test_setupdecorator_and_xunit(self, tmp_path: Path) -> None: values: list[str] = [] @@ -2100,7 +2048,8 @@ def test_package(one): reprec.assertoutcome(passed=2) # ensemble: the example is a directory tree with a conftest defining - # custom collectors for non-python files. + # custom collectors for non-python files. The items under test are not + # python at all, so there is no module to run in place. def test_collect_custom_items(self, pytester: Pytester) -> None: pytester.copy_example("fixtures/custom_item") result = pytester.runpytest("foo") @@ -5151,11 +5100,15 @@ def test_indirect(arg2): ] -# ensemble: asserts the file:line the reserved fixture name was used at. -def test_fixture_named_request(pytester: Pytester) -> None: - pytester.copy_example("fixtures/test_fixture_named_request.py") - result = pytester.runpytest() - result.stdout.fnmatch_lines( +def test_fixture_named_request() -> None: + # The reserved name is rejected by the decorator, so importing the example + # is what raises - there is nothing left to collect afterwards. Importing + # it as itself is what makes the reported location the example's own line; + # an inlined copy would name this file instead. + example = EXAMPLES / "fixtures/test_fixture_named_request.py" + with pytest.raises(pytest.fail.Exception) as excinfo: + module_from_path(example) + LineMatcher(str(excinfo.value).splitlines()).fnmatch_lines( [ "*'request' is a reserved word for fixtures, use another name:", " *test_fixture_named_request.py:8", diff --git a/testing/test_unittest.py b/testing/test_unittest.py index 12844c46631..968d07275bf 100644 --- a/testing/test_unittest.py +++ b/testing/test_unittest.py @@ -12,6 +12,7 @@ from _pytest.ensemble import build_module from _pytest.ensemble import collect_tests from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import module_from_path from _pytest.ensemble import run_tests from _pytest.monkeypatch import MonkeyPatch from _pytest.outcomes import Exit @@ -19,6 +20,10 @@ import pytest +#: The example scripts, run as themselves rather than copied somewhere first. +EXAMPLES = Path(__file__).parent / "example_scripts" + + def test_simple_unittest(tmp_path: Path) -> None: class MyTestCase(unittest.TestCase): def testpassing(self): @@ -1130,12 +1135,11 @@ def test_hello(self): assert record.reports == [] -# ensemble: driven by an example script; an in-memory copy would orphan -# testing/example_scripts/unittest/test_parametrized_fixture_error_message.py -def test_error_message_with_parametrized_fixtures(pytester: Pytester) -> None: - pytester.copy_example("unittest/test_parametrized_fixture_error_message.py") - result = pytester.runpytest() - result.stdout.fnmatch_lines( +def test_error_message_with_parametrized_fixtures() -> None: + example = EXAMPLES / "unittest/test_parametrized_fixture_error_message.py" + module = module_from_path(example) + record = run_tests(module, rootpath=example.parent, capture_output=True) + record.stdout.fnmatch_lines( [ "*test_two does not support fixtures*", "*TestSomethingElse::test_two", @@ -1144,23 +1148,21 @@ def test_error_message_with_parametrized_fixtures(pytester: Pytester) -> None: ) -# ensemble: driven by example scripts; in-memory copies would orphan -# testing/example_scripts/unittest/test_setup_skip*.py @pytest.mark.parametrize( - "test_name, expected_outcome", + "test_name, expected_outcome, outcomes", [ - ("test_setup_skip.py", "1 skipped"), - ("test_setup_skip_class.py", "1 skipped"), - ("test_setup_skip_module.py", "1 error"), + ("test_setup_skip.py", "1 skipped", {"skipped": 1}), + ("test_setup_skip_class.py", "1 skipped", {"skipped": 1}), + ("test_setup_skip_module.py", "1 error", {"errors": 1}), ], ) -def test_setup_inheritance_skipping( - pytester: Pytester, test_name, expected_outcome -) -> None: +def test_setup_inheritance_skipping(test_name, expected_outcome, outcomes) -> None: """Issue #4700""" - pytester.copy_example(f"unittest/{test_name}") - result = pytester.runpytest() - result.stdout.fnmatch_lines([f"* {expected_outcome} in *"]) + example = EXAMPLES / "unittest" / test_name + module = module_from_path(example) + record = run_tests(module, rootpath=example.parent, capture_output=True) + record.stdout.fnmatch_lines([f"* {expected_outcome} in *"]) + record.assert_outcomes(**outcomes) def test_BdbQuit(tmp_path: Path) -> None: @@ -1305,30 +1307,27 @@ def test_1(self): assert tracked == [] -# ensemble: driven by an example script; an in-memory copy would orphan -# testing/example_scripts/unittest/test_unittest_asyncio.py -def test_async_support(pytester: Pytester) -> None: +def test_async_support() -> None: pytest.importorskip("unittest.async_case") - pytester.copy_example("unittest/test_unittest_asyncio.py") - reprec = pytester.inline_run() - reprec.assertoutcome(failed=1, passed=2) + example = EXAMPLES / "unittest/test_unittest_asyncio.py" + module = module_from_path(example) + run_tests(module, rootpath=example.parent).assert_outcomes(failed=1, passed=2) -# ensemble: driven by an example script; an in-memory copy would orphan -# testing/example_scripts/unittest/test_unittest_asynctest.py @pytest.mark.skipif( sys.version_info >= (3, 11), reason="asynctest is not compatible with Python 3.11+" ) -def test_asynctest_support(pytester: Pytester) -> None: +def test_asynctest_support() -> None: """Check asynctest support (#7110)""" pytest.importorskip("asynctest") - pytester.copy_example("unittest/test_unittest_asynctest.py") - reprec = pytester.inline_run() - reprec.assertoutcome(failed=1, passed=2) + example = EXAMPLES / "unittest/test_unittest_asynctest.py" + module = module_from_path(example) + run_tests(module, rootpath=example.parent).assert_outcomes(failed=1, passed=2) -# ensemble: needs a subprocess (the unawaited coroutine warning depends on gc) +# ensemble: needs a subprocess (the unawaited coroutine warning depends on gc), +# so the example script has to be copied somewhere the subprocess can run it. def test_plain_unittest_does_not_support_async(pytester: Pytester) -> None: """Async functions in plain unittest.TestCase subclasses are not supported without plugins. From 30ed821f82a211455b930f792656999f3ab6b5ad Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 07:47:23 +0200 Subject: [PATCH 17/30] testing: port the progress-column tests in test_terminal.py 17 more tests move to _pytest.ensemble now that a private output stream counts as captured and the run goes through pytest_runtestloop: TestProgressOutputStyle (6), TestProgressWithTeardown (4) and the non-collect-only half of TestFineGrainedTestCase (7). Every pattern is kept verbatim - the column renders at the same character positions, needs no console_output_style ini, and ends on the same [100%]/[20/20] fill - and each port gains an assert_outcomes() the pytester original never had. The ported fine-grained tests pin COLUMNS=80, which pytester used to supply implicitly. Two tests are now unportable *because* of the fix and say so: test_capture_no and test_capture_no_progress_enabled assert that --capture=no suppresses the column, and an ensemble's private stream always counts as captured, so porting them would assert nothing. Running total: 70 of 159 tests in this file are off pytester. --- testing/test_terminal.py | 473 ++++++++++++++++++++++++++------------- 1 file changed, 313 insertions(+), 160 deletions(-) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 6a0360f1b49..5a264fb06d4 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -8,6 +8,7 @@ from pathlib import Path import sys import textwrap +from types import ModuleType from types import SimpleNamespace from typing import cast from typing import Literal @@ -24,6 +25,7 @@ from _pytest.ensemble import ConfigSpec from _pytest.ensemble import Ensemble from _pytest.ensemble import run_tests +from _pytest.ensemble import RunRecord from _pytest.mark.structures import Mark from _pytest.mark.structures import MarkDecorator from _pytest.monkeypatch import MonkeyPatch @@ -1518,9 +1520,10 @@ def test_pass_no_output(): ) -# ensemble: asserts the ` [100%]` progress column (only written from -# `pytest_runtestloop`, which an ensemble does not run) and the -# `test_color_yes.py:5:` crash lines, which are host-anchored. +# ensemble: asserts the `test_color_yes.py:5:` crash lines, which for an +# ensemble source are host-anchored. `module_from_path` would make them real, +# but the imported module would not be assertion-rewritten and the +# `E assert 0` explanation would go with it. def test_color_yes(pytester: Pytester, color_mapping) -> None: p1 = pytester.makepyfile( """ @@ -2397,13 +2400,37 @@ def test_console_output_style_invalid(pytester: Pytester) -> None: ) -# ensemble: every test below asserts on the progress column (` [ 50%]`, -# ` [10/20]`, a duration). The reporter only shows it when capturing is -# active - process-global state an ensemble does not install - and the -# final `[100%]` is written from a `pytest_runtestloop` wrapper, which an -# ensemble never runs. `test_zero_tests_collected` is left for the same -# reason: with the progress column off, the division it guards never happens. class TestProgressOutputStyle: + @pytest.fixture + def many_tests_sources(self) -> tuple[ModuleType, ...]: + @pytest.mark.parametrize("i", range(10)) + def test_bar(i): + pass + + @pytest.mark.parametrize("i", range(5)) + def test_foo(i): + pass + + @pytest.mark.parametrize("i", range(5)) + def test_foobar(i): + pass + + return ( + build_module("test_bar", test_bar), + build_module("test_foo", test_foo), + build_module("test_foobar", test_foobar), + ) + + @staticmethod + def _run( + tmp_path: Path, + sources: tuple[ModuleType, ...], + *args: str, + inicfg: dict[str, object] | None = None, + ) -> RunRecord: + spec = ConfigSpec(rootpath=tmp_path, args=args, inicfg=inicfg or {}) + return run_tests(*sources, spec=spec, capture_output=True) + @pytest.fixture def many_tests_files(self, pytester: Pytester) -> None: pytester.makepyfile( @@ -2439,65 +2466,71 @@ def test_foo(i): pass """, ) - def test_zero_tests_collected(self, pytester: Pytester) -> None: + def test_zero_tests_collected(self, tmp_path: Path) -> None: """Some plugins (testmon for example) might issue pytest_runtest_logreport without any tests being actually collected (#2971).""" - pytester.makeconftest( - """ - def pytest_collection_modifyitems(items, config): - from _pytest.runner import CollectReport - for node_id in ('nodeid1', 'nodeid2'): - rep = CollectReport(node_id, 'passed', None, None) - rep.when = 'passed' - rep.duration = 0.1 - config.hook.pytest_runtest_logreport(report=rep) - """ + + class LogReportsWithoutItems: + def pytest_collection_modifyitems(self, items, config): + for node_id in ("nodeid1", "nodeid2"): + rep = CollectReport(node_id, "passed", None, None) + rep.when = "passed" + rep.duration = 0.1 # type: ignore[attr-defined] + config.hook.pytest_runtest_logreport(report=rep) + + record = run_tests( + spec=ConfigSpec( + rootpath=tmp_path, extra_plugins=(LogReportsWithoutItems(),) + ), + capture_output=True, ) - output = pytester.runpytest() - output.stdout.no_fnmatch_line("*ZeroDivisionError*") - output.stdout.fnmatch_lines(["=* 2 passed in *="]) + record.stdout.no_fnmatch_line("*ZeroDivisionError*") + record.stdout.fnmatch_lines(["=* 2 passed in *="]) - def test_normal(self, many_tests_files, pytester: Pytester) -> None: - output = pytester.runpytest() - output.stdout.re_match_lines( + def test_normal(self, tmp_path: Path, many_tests_sources) -> None: + record = self._run(tmp_path, many_tests_sources) + record.stdout.re_match_lines( [ r"test_bar.py \.{10} \s+ \[ 50%\]", r"test_foo.py \.{5} \s+ \[ 75%\]", r"test_foobar.py \.{5} \s+ \[100%\]", ] ) + record.assert_outcomes(passed=20) - def test_colored_progress( - self, pytester: Pytester, monkeypatch, color_mapping - ) -> None: + def test_colored_progress(self, tmp_path: Path, monkeypatch, color_mapping) -> None: monkeypatch.setenv("PY_COLORS", "1") - pytester.makepyfile( - test_axfail=""" - import pytest - @pytest.mark.xfail - def test_axfail(): assert 0 - """, - test_bar=""" - import pytest - @pytest.mark.parametrize('i', range(10)) - def test_bar(i): pass - """, - test_foo=""" - import pytest - import warnings - @pytest.mark.parametrize('i', range(5)) - def test_foo(i): - warnings.warn(DeprecationWarning("collection")) - pass - """, - test_foobar=""" - import pytest - @pytest.mark.parametrize('i', range(5)) - def test_foobar(i): raise ValueError() - """, - ) - result = pytester.runpytest() - result.stdout.re_match_lines( + + @pytest.mark.xfail + def test_axfail(): + assert 0 + + @pytest.mark.parametrize("i", range(10)) + def test_bar(i): + pass + + @pytest.mark.parametrize("i", range(5)) + def test_foo(i): + import warnings + + warnings.warn(DeprecationWarning("collection")) + + @pytest.mark.parametrize("i", range(5)) + def test_foobar(i): + raise ValueError + + axfail_module = build_module("test_axfail", test_axfail) + sources = ( + axfail_module, + build_module("test_bar", test_bar), + build_module("test_foo", test_foo), + build_module("test_foobar", test_foobar), + ) + # The host suite turns warnings into errors; the point here is the + # yellow progress indicator a *recorded* warning produces. + inicfg: dict[str, object] = {"filterwarnings": ["always"]} + record = self._run(tmp_path, sources, inicfg=inicfg) + record.stdout.re_match_lines( color_mapping.format_for_rematch( [ r"test_axfail.py {yellow}x{reset}{green} \s+ \[ 4%\]{reset}", @@ -2507,10 +2540,11 @@ def test_foobar(i): raise ValueError() ] ) ) + record.assert_outcomes(passed=15, failed=5, xfailed=1, warnings=5) # Only xfail should have yellow progress indicator. - result = pytester.runpytest("test_axfail.py") - result.stdout.re_match_lines( + record = self._run(tmp_path, (axfail_module,)) + record.stdout.re_match_lines( color_mapping.format_for_rematch( [ r"test_axfail.py {yellow}x{reset}{yellow} \s+ \[100%\]{reset}", @@ -2518,23 +2552,25 @@ def test_foobar(i): raise ValueError() ] ) ) + record.assert_outcomes(xfailed=1) - def test_count(self, many_tests_files, pytester: Pytester) -> None: - pytester.makeini( - """ - [pytest] - console_output_style = count - """ + def test_count(self, tmp_path: Path, many_tests_sources) -> None: + record = self._run( + tmp_path, many_tests_sources, inicfg={"console_output_style": "count"} ) - output = pytester.runpytest() - output.stdout.re_match_lines( + record.stdout.re_match_lines( [ r"test_bar.py \.{10} \s+ \[10/20\]", r"test_foo.py \.{5} \s+ \[15/20\]", r"test_foobar.py \.{5} \s+ \[20/20\]", ] ) + record.assert_outcomes(passed=20) + # ensemble: `console_output_style=times` groups reports by + # ``report.location[0]``, which for a synthesized ensemble module is the + # *host* file - every item then looks like it belongs to one giant module + # and only the very last line gets a duration. See test_times_none_collected. def test_times(self, many_tests_files, pytester: Pytester) -> None: pytester.makeini( """ @@ -2551,6 +2587,7 @@ def test_times(self, many_tests_files, pytester: Pytester) -> None: ] ) + # ensemble: see test_times. def test_times_multiline( self, more_tests_files, monkeypatch, pytester: Pytester ) -> None: @@ -2571,6 +2608,8 @@ def test_times_multiline( consecutive=True, ) + # ensemble: asserts the NO_TESTS_COLLECTED exit code, which comes from + # `wrap_session`; an ensemble has no session wrapper and no exit code. def test_times_none_collected(self, pytester: Pytester) -> None: pytester.makeini( """ @@ -2581,32 +2620,31 @@ def test_times_none_collected(self, pytester: Pytester) -> None: output = pytester.runpytest() assert output.ret == ExitCode.NO_TESTS_COLLECTED - def test_verbose(self, many_tests_files, pytester: Pytester) -> None: - output = pytester.runpytest("-v") - output.stdout.re_match_lines( + def test_verbose(self, tmp_path: Path, many_tests_sources) -> None: + record = self._run(tmp_path, many_tests_sources, "-v") + record.stdout.re_match_lines( [ r"test_bar.py::test_bar\[0\] PASSED \s+ \[ 5%\]", r"test_foo.py::test_foo\[4\] PASSED \s+ \[ 75%\]", r"test_foobar.py::test_foobar\[4\] PASSED \s+ \[100%\]", ] ) + record.assert_outcomes(passed=20) - def test_verbose_count(self, many_tests_files, pytester: Pytester) -> None: - pytester.makeini( - """ - [pytest] - console_output_style = count - """ + def test_verbose_count(self, tmp_path: Path, many_tests_sources) -> None: + record = self._run( + tmp_path, many_tests_sources, "-v", inicfg={"console_output_style": "count"} ) - output = pytester.runpytest("-v") - output.stdout.re_match_lines( + record.stdout.re_match_lines( [ r"test_bar.py::test_bar\[0\] PASSED \s+ \[ 1/20\]", r"test_foo.py::test_foo\[4\] PASSED \s+ \[15/20\]", r"test_foobar.py::test_foobar\[4\] PASSED \s+ \[20/20\]", ] ) + record.assert_outcomes(passed=20) + # ensemble: see test_times. def test_verbose_times(self, many_tests_files, pytester: Pytester) -> None: pytester.makeini( """ @@ -2623,6 +2661,8 @@ def test_verbose_times(self, many_tests_files, pytester: Pytester) -> None: ] ) + # ensemble: the four xdist tests below run the tests through xdist + # workers, i.e. subprocesses. def test_xdist_normal( self, many_tests_files, pytester: Pytester, monkeypatch ) -> None: @@ -2695,6 +2735,10 @@ def test_xdist_times( ] ) + # ensemble: the point is that `--capture=no` suppresses the progress + # column. An ensemble reporter writes to its own private stream, which + # counts as captured no matter what `--capture` says, so the column stays + # on and the `no_fnmatch_line("*%]*")` half could never hold. def test_capture_no(self, many_tests_files, pytester: Pytester) -> None: output = pytester.runpytest("-s") output.stdout.re_match_lines( @@ -2704,6 +2748,10 @@ def test_capture_no(self, many_tests_files, pytester: Pytester) -> None: output = pytester.runpytest("--capture=no") output.stdout.no_fnmatch_line("*%]*") + # ensemble: the ini value under test only does anything when + # `--capture=no` would otherwise suppress the column; in an ensemble the + # column is on regardless, so this would assert nothing. See + # test_capture_no. def test_capture_no_progress_enabled( self, many_tests_files, pytester: Pytester ) -> None: @@ -2723,11 +2771,36 @@ def test_capture_no_progress_enabled( ) -# ensemble: every test below asserts on the progress column; see -# TestProgressOutputStyle. class TestProgressWithTeardown: """Ensure we show the correct percentages for tests that fail during teardown (#3088)""" + @pytest.fixture + def teardown_fixture_plugin(self) -> object: + """The ensemble equivalent of a conftest at the rootdir.""" + + class TeardownFixturePlugin: + @pytest.fixture + def fail_teardown(self): + yield + assert False + + return TeardownFixturePlugin() + + @pytest.fixture + def many_sources(self) -> tuple[ModuleType, ...]: + @pytest.mark.parametrize("i", range(5)) + def test_bar(fail_teardown, i): + pass + + @pytest.mark.parametrize("i", range(15)) + def test_foo(fail_teardown, i): + pass + + return ( + build_module("test_bar", test_bar), + build_module("test_foo", test_foo), + ) + @pytest.fixture def contest_with_teardown_fixture(self, pytester: Pytester) -> None: pytester.makeconftest( @@ -2758,47 +2831,75 @@ def test_foo(fail_teardown, i): """, ) - def test_teardown_simple( - self, pytester: Pytester, contest_with_teardown_fixture - ) -> None: - pytester.makepyfile( - """ - def test_foo(fail_teardown): - pass - """ + def test_teardown_simple(self, tmp_path: Path, teardown_fixture_plugin) -> None: + def test_foo(fail_teardown): + pass + + record = run_tests( + test_foo, + spec=ConfigSpec( + rootpath=tmp_path, extra_plugins=(teardown_fixture_plugin,) + ), + name="test_teardown_simple", + capture_output=True, ) - output = pytester.runpytest() - output.stdout.re_match_lines([r"test_teardown_simple.py \.E\s+\[100%\]"]) + record.stdout.re_match_lines([r"test_teardown_simple.py \.E\s+\[100%\]"]) + # `assertoutcome`-style categories: the teardown failure is an error. + record.assert_outcomes(passed=1, errors=1) def test_teardown_with_test_also_failing( - self, pytester: Pytester, contest_with_teardown_fixture + self, tmp_path: Path, teardown_fixture_plugin ) -> None: - pytester.makepyfile( - """ - def test_foo(fail_teardown): - assert 0 - """ + def test_foo(fail_teardown): + assert 0 + + record = run_tests( + test_foo, + spec=ConfigSpec( + rootpath=tmp_path, + args=("-rfE",), + extra_plugins=(teardown_fixture_plugin,), + ), + name="test_teardown_with_test_also_failing", + capture_output=True, ) - output = pytester.runpytest("-rfE") - output.stdout.re_match_lines( + record.stdout.re_match_lines( [ r"test_teardown_with_test_also_failing.py FE\s+\[100%\]", "FAILED test_teardown_with_test_also_failing.py::test_foo - assert 0", "ERROR test_teardown_with_test_also_failing.py::test_foo - assert False", ] ) + record.assert_outcomes(failed=1, errors=1) - def test_teardown_many(self, pytester: Pytester, many_files) -> None: - output = pytester.runpytest() - output.stdout.re_match_lines( + def test_teardown_many( + self, tmp_path: Path, many_sources, teardown_fixture_plugin + ) -> None: + record = run_tests( + *many_sources, + spec=ConfigSpec( + rootpath=tmp_path, extra_plugins=(teardown_fixture_plugin,) + ), + capture_output=True, + ) + record.stdout.re_match_lines( [r"test_bar.py (\.E){5}\s+\[ 25%\]", r"test_foo.py (\.E){15}\s+\[100%\]"] ) + record.assert_outcomes(passed=20, errors=20) def test_teardown_many_verbose( - self, pytester: Pytester, many_files, color_mapping + self, tmp_path: Path, many_sources, teardown_fixture_plugin, color_mapping ) -> None: - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + record = run_tests( + *many_sources, + spec=ConfigSpec( + rootpath=tmp_path, + args=("-v",), + extra_plugins=(teardown_fixture_plugin,), + ), + capture_output=True, + ) + record.stdout.fnmatch_lines( color_mapping.format_for_fnmatch( [ "test_bar.py::test_bar[0] PASSED * [ 5%]", @@ -2810,7 +2911,9 @@ def test_teardown_many_verbose( ] ) ) + record.assert_outcomes(passed=20, errors=20) + # ensemble: runs the tests through xdist workers, i.e. subprocesses. def test_xdist_normal(self, many_files, pytester: Pytester, monkeypatch) -> None: pytest.importorskip("xdist") monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) @@ -3329,10 +3432,6 @@ def test_no_warning_on_terminal_with_a_single_config_file( ) -# ensemble: every test below matches whole blocks of consecutive lines whose -# exact column layout is the point - the trailing ` [100%]` progress column -# (never rendered by an ensemble, see TestProgressOutputStyle) is part of that -# layout, and half of them assert on `--collect-only` rendering. class TestFineGrainedTestCase: DEFAULT_FILE_CONTENTS = """ import pytest @@ -3358,127 +3457,181 @@ def test_skip(): pass """ + @staticmethod + def _default_module(name: str) -> ModuleType: + """The in-memory equivalent of DEFAULT_FILE_CONTENTS.""" + + @pytest.mark.parametrize("i", range(4)) + def test_ok(i): + """ + some docstring + """ # noqa: D200, D403 + + def test_fail(): + assert False + + return build_module(name, test_ok, test_fail) + + @staticmethod + def _long_skip_module(name: str) -> ModuleType: + """The in-memory equivalent of LONG_SKIP_FILE_CONTENTS.""" + + @pytest.mark.skip( + "some long skip reason that will not fit on a single line with other content that goes" + " on and on and on and on and on" + ) + def test_skip(): + pass + + return build_module(name, test_skip) + + @staticmethod + def _run( + module: ModuleType, tmp_path: Path, verbosity: int, *args: str + ) -> RunRecord: + """Run *module* with ``verbosity_test_cases`` set, capturing output.""" + spec = ConfigSpec( + rootpath=tmp_path, + args=args, + inicfg={"verbosity_test_cases": str(verbosity)}, + ) + return run_tests(module, spec=spec, capture_output=True) + @pytest.mark.parametrize("verbosity", [1, 2]) - def test_execute_positive(self, verbosity, pytester: Pytester) -> None: + def test_execute_positive( + self, verbosity, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: # expected: one test case per line (with file name), word describing result - p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity) - result = pytester.runpytest(p) + # The column layout is the point, so the width must not be the host's. + monkeypatch.setenv("COLUMNS", "80") + name = "test_execute_positive.py" + record = self._run(self._default_module(name[:-3]), tmp_path, verbosity) - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 5 items", "", - f"{p.name}::test_ok[0] PASSED [ 20%]", - f"{p.name}::test_ok[1] PASSED [ 40%]", - f"{p.name}::test_ok[2] PASSED [ 60%]", - f"{p.name}::test_ok[3] PASSED [ 80%]", - f"{p.name}::test_fail FAILED [100%]", + f"{name}::test_ok[0] PASSED [ 20%]", + f"{name}::test_ok[1] PASSED [ 40%]", + f"{name}::test_ok[2] PASSED [ 60%]", + f"{name}::test_ok[3] PASSED [ 80%]", + f"{name}::test_fail FAILED [100%]", ], consecutive=True, ) + record.assert_outcomes(passed=4, failed=1) - def test_execute_0_global_1(self, pytester: Pytester) -> None: + def test_execute_0_global_1(self, tmp_path: Path, monkeypatch: MonkeyPatch) -> None: # expected: one file name per line, single character describing result - p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=0) - result = pytester.runpytest("-v", p) + monkeypatch.setenv("COLUMNS", "80") + name = "test_execute_0_global_1.py" + record = self._run(self._default_module(name[:-3]), tmp_path, 0, "-v") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collecting ... collected 5 items", "", - f"{p.name} ....F [100%]", + f"{name} ....F [100%]", ], consecutive=True, ) + record.assert_outcomes(passed=4, failed=1) @pytest.mark.parametrize("verbosity", [-1, -2]) - def test_execute_negative(self, verbosity, pytester: Pytester) -> None: + def test_execute_negative( + self, verbosity, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: # expected: single character describing result - p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity) - result = pytester.runpytest(p) + monkeypatch.setenv("COLUMNS", "80") + name = "test_execute_negative.py" + record = self._run(self._default_module(name[:-3]), tmp_path, verbosity) - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 5 items", "....F [100%]", ], consecutive=True, ) + record.assert_outcomes(passed=4, failed=1) - def test_execute_skipped_positive_2(self, pytester: Pytester) -> None: + def test_execute_skipped_positive_2( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: # expected: one test case per line (with file name), word describing result, full reason - p = TestFineGrainedTestCase._initialize_files( - pytester, - verbosity=2, - file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS, - ) - result = pytester.runpytest(p) + monkeypatch.setenv("COLUMNS", "80") + name = "test_execute_skipped_positive_2.py" + record = self._run(self._long_skip_module(name[:-3]), tmp_path, 2) - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 1 item", "", - f"{p.name}::test_skip SKIPPED (some long skip", + f"{name}::test_skip SKIPPED (some long skip", "reason that will not fit on a single line with other content that goes", "on and on and on and on and on) [100%]", ], consecutive=True, ) + record.assert_outcomes(skipped=1) - def test_execute_skipped_positive_1(self, pytester: Pytester) -> None: + def test_execute_skipped_positive_1( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: # expected: one test case per line (with file name), word describing result, reason truncated - p = TestFineGrainedTestCase._initialize_files( - pytester, - verbosity=1, - file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS, - ) - result = pytester.runpytest(p) + monkeypatch.setenv("COLUMNS", "80") + name = "test_execute_skipped_positive_1.py" + record = self._run(self._long_skip_module(name[:-3]), tmp_path, 1) - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 1 item", "", - f"{p.name}::test_skip SKIPPED (some long ski...) [100%]", + f"{name}::test_skip SKIPPED (some long ski...) [100%]", ], consecutive=True, ) + record.assert_outcomes(skipped=1) - def test_execute_skipped__0_global_1(self, pytester: Pytester) -> None: + def test_execute_skipped__0_global_1( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: # expected: one file name per line, single character describing result (no reason) - p = TestFineGrainedTestCase._initialize_files( - pytester, - verbosity=0, - file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS, - ) - result = pytester.runpytest("-v", p) + monkeypatch.setenv("COLUMNS", "80") + name = "test_execute_skipped__0_global_1.py" + record = self._run(self._long_skip_module(name[:-3]), tmp_path, 0, "-v") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collecting ... collected 1 item", "", - f"{p.name} s [100%]", + f"{name} s [100%]", ], consecutive=True, ) + record.assert_outcomes(skipped=1) @pytest.mark.parametrize("verbosity", [-1, -2]) - def test_execute_skipped_negative(self, verbosity, pytester: Pytester) -> None: + def test_execute_skipped_negative( + self, verbosity, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: # expected: single character describing result (no reason) - p = TestFineGrainedTestCase._initialize_files( - pytester, - verbosity=verbosity, - file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS, - ) - result = pytester.runpytest(p) + monkeypatch.setenv("COLUMNS", "80") + name = "test_execute_skipped_negative.py" + record = self._run(self._long_skip_module(name[:-3]), tmp_path, verbosity) - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 1 item", "s [100%]", ], consecutive=True, ) + record.assert_outcomes(skipped=1) + # ensemble: every test below asserts on `--collect-only` rendering, which + # is served from `pytest_cmdline_main`; an ensemble runs neither that hook + # nor the `` node the rendering starts from. @pytest.mark.parametrize("verbosity", [1, 2]) def test__collect_only_positive(self, verbosity, pytester: Pytester) -> None: p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity) From 0911d913a88d477c70fce6f91d1dd03e4cfd993a Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 09:28:07 +0200 Subject: [PATCH 18/30] bench: compare the two harnesses on real example files The existing benchmark contrasts pytester and _pytest.ensemble on generated sources. That measures the harness but not the usual shape of a pytester test, which is: take a script that already exists under testing/example_scripts, put it somewhere, and run it. This one runs the *same real files* through both routes - copy into the pytester tmpdir and run it there, versus module_from_path and run_tests where it lives - so the only difference is the harness. It also isolates the copy and the import, which turn out to be noise (0.1-0.2ms) next to the session bootstrap either side of them. The import-failure example is deliberately not a subject: the two harnesses would not be doing the same work. --- bench/ensemble_vs_pytester_examples.py | 159 +++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 bench/ensemble_vs_pytester_examples.py diff --git a/bench/ensemble_vs_pytester_examples.py b/bench/ensemble_vs_pytester_examples.py new file mode 100644 index 00000000000..e3728d53514 --- /dev/null +++ b/bench/ensemble_vs_pytester_examples.py @@ -0,0 +1,159 @@ +"""Compare pytester and ``_pytest.ensemble`` on the *same real files*. + +``bench/ensemble_vs_pytester.py`` compares the two harnesses on generated +sources, which measures the harness but says nothing about the usual shape +of a pytester test: take a script that already exists under +``testing/example_scripts``, put it somewhere, and run it. + +The two ways of doing that are: + +``copy`` + run + What ``pytester.copy_example`` does: put the file in the pytester + tmpdir, then run it with ``runpytest_inprocess``/``inline_run``/ + ``runpytest_subprocess``. The script is imported from the copy, so its + reported paths are the copy's. The copy is done with ``shutil.copy`` + rather than ``copy_example`` itself, which picks one fixed destination + and so cannot be called in a loop. +``module_from_path`` + ``module_from_path(path)`` imports the script where it lives - without + registering it in ``sys.modules`` or writing bytecode beside it - and + ``run_tests`` collects the resulting module. The script is never + copied, and its items report their real paths. + +Both run the same code, so the difference is the harness, not the work. + +Run with:: + + python bench/ensemble_vs_pytester_examples.py + pytest bench/ensemble_vs_pytester_examples.py -s # equivalent +""" + +from __future__ import annotations + +from collections.abc import Callable +import contextlib +import io +from pathlib import Path +import shutil +import time + +from _pytest.ensemble import module_from_path +from _pytest.ensemble import run_tests +from _pytest.pytester import Pytester + + +#: Real example scripts, relative to ``testing/example_scripts``. Examples +#: that deliberately fail at import time are not benchmark subjects - the +#: two harnesses would not be doing the same work. +EXAMPLES = [ + "fixtures/fill_fixtures/test_funcarg_basic.py", + "fixtures/fill_fixtures/test_funcarg_lookup_modulelevel.py", + "fixtures/fill_fixtures/test_funcarg_lookup_classlevel.py", + "fixtures/fill_fixtures/test_extend_fixture_module_class.py", + "unittest/test_setup_skip.py", + "unittest/test_setup_skip_class.py", + "unittest/test_setup_skip_module.py", +] + +ITERATIONS = {"subprocess": 3} +DEFAULT_ITERATIONS = 10 + +EXAMPLE_ROOT = Path(__file__).parent.parent / "testing" / "example_scripts" + + +def _files(path: Path) -> int: + """Files below *path*, ignoring bytecode caches.""" + return sum( + 1 + for p in path.rglob("*") + if p.is_file() and "__pycache__" not in p.parts and p.suffix != ".pyc" + ) + + +def _measure( + run: Callable[[int], None], iterations: int, watched: Path +) -> tuple[float, float]: + """Return (seconds, files created below *watched*) per iteration.""" + with contextlib.redirect_stdout(io.StringIO()): + run(0) # warm up + before = _files(watched) + start = time.perf_counter() + for i in range(1, iterations + 1): + run(i) + elapsed = time.perf_counter() - start + return elapsed / iterations, (_files(watched) - before) / iterations + + +def _arms(pytester: Pytester, rel: str) -> dict[str, Callable[[int], None]]: + source = EXAMPLE_ROOT / rel + + def copy(i: int) -> Path: + # Each iteration gets its own directory, which is what a real + # pytester test gets too. + target = pytester.path / f"run{i}" + target.mkdir(exist_ok=True) + dest = target / source.name + shutil.copy(source, dest) + return dest + + def subprocess_(i: int) -> None: + pytester.runpytest_subprocess(copy(i)) + + def inprocess(i: int) -> None: + pytester.runpytest_inprocess(copy(i)) + + def inline(i: int) -> None: + pytester.inline_run(copy(i)) + + def copy_only(i: int) -> None: + copy(i) + + def ensemble(i: int) -> None: + run_tests(module_from_path(source), rootpath=source.parent) + + def import_only(i: int) -> None: + module_from_path(source) + + return { + "subprocess": subprocess_, + "inprocess": inprocess, + "inline": inline, + "ensemble": ensemble, + "copy only": copy_only, + "import only": import_only, + } + + +def test_ensemble_vs_pytester_on_examples(pytester: Pytester) -> None: + """Not an assertion test - run with ``-s`` and read the table.""" + print() + totals: dict[str, float] = {} + for rel in EXAMPLES: + results = {} + for name, run in _arms(pytester, rel).items(): + iterations = ITERATIONS.get(name, DEFAULT_ITERATIONS) + results[name] = _measure(run, iterations, pytester.path) + totals[name] = totals.get(name, 0.0) + results[name][0] + + baseline = results["ensemble"][0] + print(f"\n{rel}") + print(f" {'arm':<12} {'per run':>10} {'files':>8} {'vs ensemble':>13}") + for name, (seconds, files) in results.items(): + ratio = f"{seconds / baseline:.1f}x" if baseline else "n/a" + print(f" {name:<12} {seconds * 1000:>8.2f}ms {files:>8.1f} {ratio:>13}") + + print(f"\n{'=' * 56}\nacross all {len(EXAMPLES)} examples") + base = totals["ensemble"] + print(f" {'arm':<12} {'total':>10} {'vs ensemble':>13}") + for name, seconds in totals.items(): + print(f" {name:<12} {seconds * 1000:>8.2f}ms {seconds / base:>12.1f}x") + print( + "\n(files counted below the pytester tmpdir, excluding __pycache__;\n" + " the ensemble arm writes nothing and never leaves the source tree)" + ) + + +if __name__ == "__main__": + import pytest + + raise SystemExit(pytest.main([__file__, "-s", "-q", "-p", "no:randomly"])) From 89047a2df37a9ac99dd6091f48dbeb2d92c17589 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 10:32:25 +0200 Subject: [PATCH 19/30] testing: port the non-invocation tests in acceptance_test.py 27 of 86 pytester tests move to _pytest.ensemble. The yield is deliberately low: this file is the one place where the *program* is the system under test, and roughly 40 of the 59 that stay assert something with no meaning inside a nested config - an exit code, a stderr line, what -p/--pyargs do to argv, which conftest is visible from which directory, what happens when a module fails to import. TestInvocationVariants (24 tests) should never move; its entire content is "invoked this way, pytest still works". What did move was mostly misplaced rather than slow: parametrize ids, fixture scope ordering, async rejection, StopIteration handling - unit-ish behaviour that had accreted into an acceptance file because pytester was the only tool available. Several gained real assertions on the way, with `assert result.ret == 0` becoming exact outcome counts, and test_with_deselected/test_with_not now asserting the deselection they were named for. TestDurations turns out not to be about duration measurement at all - with mock_timing the clock is fake - so it is a terminal rendering test, and ports onto capture_output=True keeping its lines verbatim. test_fixture_mock_integration runs the real example script via module_from_path rather than inlining it, so the example stays in use. Wall clock 7.82s -> 7.32s: 30% of the tests bought 6% of the time. The value here is placement, not speed. --- testing/acceptance_test.py | 706 +++++++++++++++++++++---------------- 1 file changed, 410 insertions(+), 296 deletions(-) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index f941cbe1921..da58c8d88e4 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -6,18 +6,29 @@ import importlib.metadata import os from pathlib import Path +import re import subprocess import sys import types +import unittest import setuptools from _pytest.config import ExitCode +from _pytest.ensemble import build_module +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import module_from_path +from _pytest.ensemble import run_tests +from _pytest.ensemble import RunRecord from _pytest.pathlib import symlink_or_skip from _pytest.pytester import Pytester import pytest +#: The example scripts, run as themselves rather than copied somewhere first. +EXAMPLES = Path(__file__).parent / "example_scripts" + + def prepend_pythonpath(*dirs) -> str: cur = os.getenv("PYTHONPATH") if cur: @@ -26,6 +37,8 @@ def prepend_pythonpath(*dirs) -> str: class TestGeneralUsage: + # ensemble: a conftest raising UsageError, reported on stderr with the + # usage exit code - none of the three has an ensemble equivalent. def test_config_error(self, pytester: Pytester) -> None: pytester.copy_example("conftest_usageerror/conftest.py") result = pytester.runpytest(pytester.path) @@ -33,12 +46,15 @@ def test_config_error(self, pytester: Pytester) -> None: result.stderr.fnmatch_lines(["*ERROR: hello"]) result.stdout.fnmatch_lines(["*pytest_unconfigure_called"]) + # ensemble: importing a conftest file. def test_root_conftest_syntax_error(self, pytester: Pytester) -> None: pytester.makepyfile(conftest="raise SyntaxError\n") result = pytester.runpytest() result.stderr.fnmatch_lines(["*raise SyntaxError*"]) assert result.ret != 0 + # ensemble: INTERNALERROR rendering belongs to wrap_session, and the + # traceback names the conftest file it came from. def test_early_hook_error_issue38_1(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -59,6 +75,7 @@ def pytest_sessionstart(): ["*INTERNALERROR*def pytest_sessionstart():*", "*INTERNALERROR*0 / 0*"] ) + # ensemble: as above, for a failure during configure. def test_early_hook_configure_error_issue38(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -73,11 +90,14 @@ def pytest_configure(): ["*INTERNALERROR*File*conftest.py*line 2*", "*0 / 0*"] ) + # ensemble: path arguments are resolved by the invocation; a spec's args + # are never coerced to paths and collection is preset. def test_file_not_found(self, pytester: Pytester) -> None: result = pytester.runpytest("asd") assert result.ret != 0 result.stderr.fnmatch_lines(["ERROR: file or directory not found: asd"]) + # ensemble: as above, plus conftest hooks and the usage exit code. def test_file_not_found_unconfigure_issue143(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -92,6 +112,7 @@ def pytest_unconfigure(): result.stderr.fnmatch_lines(["ERROR: file or directory not found: asd"]) result.stdout.fnmatch_lines(["*---configure", "*---unconfigure"]) + # ensemble: -p plugin loading from the command line, during preparse. def test_config_preparse_plugin_option(self, pytester: Pytester) -> None: pytester.makepyfile( pytest_xyz=""" @@ -109,6 +130,8 @@ def test_option(pytestconfig): assert result.ret == 0 result.stdout.fnmatch_lines(["*1 passed*"]) + # ensemble: setuptools entry point autoloading, which an ensemble config + # deliberately never does. @pytest.mark.parametrize("load_cov_early", [True, False]) def test_early_load_setuptools_name( self, pytester: Pytester, monkeypatch, load_cov_early @@ -155,6 +178,8 @@ def my_dists(): else: assert loaded == ["myplugin1", "myplugin2", "mycov"] + # ensemble: assertion rewriting happens at import, under an --import-mode; + # ensemble sources are never imported by pytest. @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"]) def test_assertion_rewrite(self, pytester: Pytester, import_mode) -> None: p = pytester.makepyfile( @@ -168,6 +193,7 @@ def test_this(): result.stdout.fnmatch_lines(["> assert x", "E assert 0"]) assert result.ret == 1 + # ensemble: an error raised while importing the test module. def test_nested_import_error(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ @@ -186,6 +212,8 @@ def test_this(): ) assert result.ret == 2 + # ensemble: what the invocation does with arguments that match no + # collector, down to the usage exit code. def test_not_collectable_arguments(self, pytester: Pytester) -> None: p1 = pytester.makepyfile("") p2 = pytester.makefile(".pyc", "123") @@ -199,6 +227,8 @@ def test_not_collectable_arguments(self, pytester: Pytester) -> None: ] ) + # ensemble: a conftest failing to import, and --help, which is served + # from pytest_cmdline_main. @pytest.mark.filterwarnings("default") def test_better_reporting_on_conftest_load_failure( self, pytester: Pytester @@ -230,6 +260,7 @@ def foo(): "E ModuleNotFoundError: No module named 'qwerty'", ] + # ensemble: pytest_collect_file only fires while walking a directory tree. def test_early_skip(self, pytester: Pytester) -> None: pytester.mkdir("xyz") pytester.makeconftest( @@ -243,12 +274,15 @@ def pytest_collect_file(): assert result.ret == ExitCode.NO_TESTS_COLLECTED result.stdout.fnmatch_lines(["*1 skip*"]) + # ensemble: a custom file collector from an example tree, listed by + # --collect-only. def test_issue88_initial_file_multinodes(self, pytester: Pytester) -> None: pytester.copy_example("issue88_initial_file_multinodes") p = pytester.makepyfile("def test_hello(): pass") result = pytester.runpytest(p, "--collect-only") result.stdout.fnmatch_lines(["*MyFile*test_issue88*", "*Module*test_issue88*"]) + # ensemble: global capturing around conftest import; capture does not nest. def test_issue93_initialnode_importing_capturing(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -262,6 +296,7 @@ def test_issue93_initialnode_importing_capturing(self, pytester: Pytester) -> No result.stdout.no_fnmatch_line("*should not be seen*") assert "stderr42" not in result.stderr.str() + # ensemble: as above - a conftest printing before it fails to import. def test_conftest_printing_shows_if_error(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -273,6 +308,8 @@ def test_conftest_printing_shows_if_error(self, pytester: Pytester) -> None: assert result.ret != 0 assert "should be seen" in result.stdout.str() + # ensemble: conftest visibility is per-directory; an ensemble plugin object + # stands for a rootdir conftest and nothing below it. def test_issue109_sibling_conftests_not_loaded(self, pytester: Pytester) -> None: sub1 = pytester.mkdir("sub1") sub2 = pytester.mkdir("sub2") @@ -287,6 +324,7 @@ def test_issue109_sibling_conftests_not_loaded(self, pytester: Pytester) -> None result = pytester.runpytest(sub1) assert result.ret == ExitCode.USAGE_ERROR + # ensemble: pytest_ignore_collect during a directory walk. def test_directory_skipped(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -300,6 +338,8 @@ def pytest_ignore_collect(): assert result.ret == ExitCode.NO_TESTS_COLLECTED result.stdout.fnmatch_lines(["*1 skipped*"]) + # ensemble: a custom File collector contributed by pytest_collect_file, + # then addressed by nodeid on the command line. def test_multiple_items_per_collector_byid(self, pytester: Pytester) -> None: c = pytester.makeconftest( """ @@ -319,24 +359,28 @@ def pytest_collect_file(file_path, parent): assert result.ret == 0 result.stdout.fnmatch_lines(["*1 pass*"]) - def test_skip_on_generated_funcarg_id(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - import pytest - def pytest_generate_tests(metafunc): - metafunc.parametrize('x', [3], ids=['hello-123']) - def pytest_runtest_setup(item): - print(item.keywords) - if 'hello-123' in item.keywords: + def test_skip_on_generated_funcarg_id(self, tmp_path: Path) -> None: + class ConftestPlugin: + def pytest_generate_tests(self, metafunc): + metafunc.parametrize("x", [3], ids=["hello-123"]) + + def pytest_runtest_setup(self, item): + if "hello-123" in item.keywords: pytest.skip("hello") assert 0 - """ + + def test_func(x): + pass + + record = run_tests( + test_func, + spec=ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)), ) - p = pytester.makepyfile("""def test_func(x): pass""") - res = pytester.runpytest(p) - assert res.ret == 0 - res.stdout.fnmatch_lines(["*1 skipped*"]) + record.assert_outcomes(skipped=1) + assert record["test_func[hello-123]"].skipped + # ensemble: selecting an item by nodeid argument - the point of the test; + # ensemble collection is preset, not resolved from args. def test_direct_addressing_selects(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ @@ -350,45 +394,31 @@ def test_func(i): assert res.ret == 0 res.stdout.fnmatch_lines(["*1 passed*"]) - def test_direct_addressing_selects_duplicates(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest + def test_direct_addressing_selects_duplicates(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("a", [1, 2, 10, 11, 2, 1, 12, 11]) + def test_func(a): + pass - @pytest.mark.parametrize("a", [1, 2, 10, 11, 2, 1, 12, 11]) - def test_func(a): - pass - """ - ) - result = pytester.runpytest(p) - result.assert_outcomes(failed=0, passed=8) + record = run_tests(test_func, rootpath=tmp_path) + record.assert_outcomes(failed=0, passed=8) - def test_direct_addressing_selects_duplicates_1(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest + def test_direct_addressing_selects_duplicates_1(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("a", [1, 2, 10, 11, 2, 1, 12, 1_1, 2_1]) + def test_func(a): + pass - @pytest.mark.parametrize("a", [1, 2, 10, 11, 2, 1, 12, 1_1,2_1]) - def test_func(a): - pass - """ - ) - result = pytester.runpytest(p) - result.assert_outcomes(failed=0, passed=9) + record = run_tests(test_func, rootpath=tmp_path) + record.assert_outcomes(failed=0, passed=9) - def test_direct_addressing_selects_duplicates_2(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import pytest + def test_direct_addressing_selects_duplicates_2(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("a", ["a", "b", "c", "a", "a1"]) + def test_func(a): + pass - @pytest.mark.parametrize("a", ["a","b","c","a","a1"]) - def test_func(a): - pass - """ - ) - result = pytester.runpytest(p) - result.assert_outcomes(failed=0, passed=5) + record = run_tests(test_func, rootpath=tmp_path) + record.assert_outcomes(failed=0, passed=5) + # ensemble: as above, for a nodeid that matches nothing. def test_direct_addressing_notfound(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ @@ -407,6 +437,7 @@ def test_docstring_on_hookspec(self) -> None: if name.startswith("pytest_"): assert value.__doc__, f"no docstring for {name}" + # ensemble: the internal-error exit code and its stderr rendering. def test_initialization_error_issue49(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -419,6 +450,7 @@ def pytest_configure(): result.stderr.fnmatch_lines(["INTERNAL*pytest_configure*", "INTERNAL*x*"]) assert "sessionstarttime" not in result.stderr.str() + # ensemble: a syntax error at import, reached through a nodeid argument. @pytest.mark.parametrize("lookfor", ["test_fun.py::test_a"]) def test_issue134_report_error_when_collecting_member( self, pytester: Pytester, lookfor @@ -435,6 +467,8 @@ def test_a(): result.stderr.fnmatch_lines(["*ERROR*"]) assert result.ret == 4 # usage error only if item not found + # ensemble: initial arguments that all fail, and the exit code the + # conftest asserts on in pytest_sessionfinish. def test_report_all_failed_collections_initargs(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -451,6 +485,7 @@ def pytest_sessionfinish(exitstatus): result.stdout.fnmatch_lines(["pytest_sessionfinish_called"]) assert result.ret == ExitCode.USAGE_ERROR + # ensemble: pytest's import hooks, exercised by importing the test module. def test_namespace_import_doesnt_confuse_import_hook( self, pytester: Pytester ) -> None: @@ -476,6 +511,7 @@ def test_whatever(): res = pytester.runpytest(p.name) assert res.ret == 0 + # ensemble: argument parsing errors, reported on stderr. def test_unknown_option(self, pytester: Pytester) -> None: result = pytester.runpytest("--qwlkej") result.stderr.fnmatch_lines( @@ -484,27 +520,25 @@ def test_unknown_option(self, pytester: Pytester) -> None: """ ) - def test_getsourcelines_error_issue553( - self, pytester: Pytester, monkeypatch - ) -> None: - monkeypatch.setattr("inspect.getsourcelines", None) - p = pytester.makepyfile( - """ - def raise_error(obj): - raise OSError('source code not available') + def test_getsourcelines_error_issue553(self, tmp_path: Path, monkeypatch) -> None: + def raise_error(obj): + raise OSError("source code not available") - import inspect - inspect.getsourcelines = raise_error + # The original patched inspect from inside the test module; the patch + # is process-global either way, so it is applied from here instead. + monkeypatch.setattr("inspect.getsourcelines", raise_error) - def test_foo(invalid_fixture): - pass - """ - ) - res = pytester.runpytest(p) - res.stdout.fnmatch_lines( + def test_foo(invalid_fixture): + pass + + record = run_tests(test_foo, rootpath=tmp_path, capture_output=True) + # A fixture missing at setup is an error, not a failure. + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( ["*source code not available*", "E*fixture 'invalid_fixture' not found"] ) + # ensemble: the pytest.main() entry point and its plugins= argument. def test_plugins_given_as_strings( self, pytester: Pytester, monkeypatch, _sys_snapshot ) -> None: @@ -520,33 +554,26 @@ def test_plugins_given_as_strings( monkeypatch.setitem(sys.modules, "myplugin", mod) assert pytest.main(args=[str(pytester.path)], plugins=["myplugin"]) == 0 - def test_parametrized_with_bytes_regex(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - import re - import pytest - @pytest.mark.parametrize('r', [re.compile(b'foo')]) - def test_stuff(r): - pass - """ - ) - res = pytester.runpytest(p) - res.stdout.fnmatch_lines(["*1 passed*"]) + def test_parametrized_with_bytes_regex(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("r", [re.compile(b"foo")]) + def test_stuff(r): + pass + + record = run_tests(test_stuff, rootpath=tmp_path) + record.assert_outcomes(passed=1) - def test_parametrized_with_null_bytes(self, pytester: Pytester) -> None: + def test_parametrized_with_null_bytes(self, tmp_path: Path) -> None: """Test parametrization with values that contain null bytes and unicode characters (#2644, #2957)""" - p = pytester.makepyfile( - """\ - import pytest - @pytest.mark.parametrize("data", [b"\\x00", "\\x00", 'ação']) - def test_foo(data): - assert data - """ - ) - res = pytester.runpytest(p) - res.assert_outcomes(passed=3) + @pytest.mark.parametrize("data", [b"\x00", "\x00", "ação"]) + def test_foo(data): + assert data + + record = run_tests(test_foo, rootpath=tmp_path) + record.assert_outcomes(passed=3) + # ensemble: @argsfile expansion, and the nodeid arguments it expands to. + # # Warning ignore because of: # https://github.com/python/cpython/issues/85308 # Can be removed once Python<3.12 support is dropped. @@ -575,6 +602,12 @@ def test_func(self, a): class TestInvocationVariants: + # ensemble: nothing in this class is portable, and that is the point of it + # - every test here is about pytest *as it is invoked*: as a subprocess, + # via python -m, through pytest.main(), or with --pyargs resolving import + # names against sys.path. An ensemble is the opposite end of that: no + # process, no argv, no import system. Individual reasons are noted only + # where a test looks portable at a glance. def test_earlyinit(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ @@ -809,6 +842,7 @@ def test_cmdline_python_legacy_namespace_package( ["*test_world.py::test_other*PASSED*", "*1 passed*"] ) + # ensemble: --doctest-modules collects from the file, addressed by nodeid. def test_invoke_test_and_doctestmodules(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ @@ -884,21 +918,24 @@ def test_cmdline_python_package_not_exists(self, pytester: Pytester) -> None: result.stderr.fnmatch_lines(["ERROR*module*or*package*not*found*"]) @pytest.mark.xfail(reason="decide: feature or bug") - def test_noclass_discovery_if_not_testcase(self, pytester: Pytester) -> None: - testpath = pytester.makepyfile( - """ - import unittest - class TestHello(object): - def test_hello(self): - assert self.attr + def test_noclass_discovery_if_not_testcase(self, tmp_path: Path) -> None: + class TestHello: + # Declared, never defined: the mixin only works once something + # provides it, which is the whole question the test asks. The + # source used to be a string, so mypy never saw it before. + attr: int - class RealTest(unittest.TestCase, TestHello): - attr = 42 - """ - ) - reprec = pytester.inline_run(testpath) - reprec.assertoutcome(passed=1) + def test_hello(self): + assert self.attr + + class RealTest(unittest.TestCase, TestHello): + attr = 42 + record = run_tests(TestHello, RealTest, rootpath=tmp_path) + record.assert_outcomes(passed=1) + + # ensemble: a .txt file collected by the doctest plugin, twice, addressed + # by nodeid. def test_doctest_id(self, pytester: Pytester) -> None: pytester.makefile( ".txt", @@ -951,54 +988,77 @@ def test_3(): timing.sleep(0.020) """ - def test_calls(self, pytester: Pytester, mock_timing) -> None: - pytester.makepyfile(self.source) - result = pytester.runpytest_inprocess("--durations=10") - assert result.ret == 0 + @staticmethod + def build_source() -> types.ModuleType: + """:attr:`source`, as real functions in a module of their own. + + Deliberately named: the module name is what the duration lines + report, so the nodeid assertions below stay meaningful. + """ + from _pytest import timing - result.stdout.fnmatch_lines_random( + def test_something(): + pass + + def test_2(): + timing.sleep(0.010) + + def test_1(): + timing.sleep(0.002) + + def test_3(): + timing.sleep(0.020) + + return build_module("test_durations", test_something, test_2, test_1, test_3) + + @staticmethod + def run(tmp_path: Path, *args: str) -> RunRecord: + return run_tests( + TestDurations.build_source(), + spec=ConfigSpec(rootpath=tmp_path, args=args), + capture_output=True, + ) + + def test_calls(self, tmp_path: Path, mock_timing) -> None: + record = self.run(tmp_path, "--durations=10") + record.assert_outcomes(passed=4) + + record.stdout.fnmatch_lines_random( ["*durations*", "*call*test_3*", "*call*test_2*"] ) - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( ["(8 durations < 0.005s hidden. Use -vv to show these durations.)"] ) - def test_calls_show_2(self, pytester: Pytester, mock_timing) -> None: - pytester.makepyfile(self.source) - result = pytester.runpytest_inprocess("--durations=2") - assert result.ret == 0 + def test_calls_show_2(self, tmp_path: Path, mock_timing) -> None: + record = self.run(tmp_path, "--durations=2") + record.assert_outcomes(passed=4) - lines = result.stdout.get_lines_after("*slowest*durations*") + lines = record.stdout.get_lines_after("*slowest*durations*") assert "4 passed" in lines[2] - def test_calls_showall(self, pytester: Pytester, mock_timing) -> None: - pytester.makepyfile(self.source) - result = pytester.runpytest_inprocess("--durations=0") - assert result.ret == 0 - TestDurations.check_tests_in_output(result.stdout.lines, 2, 3) + def test_calls_showall(self, tmp_path: Path, mock_timing) -> None: + record = self.run(tmp_path, "--durations=0") + record.assert_outcomes(passed=4) + TestDurations.check_tests_in_output(record.stdout.lines, 2, 3) - def test_calls_showall_verbose(self, pytester: Pytester, mock_timing) -> None: - pytester.makepyfile(self.source) - result = pytester.runpytest_inprocess("--durations=0", "-vv") - assert result.ret == 0 - TestDurations.check_tests_in_output(result.stdout.lines, 1, 2, 3) + def test_calls_showall_verbose(self, tmp_path: Path, mock_timing) -> None: + record = self.run(tmp_path, "--durations=0", "-vv") + record.assert_outcomes(passed=4) + TestDurations.check_tests_in_output(record.stdout.lines, 1, 2, 3) - def test_calls_showall_durationsmin(self, pytester: Pytester, mock_timing) -> None: - pytester.makepyfile(self.source) - result = pytester.runpytest_inprocess("--durations=0", "--durations-min=0.015") - assert result.ret == 0 - TestDurations.check_tests_in_output(result.stdout.lines, 3) + def test_calls_showall_durationsmin(self, tmp_path: Path, mock_timing) -> None: + record = self.run(tmp_path, "--durations=0", "--durations-min=0.015") + record.assert_outcomes(passed=4) + TestDurations.check_tests_in_output(record.stdout.lines, 3) def test_calls_showall_durationsmin_verbose( - self, pytester: Pytester, mock_timing + self, tmp_path: Path, mock_timing ) -> None: - pytester.makepyfile(self.source) - result = pytester.runpytest_inprocess( - "--durations=0", "--durations-min=0.015", "-vv" - ) - assert result.ret == 0 - TestDurations.check_tests_in_output(result.stdout.lines, 3) + record = self.run(tmp_path, "--durations=0", "--durations-min=0.015", "-vv") + record.assert_outcomes(passed=4) + TestDurations.check_tests_in_output(record.stdout.lines, 3) @staticmethod def check_tests_in_output( @@ -1014,13 +1074,14 @@ def check_tests_in_output( } assert found_test_numbers == set(expected_test_numbers) - def test_with_deselected(self, pytester: Pytester, mock_timing) -> None: - pytester.makepyfile(self.source) - result = pytester.runpytest_inprocess("--durations=2", "-k test_3") - assert result.ret == 0 + def test_with_deselected(self, tmp_path: Path, mock_timing) -> None: + record = self.run(tmp_path, "--durations=2", "-k", "test_3") + record.assert_outcomes(passed=1, deselected=3) - result.stdout.fnmatch_lines(["*durations*", "*call*test_3*"]) + record.stdout.fnmatch_lines(["*durations*", "*call*test_3*"]) + # ensemble: the collection error is a module that fails to import, and the + # "Interrupted: 1 error during collection" line is wrap_session's. def test_with_failing_collection(self, pytester: Pytester, mock_timing) -> None: pytester.makepyfile(self.source) pytester.makepyfile(test_collecterror="""xyz""") @@ -1032,10 +1093,10 @@ def test_with_failing_collection(self, pytester: Pytester, mock_timing) -> None: # output result.stdout.no_fnmatch_line("*duration*") - def test_with_not(self, pytester: Pytester, mock_timing) -> None: - pytester.makepyfile(self.source) - result = pytester.runpytest_inprocess("-k not 1") - assert result.ret == 0 + def test_with_not(self, tmp_path: Path, mock_timing) -> None: + record = self.run(tmp_path, "-k", "not 1") + # The original only asserted the exit code; "not 1" deselects test_1. + record.assert_outcomes(passed=3, deselected=1) class TestDurationsWithFixture: @@ -1051,12 +1112,24 @@ def test_1(setup_fixt): timing.sleep(5) """ - def test_setup_function(self, pytester: Pytester, mock_timing) -> None: - pytester.makepyfile(self.source) - result = pytester.runpytest_inprocess("--durations=10") - assert result.ret == 0 + def test_setup_function(self, tmp_path: Path, mock_timing) -> None: + from _pytest import timing - result.stdout.fnmatch_lines_random( + @pytest.fixture + def setup_fixt(): + timing.sleep(2) + + def test_1(setup_fixt): + timing.sleep(5) + + record = run_tests( + build_module("test_durations_fixture", setup_fixt, test_1), + spec=ConfigSpec(rootpath=tmp_path, args=("--durations=10",)), + capture_output=True, + ) + record.assert_outcomes(passed=1) + + record.stdout.fnmatch_lines_random( """ *durations* 5.00s call *test_1* @@ -1065,6 +1138,7 @@ def test_setup_function(self, pytester: Pytester, mock_timing) -> None: ) +# ensemble: a zipapp, run as its own process. def test_zipimport_hook(pytester: Pytester) -> None: """Test package loader is being used correctly (see #1837).""" zipapp = pytest.importorskip("zipapp") @@ -1088,6 +1162,7 @@ def main(): result.stdout.no_fnmatch_line("*INTERNALERROR>*") +# ensemble: pytest_plugins in a conftest, naming a module to import. def test_import_plugin_unicode_name(pytester: Pytester) -> None: pytester.makepyfile(myplugin="") pytester.makepyfile("def test(): pass") @@ -1096,6 +1171,7 @@ def test_import_plugin_unicode_name(pytester: Pytester) -> None: assert r.ret == 0 +# ensemble: as above - the subject is what a conftest's pytest_plugins may be. def test_pytest_plugins_as_module(pytester: Pytester) -> None: """Do not raise an error if pytest_plugins attribute is a module (#3899)""" pytester.makepyfile( @@ -1110,34 +1186,36 @@ def test_pytest_plugins_as_module(pytester: Pytester) -> None: result.stdout.fnmatch_lines(["* 1 passed in *"]) -def test_deferred_hook_checking(pytester: Pytester) -> None: +def test_deferred_hook_checking(tmp_path: Path) -> None: """Check hooks as late as possible (#1821).""" - pytester.syspathinsert() - pytester.makepyfile( - **{ - "plugin.py": """ - class Hooks(object): - def pytest_my_hook(self, config): - pass - def pytest_configure(config): + class Hooks: + def pytest_my_hook(self, config): + pass + + class Plugin: + def pytest_configure(self, config): config.pluginmanager.add_hookspecs(Hooks) - """, - "conftest.py": """ - pytest_plugins = ['plugin'] - def pytest_my_hook(config): - return 40 - """, - "test_foo.py": """ - def test(request): - assert request.config.hook.pytest_my_hook(config=request.config) == [40] - """, - } + + class ConftestPlugin: + # Registered while pytest_my_hook has no hookspec yet - the spec only + # arrives from the plugin above, at configure time. That the impl is + # still wired up to it is the whole point. + def pytest_my_hook(self, config): + return 40 + + def test(request): + assert request.config.hook.pytest_my_hook(config=request.config) == [40] + + record = run_tests( + test, + spec=ConfigSpec(rootpath=tmp_path, extra_plugins=(Plugin(), ConftestPlugin())), ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["* 1 passed *"]) + record.assert_outcomes(passed=1) +# ensemble: deliberately a subprocess - an in-process run keeps the objects +# alive through the host's HookRecorder, which is exactly what it must not do. def test_fixture_values_leak(pytester: Pytester) -> None: """Ensure that fixture objects are properly destroyed by the garbage collector at the end of their expected life-times (#2981). @@ -1186,31 +1264,30 @@ def test2(): result.stdout.fnmatch_lines(["* 2 passed *"]) -def test_fixture_order_respects_scope(pytester: Pytester) -> None: +def test_fixture_order_respects_scope(tmp_path: Path) -> None: """Ensure that fixtures are created according to scope order (#2405).""" - pytester.makepyfile( - """ - import pytest + # A module global in an ensemble source would be *this* file's global, + # so the state the fixtures share is a closure cell instead. + data: dict[str, bool] = {} - data = {} + @pytest.fixture(scope="module") + def clean_data(): + data.clear() - @pytest.fixture(scope='module') - def clean_data(): - data.clear() + @pytest.fixture(autouse=True) + def add_data(): + data.update(value=True) - @pytest.fixture(autouse=True) - def add_data(): - data.update(value=True) + @pytest.mark.usefixtures("clean_data") + def test_value(): + assert data.get("value") - @pytest.mark.usefixtures('clean_data') - def test_value(): - assert data.get('value') - """ - ) - result = pytester.runpytest() - assert result.ret == 0 + record = run_tests(clean_data, add_data, test_value, rootpath=tmp_path) + # The original only asserted the exit code. + record.assert_outcomes(passed=1) +# ensemble: as above - deliberately a subprocess, for the same reason. def test_frame_leak_on_failing_test(pytester: Pytester) -> None: """Pytest would leak garbage referencing the frames of tests that failed that could never be reclaimed (#2798). @@ -1244,36 +1321,43 @@ def test2(): result.stdout.fnmatch_lines(["*1 failed, 1 passed in*"]) -def test_fixture_mock_integration(pytester: Pytester) -> None: +def test_fixture_mock_integration() -> None: """Test that decorators applied to fixture are left working (#3774)""" - p = pytester.copy_example("acceptance/fixture_mock_integration.py") - result = pytester.runpytest(p) - result.stdout.fnmatch_lines(["*1 passed*"]) + example = EXAMPLES / "acceptance/fixture_mock_integration.py" + record = run_tests(module_from_path(example), rootpath=example.parent) + record.assert_outcomes(passed=1) +# ensemble: the exit code is the whole test. def test_usage_error_code(pytester: Pytester) -> None: result = pytester.runpytest("-unknown-option-") assert result.ret == ExitCode.USAGE_ERROR -def test_error_on_async_function(pytester: Pytester) -> None: +def test_error_on_async_function(tmp_path: Path) -> None: # In the below we .close() the coroutine only to avoid # "RuntimeWarning: coroutine 'test_2' was never awaited" # which messes with other tests. - pytester.makepyfile( - test_async=""" - async def test_1(): - pass - async def test_2(): - pass - def test_3(): - coro = test_2() - coro.close() - return coro - """ + async def test_1(): + pass + + async def test_2(): + pass + + def test_3(): + coro = test_2() + coro.close() + return coro + + record = run_tests( + test_1, + test_2, + test_3, + rootpath=tmp_path, + name="test_async", + capture_output=True, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*async def functions are not natively supported*", "*test_async.py::test_1*", @@ -1281,22 +1365,28 @@ def test_3(): "*test_async.py::test_3*", ] ) - result.assert_outcomes(failed=3) + record.assert_outcomes(failed=3) -def test_error_on_async_gen_function(pytester: Pytester) -> None: - pytester.makepyfile( - test_async=""" - async def test_1(): - yield - async def test_2(): - yield - def test_3(): - return test_2() - """ +def test_error_on_async_gen_function(tmp_path: Path) -> None: + async def test_1(): + yield + + async def test_2(): + yield + + def test_3(): + return test_2() + + record = run_tests( + test_1, + test_2, + test_3, + rootpath=tmp_path, + name="test_async", + capture_output=True, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "*async def functions are not natively supported*", "*test_async.py::test_1*", @@ -1304,29 +1394,29 @@ def test_3(): "*test_async.py::test_3*", ] ) - result.assert_outcomes(failed=3) + record.assert_outcomes(failed=3) -def test_error_on_sync_test_async_fixture(pytester: Pytester) -> None: - pytester.makepyfile( - test_sync=""" - import pytest +def test_error_on_sync_test_async_fixture(tmp_path: Path) -> None: + @pytest.fixture + async def async_fixture(): ... - @pytest.fixture - async def async_fixture(): - ... + def test_foo(async_fixture): + # suppress unawaited coroutine warning + try: + async_fixture.send(None) + except StopIteration: + pass - def test_foo(async_fixture): - # suppress unawaited coroutine warning - try: - async_fixture.send(None) - except StopIteration: - pass - """ + record = run_tests( + async_fixture, + test_foo, + rootpath=tmp_path, + name="test_sync", + capture_output=True, ) - result = pytester.runpytest() - result.assert_outcomes(errors=1) - result.stdout.fnmatch_lines( + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( [ "'test_foo' requested an async fixture 'async_fixture', with no plugin or hook that handled it. " "This is an error, as pytest does not natively support it." @@ -1334,23 +1424,24 @@ def test_foo(async_fixture): ) -def test_error_on_sync_test_async_fixture_gen(pytester: Pytester) -> None: - pytester.makepyfile( - test_sync=""" - import pytest +def test_error_on_sync_test_async_fixture_gen(tmp_path: Path) -> None: + @pytest.fixture + async def async_fixture(): + yield - @pytest.fixture - async def async_fixture(): - yield + def test_foo(async_fixture): + # async gens don't emit unawaited-coroutine + ... - def test_foo(async_fixture): - # async gens don't emit unawaited-coroutine - ... - """ + record = run_tests( + async_fixture, + test_foo, + rootpath=tmp_path, + name="test_sync", + capture_output=True, ) - result = pytester.runpytest() - result.assert_outcomes(errors=1) - result.stdout.fnmatch_lines( + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( [ "'test_foo' requested an async fixture 'async_fixture', with no plugin or hook that handled it. " "This is an error, as pytest does not natively support it." @@ -1358,27 +1449,27 @@ def test_foo(async_fixture): ) -def test_error_on_sync_test_async_autouse_fixture(pytester: Pytester) -> None: - pytester.makepyfile( - test_sync=""" - import pytest +def test_error_on_sync_test_async_autouse_fixture(tmp_path: Path) -> None: + @pytest.fixture(autouse=True) + async def async_fixture(): ... - @pytest.fixture(autouse=True) - async def async_fixture(): - ... + # We explicitly request the fixture to be able to + # suppress the RuntimeWarning for unawaited coroutine. + def test_foo(async_fixture): + try: + async_fixture.send(None) + except StopIteration: + pass - # We explicitly request the fixture to be able to - # suppress the RuntimeWarning for unawaited coroutine. - def test_foo(async_fixture): - try: - async_fixture.send(None) - except StopIteration: - pass - """ + record = run_tests( + async_fixture, + test_foo, + rootpath=tmp_path, + name="test_sync", + capture_output=True, ) - result = pytester.runpytest() - result.assert_outcomes(errors=1) - result.stdout.fnmatch_lines( + record.assert_outcomes(errors=1) + record.stdout.fnmatch_lines( [ "'test_foo' requested an async fixture 'async_fixture' with autouse=True, " "with no plugin or hook that handled it. " @@ -1387,6 +1478,8 @@ def test_foo(async_fixture): ) +# ensemble: register_assert_rewrite from a conftest, verified by the rewritten +# explanation of an assertion in an imported module. def test_pdb_can_be_rewritten(pytester: Pytester) -> None: pytester.makepyfile( **{ @@ -1422,6 +1515,7 @@ def test(): assert result.ret == 1 +# ensemble: global capturing (--capture=tee-sys) does not nest. def test_tee_stdio_captures_and_live_prints(pytester: Pytester) -> None: testpath = pytester.makepyfile( """ @@ -1449,6 +1543,7 @@ def test_simple(): assert "@this is stderr@\n" in fullXml +# ensemble: a real process whose stdout is closed under it. @pytest.mark.skipif( sys.platform == "win32", reason="Windows raises `OSError: [Errno 22] Invalid argument` instead", @@ -1470,18 +1565,22 @@ def test_no_brokenpipeerror_message(pytester: Pytester) -> None: popen.stderr.close() -@pytest.mark.filterwarnings("default") -def test_function_return_non_none_warning(pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_stuff(): - return "something" - """ +def test_function_return_non_none_warning(tmp_path: Path) -> None: + def test_stuff(): + return "something" + + # The host runs with filterwarnings=error and those filters are inherited, + # so the ensemble states its own. + record = run_tests( + test_stuff, + spec=ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["always"]}), + capture_output=True, ) - res = pytester.runpytest() - res.stdout.fnmatch_lines(["*Did you mean to use `assert` instead of `return`?*"]) + record.assert_outcomes(passed=1, warnings=1) + record.stdout.fnmatch_lines(["*Did you mean to use `assert` instead of `return`?*"]) +# ensemble: --import-mode importlib and the module identity it yields. def test_doctest_and_normal_imports_with_importlib(pytester: Pytester) -> None: """ Regression test for #10811: previously import_path with ImportMode.importlib would @@ -1517,6 +1616,7 @@ def test_boo(self): result.stdout.fnmatch_lines("*1 passed*") +# ensemble: installs a package and runs the pytest console script against it. @pytest.mark.skip(reason="Test is not isolated") def test_issue_9765(pytester: Pytester) -> None: """Reproducer for issue #9765 on Windows @@ -1582,6 +1682,9 @@ def my_fixture(self, request): ) from exc +# ensemble: an ensemble runs without the terminal plugin by default, so the +# same source here would assert nothing about `-p no:terminal` blocking it in a +# real invocation - which, with the exit code, is the whole smoke test. def test_no_terminal_plugin(pytester: Pytester) -> None: """Smoke test to ensure pytest can execute without the terminal plugin (#9422).""" pytester.makepyfile("def test(): assert 1 == 2") @@ -1589,6 +1692,8 @@ def test_no_terminal_plugin(pytester: Pytester) -> None: assert result.ret == ExitCode.TESTS_FAILED +# ensemble: StopIteration raised while *importing* the module, and the +# "Interrupted" line wrap_session prints for it. def test_stop_iteration_from_collect(pytester: Pytester) -> None: pytester.makepyfile(test_it="raise StopIteration('hello')") result = pytester.runpytest() @@ -1604,27 +1709,36 @@ def test_stop_iteration_from_collect(pytester: Pytester) -> None: ) -def test_stop_iteration_runtest_protocol(pytester: Pytester) -> None: - pytester.makepyfile( - test_it=""" - import pytest - @pytest.fixture - def fail_setup(): - raise StopIteration(1) - def test_fail_setup(fail_setup): - pass - def test_fail_teardown(request): - def stop_iteration(): - raise StopIteration(2) - request.addfinalizer(stop_iteration) - def test_fail_call(): - raise StopIteration(3) - """ +def test_stop_iteration_runtest_protocol(tmp_path: Path) -> None: + @pytest.fixture + def fail_setup(): + raise StopIteration(1) + + def test_fail_setup(fail_setup): + pass + + def test_fail_teardown(request): + def stop_iteration(): + raise StopIteration(2) + + request.addfinalizer(stop_iteration) + + def test_fail_call(): + raise StopIteration(3) + + # ensemble: the exit code has no equivalent; the outcome counts below are + # what ExitCode.TESTS_FAILED stood for. + record = run_tests( + fail_setup, + test_fail_setup, + test_fail_teardown, + test_fail_call, + rootpath=tmp_path, + name="test_it", + capture_output=True, ) - result = pytester.runpytest() - assert result.ret == ExitCode.TESTS_FAILED - result.assert_outcomes(failed=1, passed=1, errors=2) - result.stdout.fnmatch_lines( + record.assert_outcomes(failed=1, passed=1, errors=2) + record.stdout.fnmatch_lines( [ "=* short test summary info =*", "FAILED test_it.py::test_fail_call - StopIteration: 3", From d6caabdf1ec7f83439088741fefbe70ed7fa88ba Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 10:36:23 +0200 Subject: [PATCH 20/30] testing: port test_cacheprovider.py to _pytest.ensemble 44 of 50 pytester tests. cacheprovider works in an ensemble with nothing special: with_plugins("cacheprovider") against a real tmp_path rootdir gives a real config.cache, and two ensembles sharing a rootpath share the cache - which is exactly the shape --lf/--ff/--nf tests need. The one real design decision is which sources exist on disk. --lf's file-level skipping and --nf's mtime ordering both need path.exists(), so those use a written file plus module_from_path; everything else uses a synthesized module, where the last-failed wrapper ignores non-existent paths and so behaves like a file named on the command line - which is what preserves the original "N deselected" assertions. Both helpers are documented at module level. Assertions that had to change are commented at each site: --co trees lose their lines, one summary reads "- AssertionError" rather than "- assert 0" because ensemble sources are not rewritten, and NO_TESTS_COLLECTED becomes an empty outcomes() dict. A print from pytest_sessionfinish became a direct assertion on the plugin object, which is stronger. Seven of the riskiest new assertions were mutation-tested. 6 stay on pytester: real packages and nested trees (3), modules that raise at import (2), and --help (1). Wall clock 4.51s -> 3.84s. Modest, because every test here was already in-process - see the commit adding the autoload benchmark for why that is not the interesting number. --- testing/test_cacheprovider.py | 1648 +++++++++++++++++++-------------- 1 file changed, 967 insertions(+), 681 deletions(-) diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index 7ac3f38ab64..7ba2cafb26c 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -1,3 +1,4 @@ +# mypy: allow-untyped-defs from __future__ import annotations from collections.abc import Generator @@ -7,10 +8,18 @@ import os from pathlib import Path import shutil +import types from typing import Any from _pytest.compat import assert_never -from _pytest.config import ExitCode +from _pytest.ensemble import build_module +from _pytest.ensemble import collect_tests +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import configured +from _pytest.ensemble import Ensemble +from _pytest.ensemble import module_from_path +from _pytest.ensemble import run_tests +from _pytest.ensemble.collection import ensemble_collection from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester from _pytest.tmpdir import TempPathFactory @@ -20,58 +29,92 @@ pytest_plugins = ("pytester",) +def cache_spec(rootpath: Path, *args: str, **inicfg: object) -> ConfigSpec: + """A nested config with the cacheprovider plugin loaded. + + ``cacheprovider`` is not in the ensemble default plugin set, but loading + it needs nothing else: the cache lives under the config's rootdir, and an + ensemble's rootdir is a real directory. Two ensembles built from the same + *rootpath* therefore share one cache, which is what makes ``--lf``/``--ff`` + expressible as "run the same spec twice". + """ + return ConfigSpec(rootpath=rootpath, args=args, inicfg=inicfg).with_plugins( + "cacheprovider" + ) + + +def write_source(rootpath: Path, relpath: str, source: str) -> Path: + """Write a test module to disk under *rootpath* and return its path. + + Sources that must exist as files are the ones whose *path* is part of what + is being tested: ``--lf`` skips collecting whole files by path, and ``--nf`` + orders them by mtime. Both need a path that exists, which a synthesized + in-memory module deliberately does not have (see :func:`in_memory`). + """ + path = rootpath.joinpath(relpath) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + return path + + +def in_memory(name: str, *members: object) -> types.ModuleType: + """A synthesized module whose path under the rootdir does not exist. + + The last-failed collection wrapper only filters/skips *files that exist*, + so an in-memory module always reaches ``pytest_collection_modifyitems`` + with all of its items - the same way a file named on the command line does + in a real run, because it is then an initial path. This is what keeps the + "N deselected" assertions of the original tests meaningful. + """ + return build_module(name, *members) + + class TestNewAPI: - def test_config_cache_mkdir(self, pytester: Pytester) -> None: - pytester.makeini("[pytest]") - config = pytester.parseconfigure() - assert config.cache is not None - with pytest.raises(ValueError): - config.cache.mkdir("key/name") + def test_config_cache_mkdir(self, tmp_path: Path) -> None: + with configured(cache_spec(tmp_path)) as config: + assert config.cache is not None + with pytest.raises(ValueError): + config.cache.mkdir("key/name") - p = config.cache.mkdir("name") - assert p.is_dir() + p = config.cache.mkdir("name") + assert p.is_dir() - def test_cache_dir_permissions(self, pytester: Pytester) -> None: + def test_cache_dir_permissions(self, tmp_path: Path) -> None: """The .pytest_cache directory should have world-readable permissions (depending on umask). Regression test for #12308. """ - pytester.makeini("[pytest]") - config = pytester.parseconfigure() - assert config.cache is not None - p = config.cache.mkdir("name") - assert p.is_dir() - # Instead of messing with umask, make sure .pytest_cache has the same - # permissions as the default that `mkdir` gives `p`. - assert (p.parent.stat().st_mode & 0o777) == (p.stat().st_mode & 0o777) - - def test_config_cache_dataerror(self, pytester: Pytester) -> None: - pytester.makeini("[pytest]") - config = pytester.parseconfigure() - assert config.cache is not None - cache = config.cache - with pytest.raises(TypeError): - cache.set("key/name", cache) - config.cache.set("key/name", 0) - config.cache._getvaluepath("key/name").write_bytes(b"123invalid") - val = config.cache.get("key/name", -2) - assert val == -2 + with configured(cache_spec(tmp_path)) as config: + assert config.cache is not None + p = config.cache.mkdir("name") + assert p.is_dir() + # Instead of messing with umask, make sure .pytest_cache has the same + # permissions as the default that `mkdir` gives `p`. + assert (p.parent.stat().st_mode & 0o777) == (p.stat().st_mode & 0o777) + + def test_config_cache_dataerror(self, tmp_path: Path) -> None: + with configured(cache_spec(tmp_path)) as config: + assert config.cache is not None + cache = config.cache + with pytest.raises(TypeError): + cache.set("key/name", cache) + config.cache.set("key/name", 0) + config.cache._getvaluepath("key/name").write_bytes(b"123invalid") + val = config.cache.get("key/name", -2) + assert val == -2 @pytest.mark.filterwarnings("ignore:could not create cache path") - def test_cache_writefail_cachefile_silent(self, pytester: Pytester) -> None: - pytester.makeini("[pytest]") - pytester.path.joinpath(".pytest_cache").write_text( - "gone wrong", encoding="utf-8" - ) - config = pytester.parseconfigure() - cache = config.cache - assert cache is not None - cache.set("test/broken", []) + def test_cache_writefail_cachefile_silent(self, tmp_path: Path) -> None: + tmp_path.joinpath(".pytest_cache").write_text("gone wrong", encoding="utf-8") + with configured(cache_spec(tmp_path)) as config: + cache = config.cache + assert cache is not None + cache.set("test/broken", []) @pytest.fixture - def unwritable_cache_dir(self, pytester: Pytester) -> Generator[Path]: - cache_dir = pytester.path.joinpath(".pytest_cache") + def unwritable_cache_dir(self, tmp_path: Path) -> Generator[Path]: + cache_dir = tmp_path.joinpath(".pytest_cache") cache_dir.mkdir() mode = cache_dir.stat().st_mode cache_dir.chmod(0) @@ -85,28 +128,32 @@ def unwritable_cache_dir(self, pytester: Pytester) -> Generator[Path]: "ignore:could not create cache path:pytest.PytestWarning" ) def test_cache_writefail_permissions( - self, unwritable_cache_dir: Path, pytester: Pytester + self, unwritable_cache_dir: Path, tmp_path: Path ) -> None: - pytester.makeini("[pytest]") - config = pytester.parseconfigure() - cache = config.cache - assert cache is not None - cache.set("test/broken", []) + with configured(cache_spec(tmp_path)) as config: + cache = config.cache + assert cache is not None + cache.set("test/broken", []) - @pytest.mark.filterwarnings("default") def test_cache_failure_warns( self, - pytester: Pytester, - monkeypatch: MonkeyPatch, + tmp_path: Path, unwritable_cache_dir: Path, ) -> None: - monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + # The original disabled plugin autoloading so that no other plugin + # could add warnings; an ensemble never autoloads anything. The host's + # ``filterwarnings = error`` is what the ensemble's own ini overrides. + def test_error(): + raise Exception - pytester.makepyfile("def test_error(): raise Exception") - result = pytester.runpytest() - assert result.ret == 1 + record = run_tests( + test_error, + spec=cache_spec(tmp_path, filterwarnings=["always"]), + capture_output=True, + ) # warnings from nodeids and lastfailed - result.stdout.fnmatch_lines( + record.assert_outcomes(failed=1, warnings=2) + record.stdout.fnmatch_lines( [ # Validate location/stacklevel of warning from cacheprovider. "*= warnings summary =*", @@ -118,137 +165,150 @@ def test_cache_failure_warns( ] ) - def test_config_cache(self, pytester: Pytester) -> None: - pytester.makeconftest( - """ - def pytest_configure(config): + def test_config_cache(self, tmp_path: Path) -> None: + class ConftestPlugin: + def pytest_configure(self, config): # see that we get cache information early on assert hasattr(config, "cache") - """ - ) - pytester.makepyfile( - """ - def test_session(pytestconfig): - assert hasattr(pytestconfig, "cache") - """ + + def test_session(pytestconfig): + assert hasattr(pytestconfig, "cache") + + record = run_tests( + test_session, + spec=cache_spec(tmp_path).replace(extra_plugins=(ConftestPlugin(),)), + capture_output=True, ) - result = pytester.runpytest() - assert result.ret == 0 - result.stdout.fnmatch_lines(["*1 passed*"]) + record.assert_outcomes(passed=1) + record.stdout.fnmatch_lines(["*1 passed*"]) - def test_cachefuncarg(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - def test_cachefuncarg(cache): - val = cache.get("some/thing", None) - assert val is None - cache.set("some/thing", [1]) - with pytest.raises(TypeError): - cache.get("some/thing") - val = cache.get("some/thing", []) - assert val == [1] - """ + def test_cachefuncarg(self, tmp_path: Path) -> None: + def test_cachefuncarg(cache): + val = cache.get("some/thing", None) + assert val is None + cache.set("some/thing", [1]) + with pytest.raises(TypeError): + cache.get("some/thing") + val = cache.get("some/thing", []) + assert val == [1] + + record = run_tests( + test_cachefuncarg, spec=cache_spec(tmp_path), capture_output=True ) - result = pytester.runpytest() - assert result.ret == 0 - result.stdout.fnmatch_lines(["*1 passed*"]) + record.assert_outcomes(passed=1) + record.stdout.fnmatch_lines(["*1 passed*"]) - def test_custom_rel_cache_dir(self, pytester: Pytester) -> None: + def test_custom_rel_cache_dir(self, tmp_path: Path) -> None: rel_cache_dir = os.path.join("custom_cache_dir", "subdir") - pytester.makeini( - f""" - [pytest] - cache_dir = {rel_cache_dir} - """ + + def test_error(): + assert False + + run_tests( + test_error, + spec=cache_spec(tmp_path, cache_dir=rel_cache_dir), + name="test_errored", ) - pytester.makepyfile(test_errored="def test_error():\n assert False") - pytester.runpytest() - assert pytester.path.joinpath(rel_cache_dir).is_dir() + assert tmp_path.joinpath(rel_cache_dir).is_dir() def test_custom_abs_cache_dir( - self, pytester: Pytester, tmp_path_factory: TempPathFactory + self, tmp_path: Path, tmp_path_factory: TempPathFactory ) -> None: tmp = tmp_path_factory.mktemp("tmp") abs_cache_dir = tmp / "custom_cache_dir" - pytester.makeini( - f""" - [pytest] - cache_dir = {abs_cache_dir} - """ + + def test_error(): + assert False + + run_tests( + test_error, + spec=cache_spec(tmp_path, cache_dir=str(abs_cache_dir)), + name="test_errored", ) - pytester.makepyfile(test_errored="def test_error():\n assert False") - pytester.runpytest() assert abs_cache_dir.is_dir() def test_custom_cache_dir_with_env_var( - self, pytester: Pytester, monkeypatch: MonkeyPatch + self, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: monkeypatch.setenv("env_var", "custom_cache_dir") - pytester.makeini( - """ - [pytest] - cache_dir = {cache_dir} - """.format(cache_dir="$env_var") + + def test_error(): + assert False + + run_tests( + test_error, + spec=cache_spec(tmp_path, cache_dir="$env_var"), + name="test_errored", ) - pytester.makepyfile(test_errored="def test_error():\n assert False") - pytester.runpytest() - assert pytester.path.joinpath("custom_cache_dir").is_dir() + assert tmp_path.joinpath("custom_cache_dir").is_dir() @pytest.mark.parametrize("env", ((), ("TOX_ENV_DIR", "mydir/tox-env"))) def test_cache_reportheader( - env: Sequence[str], pytester: Pytester, monkeypatch: MonkeyPatch + env: Sequence[str], tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: - pytester.makepyfile("""def test_foo(): pass""") + def test_foo(): + pass + if env: monkeypatch.setenv(*env) expected = os.path.join(env[1], ".pytest_cache") else: monkeypatch.delenv("TOX_ENV_DIR", raising=False) expected = ".pytest_cache" - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines([f"cachedir: {expected}"]) + record = run_tests(test_foo, spec=cache_spec(tmp_path, "-v"), capture_output=True) + record.stdout.fnmatch_lines([f"cachedir: {expected}"]) def test_cache_reportheader_external_abspath( - pytester: Pytester, tmp_path_factory: TempPathFactory + tmp_path: Path, tmp_path_factory: TempPathFactory ) -> None: external_cache = tmp_path_factory.mktemp( "test_cache_reportheader_external_abspath_abs" ) - pytester.makepyfile("def test_hello(): pass") - pytester.makeini( - f""" - [pytest] - cache_dir = {external_cache} - """ + def test_hello(): + pass + + record = run_tests( + test_hello, + spec=cache_spec(tmp_path, "-v", cache_dir=str(external_cache)), + capture_output=True, ) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines([f"cachedir: {external_cache}"]) + record.stdout.fnmatch_lines([f"cachedir: {external_cache}"]) -def test_cache_show(pytester: Pytester) -> None: - result = pytester.runpytest("--cache-show") - assert result.ret == 0 - result.stdout.fnmatch_lines(["*cache is empty*"]) - pytester.makeconftest( - """ - def pytest_configure(config): - config.cache.set("my/name", [1,2,3]) +def test_cache_show(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + # ensemble: ``--cache-show`` is served from ``pytest_cmdline_main``, which + # an ensemble never runs - so the reporting function is driven directly, + # against a config that has the option set. It writes to a TerminalWriter + # of its own, i.e. to the host's stdout, hence capsys rather than + # capture_output. + from _pytest.cacheprovider import cacheshow + + with Ensemble(spec=cache_spec(tmp_path, "--cache-show")) as ensemble: + assert cacheshow(ensemble.config, ensemble.session) == 0 + result = capsys.readouterr().out + assert "cache is empty" in result + + class ConftestPlugin: + def pytest_configure(self, config): + config.cache.set("my/name", [1, 2, 3]) config.cache.set("my/hello", "world") - config.cache.set("other/some", {1:2}) + config.cache.set("other/some", {1: 2}) dp = config.cache.mkdir("mydb") dp.joinpath("hello").touch() dp.joinpath("world").touch() - """ + + record = run_tests( + spec=cache_spec(tmp_path).replace(extra_plugins=(ConftestPlugin(),)) ) - result = pytester.runpytest() - assert result.ret == 5 # no tests executed + assert record.outcomes() == {} # no tests executed - result = pytester.runpytest("--cache-show") - result.stdout.fnmatch_lines( + with Ensemble(spec=cache_spec(tmp_path, "--cache-show")) as ensemble: + assert cacheshow(ensemble.config, ensemble.session) == 0 + matcher = pytest.LineMatcher(capsys.readouterr().out.splitlines()) + matcher.fnmatch_lines( [ "*cachedir:*", "*- cache values for '[*]' -*", @@ -262,10 +322,12 @@ def pytest_configure(config): "*mydb/world*length 0*", ] ) - assert result.ret == 0 - result = pytester.runpytest("--cache-show", "*/hello") - result.stdout.fnmatch_lines( + with Ensemble(spec=cache_spec(tmp_path, "--cache-show", "*/hello")) as ensemble: + assert cacheshow(ensemble.config, ensemble.session) == 0 + stdout = capsys.readouterr().out + matcher = pytest.LineMatcher(stdout.splitlines()) + matcher.fnmatch_lines( [ "*cachedir:*", "*- cache values for '[*]/hello' -*", @@ -275,72 +337,97 @@ def pytest_configure(config): "d/mydb/hello*length 0*", ] ) - stdout = result.stdout.str() assert "other/some" not in stdout assert "d/mydb/world" not in stdout - assert result.ret == 0 class TestLastFailed: - def test_lastfailed_usecase( - self, pytester: Pytester, monkeypatch: MonkeyPatch - ) -> None: - monkeypatch.setattr("sys.dont_write_bytecode", True) - p = pytester.makepyfile( - """ - def test_1(): assert 0 - def test_2(): assert 0 - def test_3(): assert 1 - """ - ) - result = pytester.runpytest(str(p)) - result.stdout.fnmatch_lines(["*2 failed*"]) - p = pytester.makepyfile( - """ - def test_1(): assert 1 - def test_2(): assert 1 - def test_3(): assert 0 - """ + def test_lastfailed_usecase(self, tmp_path: Path) -> None: + def failing() -> types.ModuleType: + def test_1(): + assert 0 + + def test_2(): + assert 0 + + def test_3(): + assert 1 + + return in_memory("test_lastfailed_usecase", test_1, test_2, test_3) + + def fixed() -> types.ModuleType: + def test_1(): + assert 1 + + def test_2(): + assert 1 + + def test_3(): + assert 0 + + return in_memory("test_lastfailed_usecase", test_1, test_2, test_3) + + record = run_tests(failing(), spec=cache_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines(["*2 failed*"]) + record = run_tests( + fixed(), spec=cache_spec(tmp_path, "--lf"), capture_output=True ) - result = pytester.runpytest(str(p), "--lf") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 3 items / 1 deselected / 2 selected", "run-last-failure: rerun previous 2 failures", "*= 2 passed, 1 deselected in *", ] ) - result = pytester.runpytest(str(p), "--lf") - result.stdout.fnmatch_lines( + record = run_tests( + fixed(), spec=cache_spec(tmp_path, "--lf"), capture_output=True + ) + record.stdout.fnmatch_lines( [ "collected 3 items", "run-last-failure: no previously failed tests, not deselecting items.", "*1 failed*2 passed*", ] ) - pytester.path.joinpath(".pytest_cache", ".git").mkdir(parents=True) - result = pytester.runpytest(str(p), "--lf", "--cache-clear") - result.stdout.fnmatch_lines(["*1 failed*2 passed*"]) - assert pytester.path.joinpath(".pytest_cache", "README.md").is_file() - assert pytester.path.joinpath(".pytest_cache", ".git").is_dir() + tmp_path.joinpath(".pytest_cache", ".git").mkdir(parents=True) + record = run_tests( + fixed(), + spec=cache_spec(tmp_path, "--lf", "--cache-clear"), + capture_output=True, + ) + record.stdout.fnmatch_lines(["*1 failed*2 passed*"]) + assert tmp_path.joinpath(".pytest_cache", "README.md").is_file() + assert tmp_path.joinpath(".pytest_cache", ".git").is_dir() # Run this again to make sure clear-cache is robust - if os.path.isdir(".pytest_cache"): - shutil.rmtree(".pytest_cache") - result = pytester.runpytest("--lf", "--cache-clear") - result.stdout.fnmatch_lines(["*1 failed*2 passed*"]) + shutil.rmtree(tmp_path / ".pytest_cache") + record = run_tests( + fixed(), + spec=cache_spec(tmp_path, "--lf", "--cache-clear"), + capture_output=True, + ) + record.stdout.fnmatch_lines(["*1 failed*2 passed*"]) - def test_failedfirst_order(self, pytester: Pytester) -> None: - pytester.makepyfile( - test_a="def test_always_passes(): pass", - test_b="def test_always_fails(): assert 0", + def test_failedfirst_order(self, tmp_path: Path) -> None: + def test_always_passes(): + pass + + def test_always_fails(): + assert 0 + + test_a = in_memory("test_a", test_always_passes) + test_b = in_memory("test_b", test_always_fails) + + record = run_tests( + test_a, test_b, spec=cache_spec(tmp_path), capture_output=True ) - result = pytester.runpytest() # Test order will be collection order; alphabetical - result.stdout.fnmatch_lines(["test_a.py*", "test_b.py*"]) - result = pytester.runpytest("--ff") + record.stdout.fnmatch_lines(["test_a.py*", "test_b.py*"]) + record = run_tests( + test_a, test_b, spec=cache_spec(tmp_path, "--ff"), capture_output=True + ) # Test order will be failing tests first - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 2 items", "run-last-failure: rerun previous 1 failure first", @@ -348,44 +435,77 @@ def test_failedfirst_order(self, pytester: Pytester) -> None: "test_a.py*", ] ) + assert [ + report.nodeid for report in record.reports if report.when == "call" + ] == [ + "test_b.py::test_always_fails", + "test_a.py::test_always_passes", + ] - def test_lastfailed_failedfirst_order(self, pytester: Pytester) -> None: - pytester.makepyfile( - test_a="def test_always_passes(): assert 1", - test_b="def test_always_fails(): assert 0", + def test_lastfailed_failedfirst_order(self, tmp_path: Path) -> None: + def test_always_passes(): + assert 1 + + def test_always_fails(): + assert 0 + + test_a = in_memory("test_a", test_always_passes) + test_b = in_memory("test_b", test_always_fails) + + record = run_tests( + test_a, test_b, spec=cache_spec(tmp_path), capture_output=True ) - result = pytester.runpytest() # Test order will be collection order; alphabetical - result.stdout.fnmatch_lines(["test_a.py*", "test_b.py*"]) - result = pytester.runpytest("--lf", "--ff") + record.stdout.fnmatch_lines(["test_a.py*", "test_b.py*"]) + record = run_tests( + test_a, + test_b, + spec=cache_spec(tmp_path, "--lf", "--ff"), + capture_output=True, + ) # Test order will be failing tests first - result.stdout.fnmatch_lines(["test_b.py*"]) - result.stdout.no_fnmatch_line("*test_a.py*") + record.stdout.fnmatch_lines(["test_b.py*"]) + record.stdout.no_fnmatch_line("*test_a.py*") - def test_lastfailed_difference_invocations( - self, pytester: Pytester, monkeypatch: MonkeyPatch - ) -> None: - monkeypatch.setattr("sys.dont_write_bytecode", True) - pytester.makepyfile( - test_a=""" - def test_a1(): assert 0 - def test_a2(): assert 1 - """, - test_b="def test_b1(): assert 0", + def test_lastfailed_difference_invocations(self, tmp_path: Path) -> None: + def test_a1(): + assert 0 + + def test_a2(): + assert 1 + + test_a = in_memory("test_a", test_a1, test_a2) + + def failing_b() -> types.ModuleType: + def test_b1(): + assert 0 + + return in_memory("test_b", test_b1) + + def fixed_b() -> types.ModuleType: + def test_b1(): + assert 1 + + return in_memory("test_b", test_b1) + + record = run_tests( + test_a, failing_b(), spec=cache_spec(tmp_path), capture_output=True + ) + record.stdout.fnmatch_lines(["*2 failed*"]) + # Selecting a subset is expressed by handing the ensemble fewer sources. + record = run_tests( + failing_b(), spec=cache_spec(tmp_path, "--lf"), capture_output=True ) - p = pytester.path.joinpath("test_a.py") - p2 = pytester.path.joinpath("test_b.py") + record.stdout.fnmatch_lines(["*1 failed*"]) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*2 failed*"]) - result = pytester.runpytest("--lf", p2) - result.stdout.fnmatch_lines(["*1 failed*"]) - - pytester.makepyfile(test_b="def test_b1(): assert 1") - result = pytester.runpytest("--lf", p2) - result.stdout.fnmatch_lines(["*1 passed*"]) - result = pytester.runpytest("--lf", p) - result.stdout.fnmatch_lines( + record = run_tests( + fixed_b(), spec=cache_spec(tmp_path, "--lf"), capture_output=True + ) + record.stdout.fnmatch_lines(["*1 passed*"]) + record = run_tests( + test_a, spec=cache_spec(tmp_path, "--lf"), capture_output=True + ) + record.stdout.fnmatch_lines( [ "collected 2 items / 1 deselected / 1 selected", "run-last-failure: rerun previous 1 failure", @@ -393,53 +513,61 @@ def test_a2(): assert 1 ] ) - def test_lastfailed_usecase_splice( - self, pytester: Pytester, monkeypatch: MonkeyPatch - ) -> None: - monkeypatch.setattr("sys.dont_write_bytecode", True) - pytester.makepyfile( - "def test_1(): assert 0", test_something="def test_2(): assert 0" - ) - p2 = pytester.path.joinpath("test_something.py") - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*2 failed*"]) - result = pytester.runpytest("--lf", p2) - result.stdout.fnmatch_lines(["*1 failed*"]) - result = pytester.runpytest("--lf") - result.stdout.fnmatch_lines(["*2 failed*"]) + def test_lastfailed_usecase_splice(self, tmp_path: Path) -> None: + def test_1(): + assert 0 - def test_lastfailed_xpass(self, pytester: Pytester) -> None: - pytester.inline_runsource( - """ - import pytest - @pytest.mark.xfail - def test_hello(): - assert 1 - """ + def test_2(): + assert 0 + + main = in_memory("test_lastfailed_usecase_splice", test_1) + other = in_memory("test_something", test_2) + + record = run_tests(main, other, spec=cache_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines(["*2 failed*"]) + record = run_tests( + other, spec=cache_spec(tmp_path, "--lf"), capture_output=True ) - config = pytester.parseconfigure() - assert config.cache is not None - lastfailed = config.cache.get("cache/lastfailed", -1) - assert lastfailed == -1 + record.stdout.fnmatch_lines(["*1 failed*"]) + record = run_tests( + main, other, spec=cache_spec(tmp_path, "--lf"), capture_output=True + ) + record.stdout.fnmatch_lines(["*2 failed*"]) + + def test_lastfailed_xpass(self, tmp_path: Path) -> None: + @pytest.mark.xfail + def test_hello(): + assert 1 - def test_non_serializable_parametrize(self, pytester: Pytester) -> None: + run_tests(test_hello, spec=cache_spec(tmp_path)).assert_outcomes(xpassed=1) + with configured(cache_spec(tmp_path)) as config: + assert config.cache is not None + lastfailed = config.cache.get("cache/lastfailed", -1) + assert lastfailed == -1 + + def test_non_serializable_parametrize(self, tmp_path: Path) -> None: """Test that failed parametrized tests with unmarshable parameters don't break pytest-cache. """ - pytester.makepyfile( - r""" - import pytest - - @pytest.mark.parametrize('val', [ - b'\xac\x10\x02G', - ]) - def test_fail(val): - assert False - """ + + @pytest.mark.parametrize( + "val", + [ + b"\xac\x10\x02G", + ], ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 failed in*"]) + def test_fail(val): + assert False + + record = run_tests(test_fail, spec=cache_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines(["*1 failed in*"]) + # The unmarshable parameter must not have kept the cache from being + # written at all. + assert (tmp_path / ".pytest_cache/v/cache/lastfailed").is_file() + # ensemble: the "package" parametrization needs a real ``__init__.py`` + # package, and an ensemble's collection tree has no Package (or Dir) nodes + # at all - every module is a direct child of the session. @pytest.mark.parametrize("parent", ("directory", "package")) def test_terminal_report_lastfailed(self, pytester: Pytester, parent: str) -> None: if parent == "package": @@ -498,18 +626,22 @@ def test_b2(): assert 0 ] ) - def test_terminal_report_failedfirst(self, pytester: Pytester) -> None: - pytester.makepyfile( - test_a=""" - def test_a1(): assert 0 - def test_a2(): pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["collected 2 items", "*1 failed, 1 passed in*"]) + def test_terminal_report_failedfirst(self, tmp_path: Path) -> None: + def test_a1(): + assert 0 - result = pytester.runpytest("--ff") - result.stdout.fnmatch_lines( + def test_a2(): + pass + + test_a = in_memory("test_a", test_a1, test_a2) + + record = run_tests(test_a, spec=cache_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines(["collected 2 items", "*1 failed, 1 passed in*"]) + + record = run_tests( + test_a, spec=cache_spec(tmp_path, "--ff"), capture_output=True + ) + record.stdout.fnmatch_lines( [ "collected 2 items", "run-last-failure: rerun previous 1 failure first", @@ -517,6 +649,9 @@ def test_a2(): pass ] ) + # ensemble: the subject is a module that raises at *import* time; ensemble + # sources are real objects that were imported by the host, so there is no + # import of them left to fail. def test_lastfailed_collectfailure( self, pytester: Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -550,6 +685,7 @@ def rlf(fail_import: int, fail_run: int) -> Any: lastfailed = rlf(fail_import=0, fail_run=1) assert list(lastfailed) == ["test_maybe.py::test_hello"] + # ensemble: same as above - the failure being recorded is an import error. def test_lastfailed_failure_subset( self, pytester: Pytester, monkeypatch: MonkeyPatch ) -> None: @@ -608,248 +744,325 @@ def rlf( assert list(lastfailed) == ["test_maybe.py"] result.stdout.fnmatch_lines(["*2 passed*"]) - def test_lastfailed_creates_cache_when_needed(self, pytester: Pytester) -> None: + def test_lastfailed_creates_cache_when_needed(self, tmp_path: Path) -> None: # Issue #1342 - pytester.makepyfile(test_empty="") - pytester.runpytest("-q", "--lf") - assert not os.path.exists(".pytest_cache/v/cache/lastfailed") - - pytester.makepyfile(test_successful="def test_success():\n assert True") - pytester.runpytest("-q", "--lf") - assert not os.path.exists(".pytest_cache/v/cache/lastfailed") - - pytester.makepyfile(test_errored="def test_error():\n assert False") - pytester.runpytest("-q", "--lf") - assert os.path.exists(".pytest_cache/v/cache/lastfailed") - - def test_xfail_not_considered_failure(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail - def test(): assert 0 - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 xfailed*"]) - assert self.get_cached_last_failed(pytester) == [] - - def test_xfail_strict_considered_failure(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail(strict=True) - def test(): pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 failed*"]) - assert self.get_cached_last_failed(pytester) == [ + # The original's -q only affected rendering, and nothing here reads + # the output; an ensemble without the terminal plugin has no -q. + lastfailed = tmp_path / ".pytest_cache/v/cache/lastfailed" + + test_empty = in_memory("test_empty") + run_tests(test_empty, spec=cache_spec(tmp_path, "--lf")) + assert not lastfailed.exists() + + def test_success(): + assert True + + test_successful = in_memory("test_successful", test_success) + run_tests(test_empty, test_successful, spec=cache_spec(tmp_path, "--lf")) + assert not lastfailed.exists() + + def test_error(): + assert False + + test_errored = in_memory("test_errored", test_error) + run_tests( + test_empty, + test_successful, + test_errored, + spec=cache_spec(tmp_path, "--lf"), + ) + assert lastfailed.exists() + + def test_xfail_not_considered_failure(self, tmp_path: Path) -> None: + @pytest.mark.xfail + def test(): + assert 0 + + record = run_tests( + in_memory("test_xfail_not_considered_failure", test), + spec=cache_spec(tmp_path), + capture_output=True, + ) + record.stdout.fnmatch_lines(["*1 xfailed*"]) + assert self.get_cached_last_failed(tmp_path) == [] + + def test_xfail_strict_considered_failure(self, tmp_path: Path) -> None: + @pytest.mark.xfail(strict=True) + def test(): + pass + + record = run_tests( + in_memory("test_xfail_strict_considered_failure", test), + spec=cache_spec(tmp_path), + capture_output=True, + ) + record.stdout.fnmatch_lines(["*1 failed*"]) + assert self.get_cached_last_failed(tmp_path) == [ "test_xfail_strict_considered_failure.py::test" ] @pytest.mark.parametrize("mark", ["mark.xfail", "mark.skip"]) - def test_failed_changed_to_xfail_or_skip( - self, pytester: Pytester, mark: str - ) -> None: - pytester.makepyfile( - """ - import pytest - def test(): assert 0 - """ + def test_failed_changed_to_xfail_or_skip(self, tmp_path: Path, mark: str) -> None: + decorator, outcomes = { + "mark.xfail": (pytest.mark.xfail, {"xfailed": 1}), + "mark.skip": (pytest.mark.skip, {"skipped": 1}), + }[mark] + + def test(): + assert 0 + + record = run_tests( + in_memory("test_failed_changed_to_xfail_or_skip", test), + spec=cache_spec(tmp_path), ) - result = pytester.runpytest() - assert self.get_cached_last_failed(pytester) == [ + assert self.get_cached_last_failed(tmp_path) == [ "test_failed_changed_to_xfail_or_skip.py::test" ] - assert result.ret == 1 + # ``result.ret == 1``: the run failed. + record.assert_outcomes(failed=1) - pytester.makepyfile( - f""" - import pytest - @pytest.{mark} - def test(): assert 0 - """ + record = run_tests( + in_memory("test_failed_changed_to_xfail_or_skip", decorator(test)), + spec=cache_spec(tmp_path), ) - result = pytester.runpytest() - assert result.ret == 0 - assert self.get_cached_last_failed(pytester) == [] - assert result.ret == 0 + # ``result.ret == 0``: nothing failed any more. + record.assert_outcomes(**outcomes) + assert self.get_cached_last_failed(tmp_path) == [] @pytest.mark.parametrize("quiet", [True, False]) @pytest.mark.parametrize("opt", ["--ff", "--lf"]) def test_lf_and_ff_prints_no_needless_message( - self, quiet: bool, opt: str, pytester: Pytester + self, quiet: bool, opt: str, tmp_path: Path ) -> None: # Issue 3853 - pytester.makepyfile("def test(): assert 0") + def test(): + assert 0 + + module = in_memory("test_lf_and_ff", test) args = [opt] if quiet: args.append("-q") - result = pytester.runpytest(*args) - result.stdout.no_fnmatch_line("*run all*") + record = run_tests( + module, spec=cache_spec(tmp_path, *args), capture_output=True + ) + record.stdout.no_fnmatch_line("*run all*") - result = pytester.runpytest(*args) + record = run_tests( + module, spec=cache_spec(tmp_path, *args), capture_output=True + ) if quiet: - result.stdout.no_fnmatch_line("*run all*") + record.stdout.no_fnmatch_line("*run all*") else: - assert "rerun previous" in result.stdout.str() + assert "rerun previous" in record.output - def get_cached_last_failed(self, pytester: Pytester) -> list[str]: - config = pytester.parseconfigure() - assert config.cache is not None - return sorted(config.cache.get("cache/lastfailed", {})) + def get_cached_last_failed(self, rootpath: Path) -> list[str]: + with configured(cache_spec(rootpath)) as config: + assert config.cache is not None + return sorted(config.cache.get("cache/lastfailed", {})) - def test_cache_cumulative(self, pytester: Pytester) -> None: + def test_cache_cumulative(self, tmp_path: Path) -> None: """Test workflow where user fixes errors gradually file by file using --lf.""" + # The sources are real files here: the workflow is "file by file", and + # the file-level collection skipping that produces the "(skipped N + # files)" messages only applies to paths that exist. # 1. initial run - test_bar = pytester.makepyfile( - test_bar=""" - def test_bar_1(): pass - def test_bar_2(): assert 0 - """ + write_source( + tmp_path, + "test_bar.py", + "def test_bar_1(): pass\ndef test_bar_2(): assert 0\n", ) - test_foo = pytester.makepyfile( - test_foo=""" - def test_foo_3(): pass - def test_foo_4(): assert 0 - """ + write_source( + tmp_path, + "test_foo.py", + "def test_foo_3(): pass\ndef test_foo_4(): assert 0\n", + ) + test_bar = tmp_path / "test_bar.py" + test_foo = tmp_path / "test_foo.py" + + run_tests( + module_from_path(test_bar), + module_from_path(test_foo), + spec=cache_spec(tmp_path), ) - pytester.runpytest() - assert self.get_cached_last_failed(pytester) == [ + assert self.get_cached_last_failed(tmp_path) == [ "test_bar.py::test_bar_2", "test_foo.py::test_foo_4", ] # 2. fix test_bar_2, run only test_bar.py - pytester.makepyfile( - test_bar=""" - def test_bar_1(): pass - def test_bar_2(): pass - """ + write_source( + tmp_path, "test_bar.py", "def test_bar_1(): pass\ndef test_bar_2(): pass\n" ) - result = pytester.runpytest(test_bar) - result.stdout.fnmatch_lines(["*2 passed*"]) + record = run_tests( + module_from_path(test_bar), spec=cache_spec(tmp_path), capture_output=True + ) + record.stdout.fnmatch_lines(["*2 passed*"]) # ensure cache does not forget that test_foo_4 failed once before - assert self.get_cached_last_failed(pytester) == ["test_foo.py::test_foo_4"] + assert self.get_cached_last_failed(tmp_path) == ["test_foo.py::test_foo_4"] - result = pytester.runpytest("--last-failed") - result.stdout.fnmatch_lines( + record = run_tests( + module_from_path(test_bar), + module_from_path(test_foo), + spec=cache_spec(tmp_path, "--last-failed"), + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "collected 1 item", "run-last-failure: rerun previous 1 failure (skipped 1 file)", "*= 1 failed in *", ] ) - assert self.get_cached_last_failed(pytester) == ["test_foo.py::test_foo_4"] + assert self.get_cached_last_failed(tmp_path) == ["test_foo.py::test_foo_4"] # 3. fix test_foo_4, run only test_foo.py - test_foo = pytester.makepyfile( - test_foo=""" - def test_foo_3(): pass - def test_foo_4(): pass - """ - ) - result = pytester.runpytest(test_foo, "--last-failed") - result.stdout.fnmatch_lines( + write_source( + tmp_path, "test_foo.py", "def test_foo_3(): pass\ndef test_foo_4(): pass\n" + ) + record = run_tests( + module_from_path(test_foo), + spec=cache_spec(tmp_path, "--last-failed"), + capture_output=True, + ) + # The original passed test_foo.py as an argument, which made it an + # initial path and so exempt from the file-level filtering; here the + # known failure is filtered out during collection instead of being + # deselected afterwards, so this collects 1 item rather than + # collecting 2 and deselecting 1. + record.stdout.fnmatch_lines( [ - "collected 2 items / 1 deselected / 1 selected", + "collected 1 item", "run-last-failure: rerun previous 1 failure", - "*= 1 passed, 1 deselected in *", + "*= 1 passed in *", ] ) - assert self.get_cached_last_failed(pytester) == [] + assert self.get_cached_last_failed(tmp_path) == [] - result = pytester.runpytest("--last-failed") - result.stdout.fnmatch_lines(["*4 passed*"]) - assert self.get_cached_last_failed(pytester) == [] - - def test_lastfailed_no_failures_behavior_all_passed( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( - """ - def test_1(): pass - def test_2(): pass - """ + record = run_tests( + module_from_path(test_bar), + module_from_path(test_foo), + spec=cache_spec(tmp_path, "--last-failed"), + capture_output=True, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*2 passed*"]) - result = pytester.runpytest("--lf") - result.stdout.fnmatch_lines(["*2 passed*"]) - result = pytester.runpytest("--lf", "--lfnf", "all") - result.stdout.fnmatch_lines(["*2 passed*"]) + record.stdout.fnmatch_lines(["*4 passed*"]) + assert self.get_cached_last_failed(tmp_path) == [] - # Ensure the list passed to pytest_deselected is a copy, - # and not a reference which is cleared right after. - pytester.makeconftest( - """ - deselected = [] + def test_lastfailed_no_failures_behavior_all_passed(self, tmp_path: Path) -> None: + def test_1(): + pass - def pytest_deselected(items): - global deselected - deselected = items + def test_2(): + pass - def pytest_sessionfinish(): - print("\\ndeselected={}".format(len(deselected))) - """ + module = in_memory("test_lastfailed_no_failures", test_1, test_2) + + record = run_tests(module, spec=cache_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines(["*2 passed*"]) + record = run_tests( + module, spec=cache_spec(tmp_path, "--lf"), capture_output=True + ) + record.stdout.fnmatch_lines(["*2 passed*"]) + record = run_tests( + module, + spec=cache_spec(tmp_path, "--lf", "--lfnf", "all"), + capture_output=True, ) + record.stdout.fnmatch_lines(["*2 passed*"]) - result = pytester.runpytest("--lf", "--lfnf", "none") - result.stdout.fnmatch_lines( + # Ensure the list passed to pytest_deselected is a copy, + # and not a reference which is cleared right after. + class DeselectedPlugin: + def __init__(self) -> None: + self.deselected: list[object] = [] + + def pytest_deselected(self, items): + self.deselected = items + + plugin = DeselectedPlugin() + record = run_tests( + module, + spec=cache_spec(tmp_path, "--lf", "--lfnf", "none").replace( + extra_plugins=(plugin,) + ), + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "collected 2 items / 2 deselected / 0 selected", "run-last-failure: no previously failed tests, deselecting all items.", - "deselected=2", "* 2 deselected in *", ] ) - assert result.ret == ExitCode.NO_TESTS_COLLECTED + # The original printed this from a sessionfinish hook; asserting the + # retained list directly is what that print was a proxy for. + assert len(plugin.deselected) == 2 + # ``result.ret == ExitCode.NO_TESTS_COLLECTED`` + assert record.outcomes() == {} + assert record.deselected == 2 - def test_lastfailed_no_failures_behavior_empty_cache( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( - """ - def test_1(): pass - def test_2(): assert 0 - """ + def test_lastfailed_no_failures_behavior_empty_cache(self, tmp_path: Path) -> None: + def test_1(): + pass + + def test_2(): + assert 0 + + module = in_memory("test_lastfailed_empty_cache", test_1, test_2) + + record = run_tests( + module, + spec=cache_spec(tmp_path, "--lf", "--cache-clear"), + capture_output=True, + ) + record.stdout.fnmatch_lines(["*1 failed*1 passed*"]) + record = run_tests( + module, + spec=cache_spec(tmp_path, "--lf", "--cache-clear", "--lfnf", "all"), + capture_output=True, + ) + record.stdout.fnmatch_lines(["*1 failed*1 passed*"]) + record = run_tests( + module, + spec=cache_spec(tmp_path, "--lf", "--cache-clear", "--lfnf", "none"), + capture_output=True, ) - result = pytester.runpytest("--lf", "--cache-clear") - result.stdout.fnmatch_lines(["*1 failed*1 passed*"]) - result = pytester.runpytest("--lf", "--cache-clear", "--lfnf", "all") - result.stdout.fnmatch_lines(["*1 failed*1 passed*"]) - result = pytester.runpytest("--lf", "--cache-clear", "--lfnf", "none") - result.stdout.fnmatch_lines(["*2 desel*"]) + record.stdout.fnmatch_lines(["*2 desel*"]) - def test_lastfailed_skip_collection(self, pytester: Pytester) -> None: + def test_lastfailed_skip_collection(self, tmp_path: Path) -> None: """ Test --lf behavior regarding skipping collection of files that are not marked as failed in the cache (#5172). """ - pytester.makepyfile( - **{ - "pkg1/test_1.py": """ - import pytest - - @pytest.mark.parametrize('i', range(3)) - def test_1(i): pass - """, - "pkg2/test_2.py": """ - import pytest - - @pytest.mark.parametrize('i', range(5)) - def test_1(i): - assert i not in (1, 3) - """, - } - ) + write_source( + tmp_path, + "pkg1/test_1.py", + "import pytest\n\n" + "@pytest.mark.parametrize('i', range(3))\n" + "def test_1(i): pass\n", + ) + write_source( + tmp_path, + "pkg2/test_2.py", + "import pytest\n\n" + "@pytest.mark.parametrize('i', range(5))\n" + "def test_1(i):\n" + " assert i not in (1, 3)\n", + ) + + def sources() -> tuple[types.ModuleType, ...]: + return tuple( + module_from_path(path) + for path in sorted(tmp_path.rglob("pkg*/test_*.py")) + ) + # first run: collects 8 items (test_1: 3, test_2: 5) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["collected 8 items", "*2 failed*6 passed*"]) + record = run_tests(*sources(), spec=cache_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines(["collected 8 items", "*2 failed*6 passed*"]) # second run: collects only 5 items from test_2, because all tests from test_1 have passed - result = pytester.runpytest("--lf") - result.stdout.fnmatch_lines( + record = run_tests( + *sources(), spec=cache_spec(tmp_path, "--lf"), capture_output=True + ) + record.stdout.fnmatch_lines( [ "collected 2 items", "run-last-failure: rerun previous 2 failures (skipped 1 file)", @@ -858,15 +1071,11 @@ def test_1(i): ) # add another file and check if message is correct when skipping more than 1 file - pytester.makepyfile( - **{ - "pkg1/test_3.py": """ - def test_3(): pass - """ - } + write_source(tmp_path, "pkg1/test_3.py", "def test_3(): pass\n") + record = run_tests( + *sources(), spec=cache_spec(tmp_path, "--lf"), capture_output=True ) - result = pytester.runpytest("--lf") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 2 items", "run-last-failure: rerun previous 2 failures (skipped 2 files)", @@ -874,6 +1083,9 @@ def test_3(): pass ] ) + # ensemble: the point is a file nested at a different *level* of the + # collection tree; an ensemble's tree is flat - every module is a direct + # child of the session - and packages have no representation in it. def test_lastfailed_skip_collection_with_nesting(self, pytester: Pytester) -> None: """Check that file skipping works even when the file with failures is nested at a different level of the collection tree.""" @@ -902,20 +1114,28 @@ def test_2(): assert False ) def test_lastfailed_with_known_failures_not_being_selected( - self, pytester: Pytester + self, tmp_path: Path ) -> None: - pytester.makepyfile( - **{ - "pkg1/test_1.py": """def test_1(): assert 0""", - "pkg1/test_2.py": """def test_2(): pass""", - } - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) - - Path("pkg1/test_1.py").unlink() - result = pytester.runpytest("--lf") - result.stdout.fnmatch_lines( + write_source(tmp_path, "pkg1/test_1.py", """def test_1(): assert 0""") + write_source(tmp_path, "pkg1/test_2.py", """def test_2(): pass""") + test_1 = tmp_path / "pkg1/test_1.py" + test_2 = tmp_path / "pkg1/test_2.py" + + record = run_tests( + module_from_path(test_1), + module_from_path(test_2), + spec=cache_spec(tmp_path), + capture_output=True, + ) + record.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) + + test_1.unlink() + record = run_tests( + module_from_path(test_2), + spec=cache_spec(tmp_path, "--lf"), + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "collected 1 item", "run-last-failure: 1 known failures not in selected tests", @@ -924,9 +1144,14 @@ def test_lastfailed_with_known_failures_not_being_selected( ) # Recreate file with known failure. - pytester.makepyfile(**{"pkg1/test_1.py": """def test_1(): assert 0"""}) - result = pytester.runpytest("--lf") - result.stdout.fnmatch_lines( + write_source(tmp_path, "pkg1/test_1.py", """def test_1(): assert 0""") + record = run_tests( + module_from_path(test_1), + module_from_path(test_2), + spec=cache_spec(tmp_path, "--lf"), + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "collected 1 item", "run-last-failure: rerun previous 1 failure (skipped 1 file)", @@ -935,156 +1160,168 @@ def test_lastfailed_with_known_failures_not_being_selected( ) # Remove/rename test: collects the file again. - pytester.makepyfile(**{"pkg1/test_1.py": """def test_renamed(): assert 0"""}) - result = pytester.runpytest("--lf", "-rf") - result.stdout.fnmatch_lines( + write_source(tmp_path, "pkg1/test_1.py", """def test_renamed(): assert 0""") + record = run_tests( + module_from_path(test_1), + module_from_path(test_2), + spec=cache_spec(tmp_path, "--lf", "-rf"), + capture_output=True, + ) + record.stdout.fnmatch_lines( [ "collected 2 items", "run-last-failure: 1 known failures not in selected tests", "pkg1/test_1.py F *", "pkg1/test_2.py . *", - "FAILED pkg1/test_1.py::test_renamed - assert 0", + # Assertion rewriting is not applied to ensemble sources, so + # the one-line reason is the bare exception. + "FAILED pkg1/test_1.py::test_renamed - AssertionError", "* 1 failed, 1 passed in *", ] ) - result = pytester.runpytest("--lf", "--co") - result.stdout.fnmatch_lines( + record = run_tests( + module_from_path(test_1), + module_from_path(test_2), + spec=cache_spec(tmp_path, "--lf", "--co"), + capture_output=True, + ) + # The tree has no nodes: an ensemble collects modules directly + # under the session. + record.stdout.fnmatch_lines( [ "collected 1 item", "run-last-failure: rerun previous 1 failure (skipped 1 file)", "", - "", - " ", - " ", - " ", + "", + " ", ] ) - def test_lastfailed_args_with_deselected(self, pytester: Pytester) -> None: + def test_lastfailed_args_with_deselected(self, tmp_path: Path) -> None: """Test regression with --lf running into NoMatch error. This was caused by it not collecting (non-failed) nodes given as arguments. """ - pytester.makepyfile( - **{ - "pkg1/test_1.py": """ - def test_pass(): pass - def test_fail(): assert 0 - """, - } - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) - assert result.ret == 1 - result = pytester.runpytest("pkg1/test_1.py::test_pass", "--lf", "--co") - assert result.ret == 0 - result.stdout.fnmatch_lines( + def test_pass(): + pass + + def test_fail(): + assert 0 + + module = in_memory("pkg1/test_1", test_pass, test_fail) + + record = run_tests(module, spec=cache_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) + record.assert_outcomes(passed=1, failed=1) + + # Selecting single node ids on the command line has no ensemble + # equivalent; -k selects the same items. + record = run_tests( + module, + spec=cache_spec(tmp_path, "-k", "test_pass", "--lf", "--co"), + capture_output=True, + ) + record.stdout.fnmatch_lines( [ - "*collected 1 item", + "*collected 2 items / 1 deselected / 1 selected", "run-last-failure: 1 known failures not in selected tests", "", - "", - " ", - " ", - " ", + "", + " ", ], consecutive=True, ) - result = pytester.runpytest( - "pkg1/test_1.py::test_pass", "pkg1/test_1.py::test_fail", "--lf", "--co" + record = run_tests( + module, spec=cache_spec(tmp_path, "--lf", "--co"), capture_output=True ) - assert result.ret == 0 - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 2 items / 1 deselected / 1 selected", "run-last-failure: rerun previous 1 failure", "", - "", - " ", - " ", - " ", + "", + " ", + "", "*= 1/2 tests collected (1 deselected) in *", ], ) - def test_lastfailed_with_class_items(self, pytester: Pytester) -> None: + def test_lastfailed_with_class_items(self, tmp_path: Path) -> None: """Test regression with --lf deselecting whole classes.""" - pytester.makepyfile( - **{ - "pkg1/test_1.py": """ - class TestFoo: - def test_pass(self): pass - def test_fail(self): assert 0 - def test_other(): assert 0 - """, - } - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["collected 3 items", "* 2 failed, 1 passed in *"]) - assert result.ret == 1 + class TestFoo: + def test_pass(self): + pass - result = pytester.runpytest("--lf", "--co") - assert result.ret == 0 - result.stdout.fnmatch_lines( + def test_fail(self): + assert 0 + + def test_other(): + assert 0 + + module = in_memory("pkg1/test_1", TestFoo, test_other) + + record = run_tests(module, spec=cache_spec(tmp_path), capture_output=True) + record.stdout.fnmatch_lines(["collected 3 items", "* 2 failed, 1 passed in *"]) + record.assert_outcomes(passed=1, failed=2) + + record = run_tests( + module, spec=cache_spec(tmp_path, "--lf", "--co"), capture_output=True + ) + record.stdout.fnmatch_lines( [ "collected 3 items / 1 deselected / 2 selected", "run-last-failure: rerun previous 2 failures", "", - "", - " ", - " ", - " ", - " ", - " ", + "", + " ", + " ", + " ", "", "*= 2/3 tests collected (1 deselected) in *", ], consecutive=True, ) - def test_lastfailed_with_all_filtered(self, pytester: Pytester) -> None: - pytester.makepyfile( - **{ - "pkg1/test_1.py": """ - def test_fail(): assert 0 - def test_pass(): pass - """, - } + def test_lastfailed_with_all_filtered(self, tmp_path: Path) -> None: + def test_fail(): + assert 0 + + def test_pass(): + pass + + record = run_tests( + in_memory("pkg1/test_1", test_fail, test_pass), + spec=cache_spec(tmp_path), + capture_output=True, ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) - assert result.ret == 1 + record.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) + record.assert_outcomes(passed=1, failed=1) # Remove known failure. - pytester.makepyfile( - **{ - "pkg1/test_1.py": """ - def test_pass(): pass - """, - } + record = run_tests( + in_memory("pkg1/test_1", test_pass), + spec=cache_spec(tmp_path, "--lf", "--co"), + capture_output=True, ) - result = pytester.runpytest("--lf", "--co") - result.stdout.fnmatch_lines( + record.stdout.fnmatch_lines( [ "collected 1 item", "run-last-failure: 1 known failures not in selected tests", "", - "", - " ", - " ", - " ", + "", + " ", "", "*= 1 test collected in*", ], consecutive=True, ) - assert result.ret == 0 + # ensemble: Package nodes are the whole subject, and an ensemble has none. def test_packages(self, pytester: Pytester) -> None: """Regression test for #7758. @@ -1115,21 +1352,42 @@ def test_packages(self, pytester: Pytester) -> None: result = pytester.runpytest("--lf") result.assert_outcomes(failed=3) - def test_non_python_file_skipped( - self, - pytester: Pytester, - dummy_yaml_custom_test: None, - ) -> None: - pytester.makepyfile( - **{ - "test_bad.py": """def test_bad(): assert False""", - }, - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) + def test_non_python_file_skipped(self, tmp_path: Path) -> None: + # The yaml collector of the ``dummy_yaml_custom_test`` fixture, as a + # collector handed to the ensemble directly: ``pytest_collect_file`` is + # never called, because an ensemble is given its collectors rather than + # walking the filesystem for them. + class YamlItem(pytest.Item): + def runtest(self) -> None: + pass - result = pytester.runpytest("--lf") - result.stdout.fnmatch_lines( + class YamlFile(pytest.File): + def collect(self): + yield YamlItem.from_parent(name=self.path.name, parent=self) + + tmp_path.joinpath("test1.yaml").write_text("", encoding="utf-8") + write_source(tmp_path, "test_bad.py", """def test_bad(): assert False""") + test_bad = tmp_path / "test_bad.py" + + def run(*args: str): + with Ensemble( + module_from_path(test_bad), + spec=cache_spec(tmp_path, *args), + capture_output=True, + ) as ensemble: + ensemble_collection(ensemble.session).collectors.append( + YamlFile.from_parent( + parent=ensemble.session, path=tmp_path / "test1.yaml" + ) + ) + record = ensemble.run() + return ensemble.final_record(record) + + record = run() + record.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) + + record = run("--lf") + record.stdout.fnmatch_lines( [ "collected 1 item", "run-last-failure: rerun previous 1 failure (skipped 1 file)", @@ -1139,85 +1397,107 @@ def test_non_python_file_skipped( class TestNewFirst: - def test_newfirst_usecase(self, pytester: Pytester) -> None: - pytester.makepyfile( - **{ - "test_1/test_1.py": """ - def test_1(): assert 1 - """, - "test_2/test_2.py": """ - def test_1(): assert 1 - """, - } - ) + def test_newfirst_usecase(self, tmp_path: Path) -> None: + write_source(tmp_path, "test_1/test_1.py", "def test_1(): assert 1\n") + write_source(tmp_path, "test_2/test_2.py", "def test_1(): assert 1\n") - p1 = pytester.path.joinpath("test_1/test_1.py") + p1 = tmp_path.joinpath("test_1/test_1.py") + p2 = tmp_path.joinpath("test_2/test_2.py") os.utime(p1, ns=(p1.stat().st_atime_ns, int(1e9))) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + def sources() -> tuple[types.ModuleType, ...]: + # Distinct module names, so that both are importable side by side. + return ( + module_from_path(p1, "test_1_test_1"), + module_from_path(p2, "test_2_test_2"), + ) + + def ran(record) -> list[str]: + return [r.nodeid for r in record.reports if r.when == "call"] + + record = run_tests( + *sources(), spec=cache_spec(tmp_path, "-v"), capture_output=True + ) + record.stdout.fnmatch_lines( ["*test_1/test_1.py::test_1 PASSED*", "*test_2/test_2.py::test_1 PASSED*"] ) + assert ran(record) == ["test_1/test_1.py::test_1", "test_2/test_2.py::test_1"] - result = pytester.runpytest("-v", "--nf") - result.stdout.fnmatch_lines( + record = run_tests( + *sources(), spec=cache_spec(tmp_path, "-v", "--nf"), capture_output=True + ) + record.stdout.fnmatch_lines( ["*test_2/test_2.py::test_1 PASSED*", "*test_1/test_1.py::test_1 PASSED*"] ) + assert ran(record) == ["test_2/test_2.py::test_1", "test_1/test_1.py::test_1"] p1.write_text( "def test_1(): assert 1\ndef test_2(): assert 1\n", encoding="utf-8" ) os.utime(p1, ns=(p1.stat().st_atime_ns, int(1e9))) - result = pytester.runpytest("--nf", "--collect-only", "-q") - result.stdout.fnmatch_lines( - [ - "test_1/test_1.py::test_2", - "test_2/test_2.py::test_1", - "test_1/test_1.py::test_1", - ] - ) + items = collect_tests(*sources(), spec=cache_spec(tmp_path, "--nf", "--co")) + assert [item.nodeid for item in items] == [ + "test_1/test_1.py::test_2", + "test_2/test_2.py::test_1", + "test_1/test_1.py::test_1", + ] # Newest first with (plugin) pytest_collection_modifyitems hook. - pytester.makepyfile( - myplugin=""" - def pytest_collection_modifyitems(items): + class MyPlugin: + def __init__(self) -> None: + self.new_items: list[str] = [] + + def pytest_collection_modifyitems(self, items): items[:] = sorted(items, key=lambda item: item.nodeid) - print("new_items:", [x.nodeid for x in items]) - """ - ) - pytester.syspathinsert() - result = pytester.runpytest("--nf", "-p", "myplugin", "--collect-only", "-q") - result.stdout.fnmatch_lines( - [ - "new_items: *test_1.py*test_1.py*test_2.py*", - "test_1/test_1.py::test_2", - "test_2/test_2.py::test_1", - "test_1/test_1.py::test_1", - ] + self.new_items = [x.nodeid for x in items] + + plugin = MyPlugin() + items = collect_tests( + *sources(), + spec=cache_spec(tmp_path, "--nf", "--co").replace(extra_plugins=(plugin,)), ) + assert plugin.new_items == [ + "test_1/test_1.py::test_1", + "test_1/test_1.py::test_2", + "test_2/test_2.py::test_1", + ] + assert [item.nodeid for item in items] == [ + "test_1/test_1.py::test_2", + "test_2/test_2.py::test_1", + "test_1/test_1.py::test_1", + ] - def test_newfirst_parametrize(self, pytester: Pytester) -> None: - pytester.makepyfile( - **{ - "test_1/test_1.py": """ - import pytest - @pytest.mark.parametrize('num', [1, 2]) - def test_1(num): assert num - """, - "test_2/test_2.py": """ - import pytest - @pytest.mark.parametrize('num', [1, 2]) - def test_1(num): assert num - """, - } + def test_newfirst_parametrize(self, tmp_path: Path) -> None: + write_source( + tmp_path, + "test_1/test_1.py", + "import pytest\n" + "@pytest.mark.parametrize('num', [1, 2])\n" + "def test_1(num): assert num\n", + ) + write_source( + tmp_path, + "test_2/test_2.py", + "import pytest\n" + "@pytest.mark.parametrize('num', [1, 2])\n" + "def test_1(num): assert num\n", ) - p1 = pytester.path.joinpath("test_1/test_1.py") + p1 = tmp_path.joinpath("test_1/test_1.py") + p2 = tmp_path.joinpath("test_2/test_2.py") os.utime(p1, ns=(p1.stat().st_atime_ns, int(1e9))) - result = pytester.runpytest("-v") - result.stdout.fnmatch_lines( + def sources(*paths: Path) -> tuple[types.ModuleType, ...]: + return tuple( + module_from_path(path, f"{path.parent.name}_{path.stem}") + for path in (paths or (p1, p2)) + ) + + record = run_tests( + *sources(), spec=cache_spec(tmp_path, "-v"), capture_output=True + ) + record.stdout.fnmatch_lines( [ "*test_1/test_1.py::test_1[1*", "*test_1/test_1.py::test_1[2*", @@ -1226,8 +1506,10 @@ def test_1(num): assert num ] ) - result = pytester.runpytest("-v", "--nf") - result.stdout.fnmatch_lines( + record = run_tests( + *sources(), spec=cache_spec(tmp_path, "-v", "--nf"), capture_output=True + ) + record.stdout.fnmatch_lines( [ "*test_2/test_2.py::test_1[1*", "*test_2/test_2.py::test_1[2*", @@ -1245,13 +1527,17 @@ def test_1(num): assert num os.utime(p1, ns=(p1.stat().st_atime_ns, int(1e9))) # Running only a subset does not forget about existing ones. - result = pytester.runpytest("-v", "--nf", "test_2/test_2.py") - result.stdout.fnmatch_lines( + record = run_tests( + *sources(p2), spec=cache_spec(tmp_path, "-v", "--nf"), capture_output=True + ) + record.stdout.fnmatch_lines( ["*test_2/test_2.py::test_1[1*", "*test_2/test_2.py::test_1[2*"] ) - result = pytester.runpytest("-v", "--nf") - result.stdout.fnmatch_lines( + record = run_tests( + *sources(), spec=cache_spec(tmp_path, "-v", "--nf"), capture_output=True + ) + record.stdout.fnmatch_lines( [ "*test_1/test_1.py::test_1[3*", "*test_2/test_2.py::test_1[1*", @@ -1263,21 +1549,25 @@ def test_1(num): assert num class TestReadme: - def check_readme(self, pytester: Pytester) -> bool: - config = pytester.parseconfigure() - assert config.cache is not None - readme = config.cache._cachedir.joinpath("README.md") - return readme.is_file() + def check_readme(self, rootpath: Path) -> bool: + with configured(cache_spec(rootpath)) as config: + assert config.cache is not None + readme = config.cache._cachedir.joinpath("README.md") + return readme.is_file() + + def test_readme_passed(self, tmp_path: Path) -> None: + def test_always_passes(): + pass + + run_tests(test_always_passes, spec=cache_spec(tmp_path)) + assert self.check_readme(tmp_path) is True - def test_readme_passed(self, pytester: Pytester) -> None: - pytester.makepyfile("def test_always_passes(): pass") - pytester.runpytest() - assert self.check_readme(pytester) is True + def test_readme_failed(self, tmp_path: Path) -> None: + def test_always_fails(): + assert 0 - def test_readme_failed(self, pytester: Pytester) -> None: - pytester.makepyfile("def test_always_fails(): assert 0") - pytester.runpytest() - assert self.check_readme(pytester) is True + run_tests(test_always_fails, spec=cache_spec(tmp_path)) + assert self.check_readme(tmp_path) is True class Action(Enum): @@ -1289,76 +1579,72 @@ class Action(Enum): @pytest.mark.parametrize("action", list(Action)) def test_gitignore( - pytester: Pytester, + tmp_path: Path, action: Action, ) -> None: """Ensure we automatically create .gitignore file in the pytest_cache directory (#3286).""" from _pytest.cacheprovider import Cache - config = pytester.parseconfig() - cache = Cache.for_config(config, _ispytest=True) - if action == Action.MKDIR: - cache.mkdir("foo") - elif action == Action.SET: - cache.set("foo", "bar") - else: - assert_never(action) - msg = "# Created by pytest automatically.\n*\n" - gitignore_path = cache._cachedir.joinpath(".gitignore") - assert gitignore_path.read_text(encoding="UTF-8") == msg - - # Does not overwrite existing/custom one. - gitignore_path.write_text("custom", encoding="utf-8") - if action == Action.MKDIR: - cache.mkdir("something") - elif action == Action.SET: - cache.set("something", "else") - else: - assert_never(action) - assert gitignore_path.read_text(encoding="UTF-8") == "custom" + with configured(cache_spec(tmp_path)) as config: + cache = Cache.for_config(config, _ispytest=True) + if action == Action.MKDIR: + cache.mkdir("foo") + elif action == Action.SET: + cache.set("foo", "bar") + else: + assert_never(action) + msg = "# Created by pytest automatically.\n*\n" + gitignore_path = cache._cachedir.joinpath(".gitignore") + assert gitignore_path.read_text(encoding="UTF-8") == msg + + # Does not overwrite existing/custom one. + gitignore_path.write_text("custom", encoding="utf-8") + if action == Action.MKDIR: + cache.mkdir("something") + elif action == Action.SET: + cache.set("something", "else") + else: + assert_never(action) + assert gitignore_path.read_text(encoding="UTF-8") == "custom" -def test_preserve_keys_order(pytester: Pytester) -> None: +def test_preserve_keys_order(tmp_path: Path) -> None: """Ensure keys order is preserved when saving dicts (#9205).""" from _pytest.cacheprovider import Cache - config = pytester.parseconfig() - cache = Cache.for_config(config, _ispytest=True) - cache.set("foo", {"z": 1, "b": 2, "a": 3, "d": 10}) - read_back = cache.get("foo", None) - assert list(read_back.items()) == [("z", 1), ("b", 2), ("a", 3), ("d", 10)] + with configured(cache_spec(tmp_path)) as config: + cache = Cache.for_config(config, _ispytest=True) + cache.set("foo", {"z": 1, "b": 2, "a": 3, "d": 10}) + read_back = cache.get("foo", None) + assert list(read_back.items()) == [("z", 1), ("b", 2), ("a", 3), ("d", 10)] -def test_does_not_create_boilerplate_in_existing_dirs(pytester: Pytester) -> None: +def test_does_not_create_boilerplate_in_existing_dirs(tmp_path: Path) -> None: from _pytest.cacheprovider import Cache - pytester.makeini( - """ - [pytest] - cache_dir = . - """ - ) - config = pytester.parseconfig() - cache = Cache.for_config(config, _ispytest=True) - cache.set("foo", "bar") + with configured(cache_spec(tmp_path, cache_dir=".")) as config: + cache = Cache.for_config(config, _ispytest=True) + cache.set("foo", "bar") - assert os.path.isdir("v") # cache contents - assert not os.path.exists(".gitignore") - assert not os.path.exists("README.md") + assert tmp_path.joinpath("v").is_dir() # cache contents + assert not tmp_path.joinpath(".gitignore").exists() + assert not tmp_path.joinpath("README.md").exists() -def test_cachedir_tag(pytester: Pytester) -> None: +def test_cachedir_tag(tmp_path: Path) -> None: """Ensure we automatically create CACHEDIR.TAG file in the pytest_cache directory (#4278).""" from _pytest.cacheprovider import Cache from _pytest.cacheprovider import CACHEDIR_FILES - config = pytester.parseconfig() - cache = Cache.for_config(config, _ispytest=True) - cache.set("foo", "bar") - cachedir_tag_path = cache._cachedir.joinpath("CACHEDIR.TAG") - assert cachedir_tag_path.read_bytes() == CACHEDIR_FILES["CACHEDIR.TAG"] + with configured(cache_spec(tmp_path)) as config: + cache = Cache.for_config(config, _ispytest=True) + cache.set("foo", "bar") + cachedir_tag_path = cache._cachedir.joinpath("CACHEDIR.TAG") + assert cachedir_tag_path.read_bytes() == CACHEDIR_FILES["CACHEDIR.TAG"] +# ensemble: --help is served from pytest_cmdline_main, which an ensemble +# never runs. def test_clioption_with_cacheshow_and_help(pytester: Pytester) -> None: result = pytester.runpytest("--cache-show", "--help") assert result.ret == 0 From e65f3e9da3162139def9ce659318b12cebe0b060 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 10:43:26 +0200 Subject: [PATCH 21/30] testing: port test_junitxml.py to _pytest.ensemble 47 of 67 pytester functions (75 of 140 instances). junitxml works in an ensemble unchanged: the plugin loads, pytest_configure builds the LogXML, and the file is written from pytest_sessionfinish as the context exits, so it exists by the time run_tests returns. The classname hazard I warned about turned out not to exist: junitxml derives classname from report.nodeid via mangle_test_address, not from the code object, and the synthesized module name *is* the nodeid's first segment. Only the xunit1 file/line attributes follow the code object, and nothing here asserts on them. The two tests that do need real paths use module_from_path. run_and_parse kept its shape and became ensemble-backed; the original is preserved as run_and_parse_pytester for the tests that stayed. Every port asserts more than before, `assert result.ret` becoming assert_outcomes. test_record_fixtures_xunit2 was rebinding expected_lines and silently dropping its first expected line - both warnings are asserted now. 20 stay on pytester, 13 of them for item-level capture, which is the single largest remaining blocker in this file. Wall clock 6.47s -> 6.23s: this file's pytester runs were already inline, so the saving is only pytester's fixture setup, and the residual cost is in the 56 instances that could not move. --- testing/test_junitxml.py | 1262 +++++++++++++++++++++----------------- 1 file changed, 695 insertions(+), 567 deletions(-) diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index bee1cf2fc75..44b2fa11150 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -1,5 +1,9 @@ from __future__ import annotations +from collections.abc import Callable +from collections.abc import Generator +from collections.abc import Mapping +from collections.abc import Sequence from datetime import datetime from datetime import timezone import os @@ -13,6 +17,14 @@ import xmlschema from _pytest.config import Config +from _pytest.config import UsageError +from _pytest.ensemble import build_module +from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import DEFAULT_MODULE_NAME +from _pytest.ensemble import module_from_path +from _pytest.ensemble import run_tests +from _pytest.ensemble import RunRecord +from _pytest.ensemble import Source from _pytest.junitxml import _JunitDurationReport from _pytest.junitxml import _JunitFamily from _pytest.junitxml import _JunitLogging @@ -28,6 +40,11 @@ import pytest +#: What ``record_property``/``record_xml_attribute``/``record_testsuite_property`` +#: hand to a test. +RecordFunc = Callable[[str, object], None] + + @pytest.fixture(scope="session") def schema() -> xmlschema.XMLSchema: """Return an xmlschema.XMLSchema object for the junit-10.xsd file.""" @@ -37,6 +54,70 @@ def schema() -> xmlschema.XMLSchema: class RunAndParse: + """Run in-memory sources under ``junitxml`` and parse the XML it wrote. + + The junit report is written from ``pytest_sessionfinish``, which for an + ensemble is the exit of :func:`run_tests` - so the file only exists once + that has returned. + + ``name`` is the name of the synthesized module holding loose sources, + and the XML quotes it as the ``classname`` of every testcase, exactly + where the pytester version quoted the name of the generated file. It is + therefore part of what a test asserts, not an implementation detail. + """ + + def __init__(self, tmp_path: Path, schema: xmlschema.XMLSchema) -> None: + self.tmp_path = tmp_path + self.schema = schema + self.xml_path = tmp_path.joinpath("junit.xml") + + def __call__( + self, + *sources: Source, + name: str = DEFAULT_MODULE_NAME, + args: Sequence[str] = (), + inicfg: Mapping[str, object] | None = None, + family: _JunitFamily | None = "xunit1", + suite_name: str = "pytest", + ) -> tuple[RunRecord, DomDocument]: + argv = tuple(args) + if family: + argv = ("-o", "junit_family=" + family, *argv) + spec = ConfigSpec( + rootpath=self.tmp_path, + args=(f"--junitxml={self.xml_path}", *argv), + inicfg=inicfg if inicfg is not None else {}, + ).with_plugins("junitxml") + record = run_tests(*sources, spec=spec, name=name) + if family == "xunit2": + with self.xml_path.open(encoding="utf-8") as f: + self.schema.validate(f) + xmldoc = minidom.parse(str(self.xml_path)) + # Ensure the tests attribute of the ```` element + # always matches the number of ```` elements (#3580). + doc = DomDocument(xmldoc) + testcase_nodes = doc.find_by_tag("testcase") + test_suite_node = doc.get_first_by_tag("testsuite") + test_suite_node.assert_attr(name=suite_name, tests=len(testcase_nodes)) + return record, doc + + +@pytest.fixture +def run_and_parse(tmp_path: Path, schema: xmlschema.XMLSchema) -> RunAndParse: + """Fixture that returns a function that runs the given in-memory sources + with ``--junitxml`` and returns the run record plus the parsed + ``DomNode`` of the root xml node. + + The ``family`` parameter is used to configure the ``junit_family`` of the written report. + "xunit2" is also automatically validated against the schema. + """ + return RunAndParse(tmp_path, schema) + + +class RunAndParsePytester: + """The pytester-backed original, kept for the tests that need a real + directory tree, a conftest, a subprocess or captured item output.""" + def __init__(self, pytester: Pytester, schema: xmlschema.XMLSchema) -> None: self.pytester = pytester self.schema = schema @@ -65,14 +146,11 @@ def __call__( @pytest.fixture -def run_and_parse(pytester: Pytester, schema: xmlschema.XMLSchema) -> RunAndParse: - """Fixture that returns a function that can be used to execute pytest and - return the parsed ``DomNode`` of the root xml node. - - The ``family`` parameter is used to configure the ``junit_family`` of the written report. - "xunit2" is also automatically validated against the schema. - """ - return RunAndParse(pytester, schema) +def run_and_parse_pytester( + pytester: Pytester, schema: xmlschema.XMLSchema +) -> RunAndParsePytester: + """``run_and_parse`` for the tests that could not move to an ensemble.""" + return RunAndParsePytester(pytester, schema) def assert_attr(node: minidom.Element, **kwargs: object) -> None: @@ -210,107 +288,123 @@ def test_node_repr(self, document: DomDocument) -> None: class TestPython: @parametrize_families def test_summing_simple( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest - def test_pass(): - pass - def test_fail(): - assert 0 - def test_skip(): - pytest.skip("") - @pytest.mark.xfail - def test_xfail(): - assert 0 - @pytest.mark.xfail - def test_xpass(): - assert 1 - """ + def test_pass() -> None: + pass + + def test_fail() -> None: + assert 0 + + def test_skip() -> None: + pytest.skip("") + + @pytest.mark.xfail + def test_xfail() -> None: + assert 0 + + @pytest.mark.xfail + def test_xpass() -> None: + assert 1 + + record, dom = run_and_parse( + test_pass, + test_fail, + test_skip, + test_xfail, + test_xpass, + family=xunit_family, ) - result, dom = run_and_parse(family=xunit_family) - assert result.ret + # `assert result.ret` only said "nonzero"; the record says which. + record.assert_outcomes(passed=1, failed=1, skipped=1, xfailed=1, xpassed=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(name="pytest", errors=0, failures=1, skipped=2, tests=5) @parametrize_families def test_summing_simple_with_errors( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def fixture(): - raise Exception() - def test_pass(): - pass - def test_fail(): - assert 0 - def test_error(fixture): - pass - @pytest.mark.xfail - def test_xfail(): - assert False - @pytest.mark.xfail(strict=True) - def test_xpass(): - assert True - """ + @pytest.fixture + def fixture() -> None: + raise Exception() + + def test_pass() -> None: + pass + + def test_fail() -> None: + assert 0 + + def test_error(fixture: None) -> None: + pass + + @pytest.mark.xfail + def test_xfail() -> None: + assert False + + @pytest.mark.xfail(strict=True) + def test_xpass() -> None: + assert True + + record, dom = run_and_parse( + fixture, + test_pass, + test_fail, + test_error, + test_xfail, + test_xpass, + family=xunit_family, ) - result, dom = run_and_parse(family=xunit_family) - assert result.ret + record.assert_outcomes( + passed=1, failed=2, errors=1, xfailed=1 + ) # strict xpass is a failure node = dom.get_first_by_tag("testsuite") node.assert_attr(name="pytest", errors=1, failures=2, skipped=1, tests=5) @parametrize_families def test_hostname_in_xml( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - def test_pass(): - pass - """ - ) - _result, dom = run_and_parse(family=xunit_family) + def test_pass() -> None: + pass + + _record, dom = run_and_parse(test_pass, family=xunit_family) node = dom.get_first_by_tag("testsuite") node.assert_attr(hostname=platform.node()) @parametrize_families def test_timestamp_in_xml( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - def test_pass(): - pass - """ - ) + def test_pass() -> None: + pass + start_time = datetime.now(timezone.utc) - _result, dom = run_and_parse(family=xunit_family) + _record, dom = run_and_parse(test_pass, family=xunit_family) node = dom.get_first_by_tag("testsuite") timestamp = datetime.fromisoformat(node["timestamp"]) assert start_time <= timestamp < datetime.now(timezone.utc) def test_timing_function( self, - pytester: Pytester, run_and_parse: RunAndParse, mock_timing: _pytest.timing.MockTiming, ) -> None: - pytester.makepyfile( - """ - from _pytest import timing - def setup_module(): - timing.sleep(1) - def teardown_module(): - timing.sleep(2) - def test_sleep(): - timing.sleep(4) - """ + from _pytest import timing + + def setup_module() -> None: + timing.sleep(1) + + def teardown_module() -> None: + timing.sleep(2) + + def test_sleep() -> None: + timing.sleep(4) + + _record, dom = run_and_parse( + build_module( + "test_timing_function", setup_module, teardown_module, test_sleep + ) ) - _result, dom = run_and_parse() node = dom.get_first_by_tag("testsuite") tnode = node.get_first_by_tag("testcase") val = tnode["time"] @@ -320,7 +414,6 @@ def test_sleep(): @pytest.mark.parametrize("duration_report", ["call", "total"]) def test_junit_duration_report( self, - pytester: Pytester, monkeypatch: MonkeyPatch, duration_report: _JunitDurationReport, run_and_parse: RunAndParse, @@ -335,13 +428,12 @@ def node_reporter_wrapper(s: Any, report: TestReport) -> Any: monkeypatch.setattr(LogXML, "node_reporter", node_reporter_wrapper) - pytester.makepyfile( - """ - def test_foo(): - pass - """ + def test_foo() -> None: + pass + + _record, dom = run_and_parse( + test_foo, args=("-o", f"junit_duration_report={duration_report}") ) - _result, dom = run_and_parse("-o", f"junit_duration_report={duration_report}") node = dom.get_first_by_tag("testsuite") tnode = node.get_first_by_tag("testcase") val = float(tnode["time"]) @@ -353,21 +445,20 @@ def test_foo(): @parametrize_families def test_setup_error( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest + @pytest.fixture + def arg(request: pytest.FixtureRequest) -> None: + raise ValueError("Error reason") - @pytest.fixture - def arg(request): - raise ValueError("Error reason") - def test_function(arg): - pass - """ + def test_function(arg: None) -> None: + pass + + record, dom = run_and_parse( + arg, test_function, name="test_setup_error", family=xunit_family ) - result, dom = run_and_parse(family=xunit_family) - assert result.ret + # A fixture blowing up at setup is an error, not a failure. + record.assert_outcomes(errors=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) tnode = node.get_first_by_tag("testcase") @@ -378,22 +469,21 @@ def test_function(arg): @parametrize_families def test_teardown_error( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest + @pytest.fixture + def arg() -> Generator[None]: + yield + raise ValueError("Error reason") - @pytest.fixture - def arg(): - yield - raise ValueError('Error reason') - def test_function(arg): - pass - """ + def test_function(arg: None) -> None: + pass + + record, dom = run_and_parse( + arg, test_function, name="test_teardown_error", family=xunit_family ) - result, dom = run_and_parse(family=xunit_family) - assert result.ret + # The call passed; the teardown is what errored. + record.assert_outcomes(passed=1, errors=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) tnode = node.get_first_by_tag("testcase") @@ -404,22 +494,18 @@ def test_function(arg): @parametrize_families def test_call_failure_teardown_error( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest + @pytest.fixture + def arg() -> Generator[None]: + yield + raise Exception("Teardown Exception") - @pytest.fixture - def arg(): - yield - raise Exception("Teardown Exception") - def test_function(arg): - raise Exception("Call Exception") - """ - ) - result, dom = run_and_parse(family=xunit_family) - assert result.ret + def test_function(arg: None) -> None: + raise Exception("Call Exception") + + record, dom = run_and_parse(arg, test_function, family=xunit_family) + record.assert_outcomes(failed=1, errors=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=1, failures=1, tests=2) first, second = dom.find_by_tag("testcase") @@ -435,17 +521,15 @@ def test_function(arg): @parametrize_families def test_skip_contains_name_reason( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest - def test_skip(): - pytest.skip("hello23") - """ + def test_skip() -> None: + pytest.skip("hello23") + + record, dom = run_and_parse( + test_skip, name="test_skip_contains_name_reason", family=xunit_family ) - result, dom = run_and_parse(family=xunit_family) - assert result.ret == 0 + record.assert_outcomes(skipped=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=1) tnode = node.get_first_by_tag("testcase") @@ -455,18 +539,16 @@ def test_skip(): @parametrize_families def test_mark_skip_contains_name_reason( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip(reason="hello24") - def test_skip(): - assert True - """ + @pytest.mark.skip(reason="hello24") + def test_skip() -> None: + assert True + + record, dom = run_and_parse( + test_skip, name="test_mark_skip_contains_name_reason", family=xunit_family ) - result, dom = run_and_parse(family=xunit_family) - assert result.ret == 0 + record.assert_outcomes(skipped=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=1) tnode = node.get_first_by_tag("testcase") @@ -478,19 +560,20 @@ def test_skip(): @parametrize_families def test_mark_skipif_contains_name_reason( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest - GLOBAL_CONDITION = True - @pytest.mark.skipif(GLOBAL_CONDITION, reason="hello25") - def test_skip(): - assert True - """ + # The module global of the original is a closure variable here; the + # mark stores the evaluated condition either way. + global_condition = True + + @pytest.mark.skipif(global_condition, reason="hello25") + def test_skip() -> None: + assert True + + record, dom = run_and_parse( + test_skip, name="test_mark_skipif_contains_name_reason", family=xunit_family ) - result, dom = run_and_parse(family=xunit_family) - assert result.ret == 0 + record.assert_outcomes(skipped=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=1) tnode = node.get_first_by_tag("testcase") @@ -500,9 +583,15 @@ def test_skip(): snode = tnode.get_first_by_tag("skipped") snode.assert_attr(type="pytest.skip", message="hello25") + # ensemble: asserts that captured output is *absent*; without item-level + # capture in an ensemble nothing is ever captured and the assertion would + # hold for the wrong reason. @parametrize_families def test_mark_skip_doesnt_capture_output( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, + pytester: Pytester, + run_and_parse_pytester: RunAndParsePytester, + xunit_family: _JunitFamily, ) -> None: pytester.makepyfile( """ @@ -512,24 +601,23 @@ def test_skip(): print("bar!") """ ) - result, dom = run_and_parse(family=xunit_family) + result, dom = run_and_parse_pytester(family=xunit_family) assert result.ret == 0 node_xml = dom.get_first_by_tag("testsuite").toxml() assert "bar!" not in node_xml @parametrize_families def test_classname_instance( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - class TestClass(object): - def test_method(self): - assert 0 - """ + class TestClass: + def test_method(self) -> None: + assert 0 + + record, dom = run_and_parse( + TestClass, name="test_classname_instance", family=xunit_family ) - result, dom = run_and_parse(family=xunit_family) - assert result.ret + record.assert_outcomes(failed=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(failures=1) tnode = node.get_first_by_tag("testcase") @@ -539,24 +627,34 @@ def test_method(self): @parametrize_families def test_classname_nested_dir( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, tmp_path: Path, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - p = pytester.mkdir("sub").joinpath("test_hello.py") + # The classname comes from the nodeid, so the module has to genuinely + # live in a subdirectory of the rootdir: written to disk and imported + # with module_from_path, rather than synthesized. + sub = tmp_path.joinpath("sub") + sub.mkdir() + p = sub.joinpath("test_hello.py") p.write_text("def test_func(): 0/0", encoding="utf-8") - result, dom = run_and_parse(family=xunit_family) - assert result.ret + record, dom = run_and_parse(module_from_path(p), family=xunit_family) + record.assert_outcomes(failed=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(failures=1) tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="sub.test_hello", name="test_func") + # ensemble: an internal error is reported by wrap_session, which an + # ensemble does not run - the exception escapes the run instead. @parametrize_families def test_internal_error( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, + pytester: Pytester, + run_and_parse_pytester: RunAndParsePytester, + xunit_family: _JunitFamily, ) -> None: pytester.makeconftest("def pytest_runtest_protocol(): 0 / 0") pytester.makepyfile("def test_function(): pass") - result, dom = run_and_parse(family=xunit_family) + result, dom = run_and_parse_pytester(family=xunit_family) assert result.ret node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) @@ -566,6 +664,9 @@ def test_internal_error( fnode.assert_attr(message="internal error") assert "Division" in fnode.toxml() + # ensemble: the system-out/system-err sections come from the item-level + # capture an ensemble has no way to start (the CaptureManager is created + # in pytest_load_initial_conftests, which an ensemble never runs). @pytest.mark.parametrize( "junit_logging", ["no", "log", "system-out", "system-err", "out-err", "all"] ) @@ -574,7 +675,7 @@ def test_failure_function( self, pytester: Pytester, junit_logging: _JunitLogging, - run_and_parse: RunAndParse, + run_and_parse_pytester: RunAndParsePytester, xunit_family: _JunitFamily, ) -> None: pytester.makepyfile( @@ -591,7 +692,7 @@ def test_fail(): """ ) - result, dom = run_and_parse( + result, dom = run_and_parse_pytester( "-o", f"junit_logging={junit_logging}", family=xunit_family ) assert result.ret, "Expected ret > 0" @@ -639,24 +740,25 @@ def test_fail(): @parametrize_families def test_failure_verbose_message( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import sys - def test_fail(): - assert 0, "An error" - """ - ) - _result, dom = run_and_parse(family=xunit_family) + def test_fail() -> None: + assert 0, "An error" + + record, dom = run_and_parse(test_fail, family=xunit_family) + record.assert_outcomes(failed=1) node = dom.get_first_by_tag("testsuite") tnode = node.get_first_by_tag("testcase") fnode = tnode.get_first_by_tag("failure") fnode.assert_attr(message="AssertionError: An error\nassert 0") + # ensemble: asserts on the captured system-out of each parametrized item. @parametrize_families def test_failure_escape( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, + pytester: Pytester, + run_and_parse_pytester: RunAndParsePytester, + xunit_family: _JunitFamily, ) -> None: pytester.makepyfile( """ @@ -667,7 +769,7 @@ def test_func(arg1): assert 0 """ ) - result, dom = run_and_parse( + result, dom = run_and_parse_pytester( "-o", "junit_logging=system-out", family=xunit_family ) assert result.ret @@ -684,19 +786,23 @@ def test_func(arg1): @parametrize_families def test_junit_prefixing( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - def test_func(): - assert 0 - class TestHello(object): - def test_hello(self): - pass - """ + def test_func() -> None: + assert 0 + + class TestHello: + def test_hello(self) -> None: + pass + + record, dom = run_and_parse( + test_func, + TestHello, + name="test_junit_prefixing", + args=("--junitprefix=xyz",), + family=xunit_family, ) - result, dom = run_and_parse("--junitprefix=xyz", family=xunit_family) - assert result.ret + record.assert_outcomes(passed=1, failed=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(failures=1, tests=2) tnode = node.get_first_by_tag("testcase") @@ -708,17 +814,15 @@ def test_hello(self): @parametrize_families def test_xfailure_function( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest - def test_xfail(): - pytest.xfail("42") - """ + def test_xfail() -> None: + pytest.xfail("42") + + record, dom = run_and_parse( + test_xfail, name="test_xfailure_function", family=xunit_family ) - result, dom = run_and_parse(family=xunit_family) - assert not result.ret + record.assert_outcomes(xfailed=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=1, tests=1) tnode = node.get_first_by_tag("testcase") @@ -728,18 +832,16 @@ def test_xfail(): @parametrize_families def test_xfailure_marker( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail(reason="42") - def test_xfail(): - assert False - """ + @pytest.mark.xfail(reason="42") + def test_xfail() -> None: + assert False + + record, dom = run_and_parse( + test_xfail, name="test_xfailure_marker", family=xunit_family ) - result, dom = run_and_parse(family=xunit_family) - assert not result.ret + record.assert_outcomes(xfailed=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=1, tests=1) tnode = node.get_first_by_tag("testcase") @@ -747,6 +849,7 @@ def test_xfail(): fnode = tnode.get_first_by_tag("skipped") fnode.assert_attr(type="pytest.xfail", message="42") + # ensemble: counts the captured output sections of an item. @pytest.mark.parametrize( "junit_logging", ["no", "log", "system-out", "system-err", "out-err", "all"] ) @@ -754,7 +857,7 @@ def test_xfail_captures_output_once( self, pytester: Pytester, junit_logging: _JunitLogging, - run_and_parse: RunAndParse, + run_and_parse_pytester: RunAndParsePytester, ) -> None: pytester.makepyfile( """ @@ -768,7 +871,7 @@ def test_fail(): assert 0 """ ) - _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + _result, dom = run_and_parse_pytester("-o", f"junit_logging={junit_logging}") node = dom.get_first_by_tag("testsuite") tnode = node.get_first_by_tag("testcase") @@ -783,18 +886,16 @@ def test_fail(): @parametrize_families def test_xfailure_xpass( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail - def test_xpass(): - pass - """ + @pytest.mark.xfail + def test_xpass() -> None: + pass + + record, dom = run_and_parse( + test_xpass, name="test_xfailure_xpass", family=xunit_family ) - _result, dom = run_and_parse(family=xunit_family) - # assert result.ret + record.assert_outcomes(xpassed=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=0, tests=1) tnode = node.get_first_by_tag("testcase") @@ -802,18 +903,17 @@ def test_xpass(): @parametrize_families def test_xfailure_xpass_strict( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.xfail(strict=True, reason="This needs to fail!") - def test_xpass(): - pass - """ + @pytest.mark.xfail(strict=True, reason="This needs to fail!") + def test_xpass() -> None: + pass + + record, dom = run_and_parse( + test_xpass, name="test_xfailure_xpass_strict", family=xunit_family ) - _result, dom = run_and_parse(family=xunit_family) - # assert result.ret + # A strict xpass is a plain failure. + record.assert_outcomes(failed=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=0, tests=1) tnode = node.get_first_by_tag("testcase") @@ -821,12 +921,17 @@ def test_xpass(): fnode = tnode.get_first_by_tag("failure") fnode.assert_attr(message="[XPASS(strict)] This needs to fail!") + # ensemble: needs a module that fails at import time; ensemble sources are + # already-imported objects. @parametrize_families def test_collect_error( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, + pytester: Pytester, + run_and_parse_pytester: RunAndParsePytester, + xunit_family: _JunitFamily, ) -> None: pytester.makepyfile("syntax error") - result, dom = run_and_parse(family=xunit_family) + result, dom = run_and_parse_pytester(family=xunit_family) assert result.ret node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) @@ -835,44 +940,41 @@ def test_collect_error( fnode.assert_attr(message="collection failure") assert "SyntaxError" in fnode.toxml() - def test_unicode(self, pytester: Pytester, run_and_parse: RunAndParse) -> None: + def test_unicode(self, tmp_path: Path, run_and_parse: RunAndParse) -> None: value = "hx\xc4\x85\xc4\x87\n" - pytester.makepyfile( - f"""\ - # coding: latin1 - def test_hello(): - print({value!r}) - assert 0 - """ + # The latin1 coding cookie is the point of the test, so the module has + # to be a real file decoded by the import machinery, not a synthesized + # one whose source lines would come from this file. + p = tmp_path.joinpath("test_unicode.py") + p.write_text( + f"# coding: latin1\ndef test_hello():\n print({value!r})\n assert 0\n", + encoding="latin1", ) - result, dom = run_and_parse() - assert result.ret == 1 + record, dom = run_and_parse(module_from_path(p)) + record.assert_outcomes(failed=1) tnode = dom.get_first_by_tag("testcase") fnode = tnode.get_first_by_tag("failure") assert "hx" in fnode.toxml() - def test_assertion_binchars( - self, pytester: Pytester, run_and_parse: RunAndParse - ) -> None: + def test_assertion_binchars(self, run_and_parse: RunAndParse) -> None: """This test did fail when the escaping wasn't strict.""" - pytester.makepyfile( - """ + # Module globals in the original; closure variables here. + m1 = "\x01\x02\x03\x04" + m2 = "\x01\x02\x03\x05" - M1 = '\x01\x02\x03\x04' - M2 = '\x01\x02\x03\x05' + def test_str_compare() -> None: + assert m1 == m2 - def test_str_compare(): - assert M1 == M2 - """ - ) - _result, dom = run_and_parse() + record, dom = run_and_parse(test_str_compare) + record.assert_outcomes(failed=1) print(dom.toxml()) + # ensemble: needs item-level capture. @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) def test_pass_captures_stdout( self, pytester: Pytester, - run_and_parse: RunAndParse, + run_and_parse_pytester: RunAndParsePytester, junit_logging: _JunitLogging, ) -> None: pytester.makepyfile( @@ -881,7 +983,7 @@ def test_pass(): print('hello-stdout') """ ) - _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + _result, dom = run_and_parse_pytester("-o", f"junit_logging={junit_logging}") node = dom.get_first_by_tag("testsuite") pnode = node.get_first_by_tag("testcase") if junit_logging == "no": @@ -894,11 +996,12 @@ def test_pass(): "'hello-stdout' should be in system-out" ) + # ensemble: needs item-level capture. @pytest.mark.parametrize("junit_logging", ["no", "system-err"]) def test_pass_captures_stderr( self, pytester: Pytester, - run_and_parse: RunAndParse, + run_and_parse_pytester: RunAndParsePytester, junit_logging: _JunitLogging, ) -> None: pytester.makepyfile( @@ -908,7 +1011,7 @@ def test_pass(): sys.stderr.write('hello-stderr') """ ) - _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + _result, dom = run_and_parse_pytester("-o", f"junit_logging={junit_logging}") node = dom.get_first_by_tag("testsuite") pnode = node.get_first_by_tag("testcase") if junit_logging == "no": @@ -921,11 +1024,12 @@ def test_pass(): "'hello-stderr' should be in system-err" ) + # ensemble: needs item-level capture. @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) def test_setup_error_captures_stdout( self, pytester: Pytester, - run_and_parse: RunAndParse, + run_and_parse_pytester: RunAndParsePytester, junit_logging: _JunitLogging, ) -> None: pytester.makepyfile( @@ -940,7 +1044,7 @@ def test_function(arg): pass """ ) - _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + _result, dom = run_and_parse_pytester("-o", f"junit_logging={junit_logging}") node = dom.get_first_by_tag("testsuite") pnode = node.get_first_by_tag("testcase") if junit_logging == "no": @@ -953,11 +1057,12 @@ def test_function(arg): "'hello-stdout' should be in system-out" ) + # ensemble: needs item-level capture. @pytest.mark.parametrize("junit_logging", ["no", "system-err"]) def test_setup_error_captures_stderr( self, pytester: Pytester, - run_and_parse: RunAndParse, + run_and_parse_pytester: RunAndParsePytester, junit_logging: _JunitLogging, ) -> None: pytester.makepyfile( @@ -973,7 +1078,7 @@ def test_function(arg): pass """ ) - _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + _result, dom = run_and_parse_pytester("-o", f"junit_logging={junit_logging}") node = dom.get_first_by_tag("testsuite") pnode = node.get_first_by_tag("testcase") if junit_logging == "no": @@ -986,11 +1091,12 @@ def test_function(arg): "'hello-stderr' should be in system-err" ) + # ensemble: needs item-level capture. @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) def test_avoid_double_stdout( self, pytester: Pytester, - run_and_parse: RunAndParse, + run_and_parse_pytester: RunAndParsePytester, junit_logging: _JunitLogging, ) -> None: pytester.makepyfile( @@ -1007,7 +1113,7 @@ def test_function(arg): sys.stdout.write('hello-stdout call') """ ) - _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + _result, dom = run_and_parse_pytester("-o", f"junit_logging={junit_logging}") node = dom.get_first_by_tag("testsuite") pnode = node.get_first_by_tag("testcase") if junit_logging == "no": @@ -1059,9 +1165,14 @@ def getini(self, name: str) -> str: class TestNonPython: + # ensemble: collects a non-python file through pytest_collect_file; an + # ensemble serves preset collectors and never walks the filesystem. @parametrize_families def test_summing_simple( - self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + self, + pytester: Pytester, + run_and_parse_pytester: RunAndParsePytester, + xunit_family: _JunitFamily, ) -> None: pytester.makeconftest( """ @@ -1077,7 +1188,7 @@ def repr_failure(self, excinfo): """ ) pytester.path.joinpath("myfile.xyz").write_text("hello", encoding="utf-8") - result, dom = run_and_parse(family=xunit_family) + result, dom = run_and_parse_pytester(family=xunit_family) assert result.ret node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=0, failures=1, skipped=0, tests=1) @@ -1088,6 +1199,8 @@ def repr_failure(self, excinfo): assert "custom item runtest failed" in fnode.toxml() +# ensemble: needs item-level capture (the null byte reaches the xml through +# the captured stdout section). @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) def test_nullbyte(pytester: Pytester, junit_logging: _JunitLogging) -> None: # A null byte cannot occur in XML (see section 2.2 of the spec) @@ -1110,6 +1223,7 @@ def test_print_nullbyte(): assert "#x00" not in text +# ensemble: needs item-level capture. @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) def test_nullbyte_replace(pytester: Pytester, junit_logging: _JunitLogging) -> None: # Check if the null byte gets replaced @@ -1174,95 +1288,90 @@ def test_logxml_path_expansion(tmp_path: Path, monkeypatch: MonkeyPatch) -> None assert xml_var.logfile == str(home_var) -def test_logxml_changingdir(pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_func(): - import os - os.chdir("a") - """ +def test_logxml_changingdir(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + def test_func() -> None: + import os + + os.chdir("a") + + # The relative --junitxml path is resolved against the invocation cwd, so + # the ensemble has to be run from the rootdir like pytester would. + tmp_path.joinpath("a").mkdir() + monkeypatch.chdir(tmp_path) + spec = ConfigSpec(rootpath=tmp_path, args=("--junitxml=a/x.xml",)).with_plugins( + "junitxml" ) - pytester.mkdir("a") - result = pytester.runpytest("--junitxml=a/x.xml") - assert result.ret == 0 - assert pytester.path.joinpath("a/x.xml").exists() + record = run_tests(test_func, spec=spec) + record.assert_outcomes(passed=1) + assert tmp_path.joinpath("a/x.xml").exists() -def test_logxml_makedir(pytester: Pytester) -> None: +def test_logxml_makedir(tmp_path: Path) -> None: """--junitxml should automatically create directories for the xml file""" - pytester.makepyfile( - """ - def test_pass(): - pass - """ - ) - result = pytester.runpytest("--junitxml=path/to/results.xml") - assert result.ret == 0 - assert pytester.path.joinpath("path/to/results.xml").exists() + def test_pass() -> None: + pass + spec = ConfigSpec( + rootpath=tmp_path, + args=(f"--junitxml={tmp_path.joinpath('path/to/results.xml')}",), + ).with_plugins("junitxml") + record = run_tests(test_pass, spec=spec) + record.assert_outcomes(passed=1) + assert tmp_path.joinpath("path/to/results.xml").exists() + + +# ensemble: the UsageError is raised while the args are parsed, and +# `configured()` then masks it with a KeyError - its finally clause reads +# `config.stash[config_warnings_key]`, which is only set further down. def test_logxml_check_isdir(pytester: Pytester) -> None: """Give an error if --junit-xml is a directory (#2089)""" result = pytester.runpytest("--junit-xml=.") result.stderr.fnmatch_lines(["*--junitxml must be a filename*"]) -def test_escaped_parametrized_names_xml( - pytester: Pytester, run_and_parse: RunAndParse -) -> None: - pytester.makepyfile( - """\ - import pytest - @pytest.mark.parametrize('char', ["\\x00"]) - def test_func(char): - assert char - """ - ) - result, dom = run_and_parse() - assert result.ret == 0 +def test_escaped_parametrized_names_xml(run_and_parse: RunAndParse) -> None: + @pytest.mark.parametrize("char", ["\x00"]) + def test_func(char: str) -> None: + assert char + + record, dom = run_and_parse(test_func) + record.assert_outcomes(passed=1) node = dom.get_first_by_tag("testcase") node.assert_attr(name="test_func[\\x00]") -def test_double_colon_split_function_issue469( - pytester: Pytester, run_and_parse: RunAndParse -) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.parametrize('param', ["double::colon"]) - def test_func(param): - pass - """ +def test_double_colon_split_function_issue469(run_and_parse: RunAndParse) -> None: + @pytest.mark.parametrize("param", ["double::colon"]) + def test_func(param: str) -> None: + pass + + record, dom = run_and_parse( + test_func, name="test_double_colon_split_function_issue469" ) - result, dom = run_and_parse() - assert result.ret == 0 + record.assert_outcomes(passed=1) node = dom.get_first_by_tag("testcase") node.assert_attr(classname="test_double_colon_split_function_issue469") node.assert_attr(name="test_func[double::colon]") -def test_double_colon_split_method_issue469( - pytester: Pytester, run_and_parse: RunAndParse -) -> None: - pytester.makepyfile( - """ - import pytest - class TestClass(object): - @pytest.mark.parametrize('param', ["double::colon"]) - def test_func(self, param): - pass - """ +def test_double_colon_split_method_issue469(run_and_parse: RunAndParse) -> None: + class TestClass: + @pytest.mark.parametrize("param", ["double::colon"]) + def test_func(self, param: str) -> None: + pass + + record, dom = run_and_parse( + TestClass, name="test_double_colon_split_method_issue469" ) - result, dom = run_and_parse() - assert result.ret == 0 + record.assert_outcomes(passed=1) node = dom.get_first_by_tag("testcase") node.assert_attr(classname="test_double_colon_split_method_issue469.TestClass") node.assert_attr(name="test_func[double::colon]") -def test_unicode_issue368(pytester: Pytester) -> None: - path = pytester.path.joinpath("test.xml") +def test_unicode_issue368(tmp_path: Path) -> None: + path = tmp_path.joinpath("test.xml") log = LogXML(str(path), None) ustr = "ВНИ!" @@ -1291,47 +1400,39 @@ class Report(BaseReport): log.pytest_sessionfinish() -def test_record_property(pytester: Pytester, run_and_parse: RunAndParse) -> None: - pytester.makepyfile( - """ - import pytest +def test_record_property(run_and_parse: RunAndParse) -> None: + @pytest.fixture + def other(record_property: RecordFunc) -> None: + record_property("bar", 1) - @pytest.fixture - def other(record_property): - record_property("bar", 1) - def test_record(record_property, other): - record_property("foo", "<1"); - """ - ) - result, dom = run_and_parse() + def test_record(record_property: RecordFunc, other: None) -> None: + record_property("foo", "<1") + + record, dom = run_and_parse(other, test_record) node = dom.get_first_by_tag("testsuite") tnode = node.get_first_by_tag("testcase") psnode = tnode.get_first_by_tag("properties") pnodes = psnode.find_by_tag("property") pnodes[0].assert_attr(name="bar", value="1") pnodes[1].assert_attr(name="foo", value="<1") - result.stdout.fnmatch_lines(["*= 1 passed in *"]) + # was: result.stdout.fnmatch_lines(["*= 1 passed in *"]) + record.assert_outcomes(passed=1) def test_record_property_on_test_and_teardown_failure( - pytester: Pytester, run_and_parse: RunAndParse + run_and_parse: RunAndParse, ) -> None: - pytester.makepyfile( - """ - import pytest + @pytest.fixture + def other(record_property: RecordFunc) -> Generator[None]: + record_property("bar", 1) + yield + assert 0 - @pytest.fixture - def other(record_property): - record_property("bar", 1) - yield - assert 0 + def test_record(record_property: RecordFunc, other: None) -> None: + record_property("foo", "<1") + assert 0 - def test_record(record_property, other): - record_property("foo", "<1") - assert 0 - """ - ) - result, dom = run_and_parse() + record, dom = run_and_parse(other, test_record) node = dom.get_first_by_tag("testsuite") tnodes = node.find_by_tag("testcase") for tnode in tnodes: @@ -1340,20 +1441,18 @@ def test_record(record_property, other): pnodes = psnode.find_by_tag("property") pnodes[0].assert_attr(name="bar", value="1") pnodes[1].assert_attr(name="foo", value="<1") - result.stdout.fnmatch_lines(["*= 1 failed, 1 error *"]) + # was: result.stdout.fnmatch_lines(["*= 1 failed, 1 error *"]) + record.assert_outcomes(failed=1, errors=1) -def test_record_property_same_name( - pytester: Pytester, run_and_parse: RunAndParse -) -> None: - pytester.makepyfile( - """ - def test_record_with_same_name(record_property): - record_property("foo", "bar") - record_property("foo", "baz") - """ - ) - _result, dom = run_and_parse() +def test_record_property_same_name(run_and_parse: RunAndParse) -> None: + def test_record_with_same_name( + record_property: RecordFunc, + ) -> None: + record_property("foo", "bar") + record_property("foo", "baz") + + _record, dom = run_and_parse(test_record_with_same_name) node = dom.get_first_by_tag("testsuite") tnode = node.get_first_by_tag("testcase") psnode = tnode.get_first_by_tag("properties") @@ -1362,88 +1461,124 @@ def test_record_with_same_name(record_property): pnodes[1].assert_attr(name="foo", value="baz") +def _record_property_test() -> Callable[..., None]: + def test_record(record_property: RecordFunc) -> None: + record_property("foo", "bar") + + return test_record + + +def _record_xml_attribute_test() -> Callable[..., None]: + def test_record(record_xml_attribute: RecordFunc) -> None: + record_xml_attribute("foo", "bar") + + return test_record + + +#: The two record fixtures, as sources requesting them by real parameter name. +#: A source requests a fixture through its own signature, so the two variants +#: the original built by string formatting are two real functions here. +RECORD_FIXTURE_TESTS = { + "record_property": _record_property_test, + "record_xml_attribute": _record_xml_attribute_test, +} + + +def _record_property_pair() -> tuple[Callable[..., None], Callable[..., None]]: + @pytest.fixture + def other(record_property: RecordFunc) -> None: + record_property("bar", 1) + + def test_record(record_property: RecordFunc, other: None) -> None: + record_property("foo", "<1") + + return other, test_record + + +def _record_xml_attribute_pair() -> tuple[Callable[..., None], Callable[..., None]]: + @pytest.fixture + def other(record_xml_attribute: RecordFunc) -> None: + record_xml_attribute("bar", 1) + + def test_record(record_xml_attribute: RecordFunc, other: None) -> None: + record_xml_attribute("foo", "<1") + + return other, test_record + + +#: The same two, as the fixture/test pair of ``test_record_fixtures_xunit2``. +RECORD_FIXTURE_PAIRS = { + "record_property": _record_property_pair, + "record_xml_attribute": _record_xml_attribute_pair, +} + + @pytest.mark.parametrize("fixture_name", ["record_property", "record_xml_attribute"]) -def test_record_fixtures_without_junitxml( - pytester: Pytester, fixture_name: str -) -> None: - pytester.makepyfile( - f""" - def test_record({fixture_name}): - {fixture_name}("foo", "bar") - """ - ) - result = pytester.runpytest() - assert result.ret == 0 +def test_record_fixtures_without_junitxml(tmp_path: Path, fixture_name: str) -> None: + test_record = RECORD_FIXTURE_TESTS[fixture_name]() + spec = ConfigSpec(rootpath=tmp_path).with_plugins("junitxml") + record = run_tests(test_record, spec=spec) + record.assert_outcomes(passed=1) -@pytest.mark.filterwarnings("default") -def test_record_attribute(pytester: Pytester, run_and_parse: RunAndParse) -> None: - pytester.makeini( - """ - [pytest] - junit_family = xunit1 - """ - ) - pytester.makepyfile( - """ - import pytest - @pytest.fixture - def other(record_xml_attribute): - record_xml_attribute("bar", 1) - def test_record(record_xml_attribute, other): - record_xml_attribute("foo", "<1"); - """ +def test_record_attribute(run_and_parse: RunAndParse) -> None: + @pytest.fixture + def other(record_xml_attribute: RecordFunc) -> None: + record_xml_attribute("bar", 1) + + def test_record(record_xml_attribute: RecordFunc, other: None) -> None: + record_xml_attribute("foo", "<1") + + record, dom = run_and_parse( + other, + test_record, + family=None, + # "always" is the ensemble's stand-in for the host-level + # `@pytest.mark.filterwarnings("default")` of the original: process + # global filters are inherited, and this suite both errors on + # warnings and ignores PytestExperimentalApiWarning. + inicfg={"junit_family": "xunit1", "filterwarnings": ["always"]}, ) - result, dom = run_and_parse() node = dom.get_first_by_tag("testsuite") tnode = node.get_first_by_tag("testcase") tnode.assert_attr(bar="1") tnode.assert_attr(foo="<1") - result.stdout.fnmatch_lines( - ["*test_record_attribute.py:6:*record_xml_attribute is an experimental feature"] - ) + # The rendered warning quoted the generated file and line, which for an + # ensemble source would be this file; the recorded warning is the same + # warning, asserted as an object rather than as a line of output. + assert [str(w.message) for w in record.warnings] == [ + "record_xml_attribute is an experimental feature" + ] -@pytest.mark.filterwarnings("default") @pytest.mark.parametrize("fixture_name", ["record_xml_attribute", "record_property"]) -def test_record_fixtures_xunit2( - pytester: Pytester, fixture_name: str, run_and_parse: RunAndParse -) -> None: +def test_record_fixtures_xunit2(fixture_name: str, run_and_parse: RunAndParse) -> None: """Ensure record_xml_attribute and record_property drop values when outside of legacy family.""" - pytester.makeini( - """ - [pytest] - junit_family = xunit2 - """ - ) - pytester.makepyfile( - f""" - import pytest + other, test_record = RECORD_FIXTURE_PAIRS[fixture_name]() - @pytest.fixture - def other({fixture_name}): - {fixture_name}("bar", 1) - def test_record({fixture_name}, other): - {fixture_name}("foo", "<1"); - """ + record, _dom = run_and_parse( + other, + test_record, + family=None, + inicfg={"junit_family": "xunit2", "filterwarnings": ["always"]}, ) - - result, _dom = run_and_parse(family=None) - expected_lines = [] - if fixture_name == "record_xml_attribute": - expected_lines.append( - "*test_record_fixtures_xunit2.py:6:*record_xml_attribute is an experimental feature" - ) - expected_lines = [ - f"*test_record_fixtures_xunit2.py:6:*{fixture_name} is incompatible " + expected = [ + f"{fixture_name} is incompatible " "with junit_family 'xunit2' (use 'legacy' or 'xunit1')" ] - result.stdout.fnmatch_lines(expected_lines) + if fixture_name == "record_xml_attribute": + expected.insert(0, "record_xml_attribute is an experimental feature") + # The original only ever asserted the last line it built; both warnings + # are checked here. + assert [str(w.message) for w in record.warnings] == expected +# ensemble: xdist. def test_random_report_log_xdist( - pytester: Pytester, monkeypatch: MonkeyPatch, run_and_parse: RunAndParse + pytester: Pytester, + monkeypatch: MonkeyPatch, + run_and_parse_pytester: RunAndParsePytester, ) -> None: """`xdist` calls pytest_runtest_logreport as they are executed by the workers, with nodes from several nodes overlapping, so junitxml must cope with that @@ -1458,7 +1593,7 @@ def test_x(i): assert i != 22 """ ) - _, dom = run_and_parse("-n2") + _, dom = run_and_parse_pytester("-n2") suite_node = dom.get_first_by_tag("testsuite") failed = [] for case_node in suite_node.find_by_tag("testcase"): @@ -1470,15 +1605,13 @@ def test_x(i): @parametrize_families def test_root_testsuites_tag( - pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - def test_x(): - pass - """ - ) - _, dom = run_and_parse(family=xunit_family) + def test_x() -> None: + pass + + record, dom = run_and_parse(test_x, family=xunit_family) + record.assert_outcomes(passed=1) root = dom.get_unique_child assert root.tag == "testsuites" root.assert_attr(name="pytest tests") @@ -1486,22 +1619,27 @@ def test_x(): assert suite_node.tag == "testsuite" -def test_runs_twice(pytester: Pytester, run_and_parse: RunAndParse) -> None: - f = pytester.makepyfile( - """ - def test_pass(): - pass - """ - ) +def test_runs_twice(run_and_parse: RunAndParse) -> None: + def test_pass() -> None: + pass - result, dom = run_and_parse("--keep-duplicates", f, f) - result.stdout.no_fnmatch_line("*INTERNALERROR*") + # `--keep-duplicates` plus the same file twice is how the original got one + # module collected twice; handing the same module object over twice is the + # ensemble equivalent, and produces the same duplicated nodeids that + # junitxml has to cope with. + module = build_module("test_runs_twice", test_pass) + record, dom = run_and_parse(module, module) + # was: result.stdout.no_fnmatch_line("*INTERNALERROR*") + record.assert_outcomes(passed=2) first, second = (x["classname"] for x in dom.find_by_tag("testcase")) assert first == second +# ensemble: xdist. def test_runs_twice_xdist( - pytester: Pytester, monkeypatch: MonkeyPatch, run_and_parse: RunAndParse + pytester: Pytester, + monkeypatch: MonkeyPatch, + run_and_parse_pytester: RunAndParsePytester, ) -> None: pytest.importorskip("xdist") monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") @@ -1512,13 +1650,17 @@ def test_pass(): """ ) - result, dom = run_and_parse(f, "--dist", "each", "--tx", "2*popen") + result, dom = run_and_parse_pytester(f, "--dist", "each", "--tx", "2*popen") result.stdout.no_fnmatch_line("*INTERNALERROR*") first, second = (x["classname"] for x in dom.find_by_tag("testcase")) assert first == second -def test_fancy_items_regression(pytester: Pytester, run_and_parse: RunAndParse) -> None: +# ensemble: custom File/Item collectors served from pytest_collect_file, which +# needs the filesystem collection an ensemble replaces. +def test_fancy_items_regression( + pytester: Pytester, run_and_parse_pytester: RunAndParsePytester +) -> None: # issue 1259 pytester.makeconftest( """ @@ -1551,7 +1693,7 @@ def test_pass(): """ ) - result, dom = run_and_parse() + result, dom = run_and_parse_pytester() result.stdout.no_fnmatch_line("*INTERNALERROR*") @@ -1576,8 +1718,8 @@ def test_pass(): @parametrize_families -def test_global_properties(pytester: Pytester, xunit_family: _JunitFamily) -> None: - path = pytester.path.joinpath("test_global_properties.xml") +def test_global_properties(tmp_path: Path, xunit_family: _JunitFamily) -> None: + path = tmp_path.joinpath("test_global_properties.xml") log = LogXML(str(path), None, family=xunit_family) class Report(BaseReport): @@ -1610,9 +1752,9 @@ class Report(BaseReport): assert actual == expected -def test_url_property(pytester: Pytester) -> None: +def test_url_property(tmp_path: Path) -> None: test_url = "http://www.github.com/pytest-dev" - path = pytester.path.joinpath("test_url_property.xml") + path = tmp_path.joinpath("test_url_property.xml") log = LogXML(str(path), None) class Report(BaseReport): @@ -1638,19 +1780,16 @@ class Report(BaseReport): @parametrize_families def test_record_testsuite_property( - pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + run_and_parse: RunAndParse, xunit_family: _JunitFamily ) -> None: - pytester.makepyfile( - """ - def test_func1(record_testsuite_property): - record_testsuite_property("stats", "all good") + def test_func1(record_testsuite_property: RecordFunc) -> None: + record_testsuite_property("stats", "all good") - def test_func2(record_testsuite_property): - record_testsuite_property("stats", 10) - """ - ) - result, dom = run_and_parse(family=xunit_family) - assert result.ret == 0 + def test_func2(record_testsuite_property: RecordFunc) -> None: + record_testsuite_property("stats", 10) + + record, dom = run_and_parse(test_func1, test_func2, family=xunit_family) + record.assert_outcomes(passed=2) node = dom.get_first_by_tag("testsuite") properties_node = node.get_first_by_tag("properties") p1_node, p2_node = properties_node.find_by_tag( @@ -1660,31 +1799,25 @@ def test_func2(record_testsuite_property): p2_node.assert_attr(name="stats", value="10") -def test_record_testsuite_property_junit_disabled(pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_func1(record_testsuite_property): - record_testsuite_property("stats", "all good") - """ - ) - result = pytester.runpytest() - assert result.ret == 0 +def test_record_testsuite_property_junit_disabled(tmp_path: Path) -> None: + def test_func1(record_testsuite_property: RecordFunc) -> None: + record_testsuite_property("stats", "all good") + + spec = ConfigSpec(rootpath=tmp_path).with_plugins("junitxml") + record = run_tests(test_func1, spec=spec) + record.assert_outcomes(passed=1) @pytest.mark.parametrize("junit", [True, False]) -def test_record_testsuite_property_type_checking( - pytester: Pytester, junit: bool -) -> None: - pytester.makepyfile( - """ - def test_func1(record_testsuite_property): - record_testsuite_property(1, 2) - """ - ) - args = ("--junitxml=tests.xml",) if junit else () - result = pytester.runpytest(*args) - assert result.ret == 1 - result.stdout.fnmatch_lines( +def test_record_testsuite_property_type_checking(tmp_path: Path, junit: bool) -> None: + def test_func1(record_testsuite_property: RecordFunc) -> None: + record_testsuite_property(1, 2) # type: ignore[arg-type] + + args = (f"--junitxml={tmp_path.joinpath('tests.xml')}",) if junit else () + spec = ConfigSpec(rootpath=tmp_path, args=args).with_plugins("junitxml") + record = run_tests(test_func1, spec=spec, capture_output=True) + record.assert_outcomes(failed=1) + record.stdout.fnmatch_lines( ["*TypeError: name parameter needs to be a string, but int given"] ) @@ -1692,96 +1825,79 @@ def test_func1(record_testsuite_property): @pytest.mark.parametrize("suite_name", ["my_suite", ""]) @parametrize_families def test_set_suite_name( - pytester: Pytester, suite_name: str, run_and_parse: RunAndParse, xunit_family: _JunitFamily, ) -> None: + inicfg: dict[str, object] = {} if suite_name: - pytester.makeini( - f""" - [pytest] - junit_suite_name={suite_name} - junit_family={xunit_family} - """ - ) + inicfg = {"junit_suite_name": suite_name, "junit_family": xunit_family} expected = suite_name else: expected = "pytest" - pytester.makepyfile( - """ - import pytest - def test_func(): - pass - """ + def test_func() -> None: + pass + + record, dom = run_and_parse( + test_func, inicfg=inicfg, family=xunit_family, suite_name=expected ) - result, dom = run_and_parse(family=xunit_family, suite_name=expected) - assert result.ret == 0 + record.assert_outcomes(passed=1) node = dom.get_first_by_tag("testsuite") node.assert_attr(name=expected) -def test_escaped_skipreason_issue3533( - pytester: Pytester, run_and_parse: RunAndParse -) -> None: - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip(reason='1 <> 2') - def test_skip(): - pass - """ - ) - _, dom = run_and_parse() +def test_escaped_skipreason_issue3533(run_and_parse: RunAndParse) -> None: + @pytest.mark.skip(reason="1 <> 2") + def test_skip() -> None: + pass + + record, dom = run_and_parse(test_skip) + record.assert_outcomes(skipped=1) node = dom.get_first_by_tag("testcase") snode = node.get_first_by_tag("skipped") assert "1 <> 2" in snode.text snode.assert_attr(message="1 <> 2") -def test_bin_escaped_skipreason(pytester: Pytester, run_and_parse: RunAndParse) -> None: +def test_bin_escaped_skipreason(run_and_parse: RunAndParse) -> None: """Escape special characters from mark.skip reason (#11842).""" - pytester.makepyfile( - """ - import pytest - @pytest.mark.skip("\33[31;1mred\33[0m") - def test_skip(): - pass - """ - ) - _, dom = run_and_parse() + + @pytest.mark.skip("\33[31;1mred\33[0m") + def test_skip() -> None: + pass + + record, dom = run_and_parse(test_skip) + record.assert_outcomes(skipped=1) node = dom.get_first_by_tag("testcase") snode = node.get_first_by_tag("skipped") assert "#x1B[31;1mred#x1B[0m" in snode.text snode.assert_attr(message="#x1B[31;1mred#x1B[0m") -def test_escaped_setup_teardown_error( - pytester: Pytester, run_and_parse: RunAndParse -) -> None: - pytester.makepyfile( - """ - import pytest +def test_escaped_setup_teardown_error(run_and_parse: RunAndParse) -> None: + @pytest.fixture + def my_setup() -> None: + raise Exception("error: \033[31mred\033[m") - @pytest.fixture() - def my_setup(): - raise Exception("error: \033[31mred\033[m") + def test_esc(my_setup: None) -> None: + pass - def test_esc(my_setup): - pass - """ - ) - _, dom = run_and_parse() + record, dom = run_and_parse(my_setup, test_esc) + record.assert_outcomes(errors=1) node = dom.get_first_by_tag("testcase") snode = node.get_first_by_tag("error") assert "#x1B[31mred#x1B[m" in snode["message"] assert "#x1B[31mred#x1B[m" in snode.text +# ensemble: junit_log_passing_tests only shows in the presence of item-level +# capture; with nothing captured the assertions would hold vacuously. @parametrize_families def test_logging_passing_tests_disabled_does_not_log_test_output( - pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily + pytester: Pytester, + run_and_parse_pytester: RunAndParsePytester, + xunit_family: _JunitFamily, ) -> None: pytester.makeini( f""" @@ -1803,19 +1919,20 @@ def test_func(): logging.warning('hello') """ ) - result, dom = run_and_parse(family=xunit_family) + result, dom = run_and_parse_pytester(family=xunit_family) assert result.ret == 0 node = dom.get_first_by_tag("testcase") assert len(node.find_by_tag("system-err")) == 0 assert len(node.find_by_tag("system-out")) == 0 +# ensemble: needs item-level capture. @parametrize_families @pytest.mark.parametrize("junit_logging", ["no", "system-out", "system-err"]) def test_logging_passing_tests_disabled_logs_output_for_failing_test_issue5430( pytester: Pytester, junit_logging: _JunitLogging, - run_and_parse: RunAndParse, + run_and_parse_pytester: RunAndParsePytester, xunit_family: _JunitFamily, ) -> None: pytester.makeini( @@ -1836,7 +1953,7 @@ def test_func(): assert 0 """ ) - result, dom = run_and_parse( + result, dom = run_and_parse_pytester( "-o", f"junit_logging={junit_logging}", family=xunit_family ) assert result.ret == 1 @@ -1853,14 +1970,24 @@ def test_func(): assert len(node.find_by_tag("system-out")) == 0 -def test_no_message_quiet(pytester: Pytester) -> None: +def test_no_message_quiet(tmp_path: Path) -> None: """Do not show the summary banner when --quiet is given (#13700).""" - pytester.makepyfile("def test(): pass") - result = pytester.runpytest("--junitxml=pytest.xml") - result.stdout.fnmatch_lines("* generated xml file: *") - result = pytester.runpytest("--junitxml=pytest.xml", "--quiet") - result.stdout.no_fnmatch_line("* generated xml file: *") + def test() -> None: + pass + + xml = tmp_path.joinpath("pytest.xml") + spec = ConfigSpec(rootpath=tmp_path, args=(f"--junitxml={xml}",)).with_plugins( + "junitxml" + ) + record = run_tests(test, spec=spec, capture_output=True) + record.stdout.fnmatch_lines(["* generated xml file: *"]) + + spec = ConfigSpec( + rootpath=tmp_path, args=(f"--junitxml={xml}", "--quiet") + ).with_plugins("junitxml") + record = run_tests(test, spec=spec, capture_output=True) + record.stdout.no_fnmatch_line("* generated xml file: *") @pytest.mark.parametrize( @@ -1871,16 +1998,17 @@ def test_no_message_quiet(pytester: Pytester) -> None: ("junit_family", "xunit3"), ], ) -def test_invalid_junit_option_value(pytester: Pytester, name: str, value: str) -> None: +def test_invalid_junit_option_value(tmp_path: Path, name: str, value: str) -> None: """Invalid junit option values fail with a clean usage error.""" - pytester.makeini( - f""" - [pytest] - {name} = {value} - """ - ) - result = pytester.runpytest("--junitxml=junit.xml") - assert result.ret == pytest.ExitCode.USAGE_ERROR - result.stderr.fnmatch_lines( - [f"*ERROR: *config option '{name}' expects one of *, got '{value}'"] - ) + spec = ConfigSpec( + rootpath=tmp_path, + args=(f"--junitxml={tmp_path.joinpath('junit.xml')}",), + inicfg={name: value}, + ).with_plugins("junitxml") + # pytester saw the usage error rendered on stderr and the USAGE_ERROR exit + # code; the ensemble sees the UsageError itself, raised while junitxml is + # being configured. + with pytest.raises( + UsageError, match=f"config option '{name}' expects one of .*, got '{value}'" + ): + run_tests(spec=spec) From f29b767781c86a3d9c0fbe6dae266a577d0c2b2f Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 11:06:13 +0200 Subject: [PATCH 22/30] testing: run assert-rewrite tests through the rewriter directly The rewriter's output is a pure function of source and config: the message an ``assert`` produces is fully determined by ``rewrite_asserts`` before any reporter sees it -- ``_format_explanation`` runs inside the rewritten ``raise``, so ``str(exc)`` is already the rendered message, escaping and all. Tests that fnmatched ``*AssertionError: ...*`` on terminal output were paying for a session to read a string the rewriter had already decided. Convert 34 such tests to the file's existing direct path, and add ``getmsg_src`` for sources that need module-level statements -- imports, class definitions, a ``PYTEST_DONT_REWRITE`` docstring, a module-level ``assert``. It execs the module and then calls every ``test_*`` it defines, in definition order, in the module's own namespace, which is what the walrus-leak regressions need: they are about state leaking through module-level ``@py_assert`` temporaries from one test function to the next. The 45 remaining pytester tests are tagged with why they stay. The line falls in three places, and none of them are the rewriter's output: * who decides to rewrite -- the import hook, ``register_assert_rewrite``, ``python_files``, ``--assert=plain``. Note ``PYTEST_DONT_REWRITE`` is checked in both ``AssertionRewriter.run`` and the hook's ``_should_rewrite``; the direct path covers only the former. * where the result is stored -- pyc headers, cache dirs, marshalling. * what config the rewriter reads -- ``enable_assertion_pass_hook`` at rewrite time, ``util._config`` at run time, both session-installed. Verbosity-threshold tests are left alone for that last reason: the message is rewriter output, but the threshold arrives via ``util._config`` and stubbing it would leave the wiring untested. Co-Authored-By: Claude Opus 5 (1M context) --- testing/test_assertrewrite.py | 503 +++++++++++++++++----------------- 1 file changed, 251 insertions(+), 252 deletions(-) diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py index 9740bf3c05e..3eaf5ff922e 100644 --- a/testing/test_assertrewrite.py +++ b/testing/test_assertrewrite.py @@ -46,6 +46,14 @@ def rewrite(src: str) -> ast.Module: return tree +def _assertion_message(exc: BaseException) -> str: + """Render a raised ``AssertionError`` the way a failure report would.""" + s = str(exc) + if not s.startswith("assert"): + return "AssertionError: " + s + return s + + def getmsg( f, extra_ns: Mapping[str, object] | None = None, *, must_pass: bool = False ) -> str | None: @@ -60,19 +68,56 @@ def getmsg( func = ns[f.__name__] try: func() # type: ignore[operator] - except AssertionError: + except AssertionError as exc: if must_pass: pytest.fail("shouldn't have raised") - s = str(sys.exc_info()[1]) - if not s.startswith("assert"): - return "AssertionError: " + s - return s + return _assertion_message(exc) else: if not must_pass: pytest.fail("function didn't raise at all") return None +def getmsg_src( + src: str, + extra_ns: Mapping[str, object] | None = None, + *, + must_pass: bool = False, +) -> str | None: + """Rewrite a whole module source, exec it, and get the failure message. + + The module-level counterpart of :func:`getmsg`, for sources that need + module-level statements: imports, class/function definitions, a module + docstring (``PYTEST_DONT_REWRITE``), or a module-level ``assert``. + + After the module body has run, every ``test_*`` function it defines is + called in definition order, in the module's own namespace -- so state the + rewriter leaks from one test function into the next is exercised exactly + as a real run would exercise it. + + Returns the message of the first assertion that fails, or ``None`` when + ``must_pass`` is set and nothing failed. + """ + src = textwrap.dedent(src) + code = compile(rewrite(src), "", "exec") + ns: dict[str, object] = {} + if extra_ns is not None: + ns.update(extra_ns) + try: + exec(code, ns) + for name, obj in list(ns.items()): + if name.startswith("test_") and callable(obj): + obj() + except AssertionError as exc: + if must_pass: + pytest.fail(f"shouldn't have raised: {_assertion_message(exc)}") + return _assertion_message(exc) + else: + if not must_pass: + pytest.fail("module didn't raise at all") + return None + + class TestAssertionRewrite: def test_place_initial_imports(self) -> None: s = """'Doc string'\nother = stuff""" @@ -346,6 +391,8 @@ def test_dont_rewrite(self) -> None: assert isinstance(m.body[1], ast.Assert) assert m.body[1].msg is None + # rewriter: import hook -- PYTEST_DONT_REWRITE has to be honoured by the + # hook when it decides whether to rewrite a plugin module at all. def test_dont_rewrite_plugin(self, pytester: Pytester) -> None: contents = { "conftest.py": "pytest_plugins = 'plugin'; import plugin", @@ -356,6 +403,7 @@ def test_dont_rewrite_plugin(self, pytester: Pytester) -> None: result = pytester.runpytest_subprocess() assert "warning" not in "".join(result.outlines) + # rewriter: import hook -- which files get rewritten (a plugin package). def test_rewrites_plugin_as_a_package(self, pytester: Pytester) -> None: pkgdir = pytester.mkpydir("plugin") pkgdir.joinpath("__init__.py").write_text( @@ -372,6 +420,7 @@ def test_rewrites_plugin_as_a_package(self, pytester: Pytester) -> None: result = pytester.runpytest() result.stdout.fnmatch_lines(["*assert 1 == 2*"]) + # rewriter: module import semantics -- PEP 235 case sensitivity on import. def test_honors_pep_235(self, pytester: Pytester, monkeypatch) -> None: # note: couldn't make it fail on macos with a single `sys.path` entry # note: these modules are named `test_*` to trigger rewriting @@ -477,73 +526,46 @@ def f(): assert getmsg(f) == "AssertionError: something bad!\nassert False" - def test_assertion_message(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_foo(): - assert 1 == 2, "The failure message" - """ - ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines( - ["*AssertionError*The failure message*", "*assert 1 == 2*"] - ) + def test_assertion_message(self) -> None: + def test_foo(): + assert 1 == 2, "The failure message" # type: ignore[comparison-overlap] - def test_assertion_message_multiline(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_foo(): - assert 1 == 2, "A multiline\\nfailure message" - """ - ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines( - ["*AssertionError*A multiline*", "*failure message*", "*assert 1 == 2*"] - ) + assert getmsg(test_foo) == "AssertionError: The failure message\nassert 1 == 2" - def test_assertion_message_tuple(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_foo(): - assert 1 == 2, (1, 2) - """ - ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines([f"*AssertionError*{(1, 2)!r}*", "*assert 1 == 2*"]) + def test_assertion_message_multiline(self) -> None: + def test_foo(): + assert 1 == 2, "A multiline\nfailure message" # type: ignore[comparison-overlap] - def test_assertion_message_expr(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_foo(): - assert 1 == 2, 1 + 2 - """ + assert getmsg(test_foo) == ( + "AssertionError: A multiline\n failure message\nassert 1 == 2" ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines(["*AssertionError*3*", "*assert 1 == 2*"]) - def test_assertion_message_escape(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - def test_foo(): - assert 1 == 2, 'To be escaped: %' - """ - ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines( - ["*AssertionError: To be escaped: %", "*assert 1 == 2"] - ) + def test_assertion_message_tuple(self) -> None: + def test_foo(): + assert 1 == 2, (1, 2) # type: ignore[comparison-overlap] - def test_assertion_messages_bytes(self, pytester: Pytester) -> None: - pytester.makepyfile("def test_bytes_assertion():\n assert False, b'ohai!'\n") - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines(["*AssertionError: b'ohai!'", "*assert False"]) + assert getmsg(test_foo) == f"AssertionError: {(1, 2)!r}\nassert 1 == 2" + + def test_assertion_message_expr(self) -> None: + def test_foo(): + assert 1 == 2, 1 + 2 # type: ignore[comparison-overlap] + + assert getmsg(test_foo) == "AssertionError: 3\nassert 1 == 2" + + def test_assertion_message_escape(self) -> None: + def test_foo(): + assert 1 == 2, "To be escaped: %" # type: ignore[comparison-overlap] + assert getmsg(test_foo) == "AssertionError: To be escaped: %\nassert 1 == 2" + + def test_assertion_messages_bytes(self) -> None: + def test_bytes_assertion(): + assert False, b"ohai!" # noqa: RUF040 + + assert getmsg(test_bytes_assertion) == "AssertionError: b'ohai!'\nassert False" + + # rewriter: config plumbing -- the abbreviation threshold comes from the + # session config via ``util._config``, so a real run is what is under test. def test_assertion_message_verbosity(self, pytester: Pytester) -> None: """ Obey verbosity levels when printing the "message" part of assertions, when they are @@ -572,6 +594,7 @@ def test_assertion_verbosity(): assert result.ret == 1 result.stdout.re_match_lines([r".*AssertionError: A+$", ".*assert False"]) + # rewriter: config plumbing -- same as above, `-vv` must reach the rewriter. def test_assertion_message_verbosity_collection(self, pytester: Pytester) -> None: """ With -vv, the "message" part of assertions must not elide collection @@ -734,8 +757,8 @@ def f2() -> None: assert getmsg(f2) == "assert (False or (4 % 2))" - def test_at_operator_issue1290(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_at_operator_issue1290(self) -> None: + getmsg_src( """ class Matrix(object): def __init__(self, num): @@ -744,21 +767,20 @@ def __matmul__(self, other): return self.num * other.num def test_multmat_operator(): - assert Matrix(2) @ Matrix(3) == 6""" + assert Matrix(2) @ Matrix(3) == 6 + """, + must_pass=True, ) - pytester.runpytest().assert_outcomes(passed=1) - def test_starred_with_side_effect(self, pytester: Pytester) -> None: + def test_starred_with_side_effect(self) -> None: """See #4412""" - pytester.makepyfile( - """\ - def test(): - f = lambda x: x - x = iter([1, 2, 3]) - assert 2 * next(x) == f(*[next(x)]) - """ - ) - pytester.runpytest().assert_outcomes(passed=1) + + def test() -> None: + f = lambda x: x # noqa: E731 + x = iter([1, 2, 3]) + assert 2 * next(x) == f(*[next(x)]) + + getmsg(test, must_pass=True) def test_call(self) -> None: def g(a=42, *args, **kwargs) -> bool: @@ -938,8 +960,8 @@ def myany(x) -> bool: assert msg is not None assert " < 0" in msg - def test_assert_handling_raise_in__iter__(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_assert_handling_raise_in__iter__(self) -> None: + msg = getmsg_src( """\ class A: def __iter__(self): @@ -954,8 +976,8 @@ def __repr__(self): assert A() == A() """ ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*E*assert == "]) + assert msg is not None + assert msg.splitlines()[0] == "assert == " def test_formatchar(self) -> None: def f() -> None: @@ -997,25 +1019,27 @@ def __repr__(self): assert "UnicodeDecodeError" not in msg assert "UnicodeEncodeError" not in msg - def test_assert_fixture(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_assert_fixture(self) -> None: + msg = getmsg_src( """\ - import pytest - @pytest.fixture - def fixt(): - return 42 + import pytest + @pytest.fixture + def fixt(): + return 42 - def test_something(): # missing "fixt" argument - assert fixt == 42 + def test_something(): # missing "fixt" argument + assert fixt == 42 """ ) - result = pytester.runpytest() - result.stdout.fnmatch_lines( - ["*assert )> == 42*"] + assert msg is not None + assert re.fullmatch( + r"assert \)> == 42", msg ) class TestRewriteOnImport: + # rewriter: import hook and .pyc caching throughout -- every test here needs + # a real file imported through AssertionRewritingHook. def test_pycache_is_a_file(self, pytester: Pytester) -> None: pytester.path.joinpath("__pycache__").write_text("Hello", encoding="utf-8") pytester.makepyfile( @@ -1309,6 +1333,7 @@ def test(): class TestAssertionRewriteHookDetails: + # rewriter: import hook / .pyc caching / module import semantics throughout. def test_sys_meta_path_munged(self, pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1504,58 +1529,61 @@ def test_foo(self): result.stdout.fnmatch_lines(["*1 passed*"]) -def test_issue731(pytester: Pytester) -> None: - pytester.makepyfile( +def test_issue731() -> None: + # Braces in a custom repr must not unbalance the mini format language that + # ``_format_explanation`` speaks; if they did, formatting the message would + # blow up (historically reported as "unbalanced braces"). + msg = getmsg_src( """ - class LongReprWithBraces(object): - def __repr__(self): - return 'LongReprWithBraces({' + ('a' * 80) + '}' + ('a' * 120) + ')' + class LongReprWithBraces(object): + def __repr__(self): + return 'LongReprWithBraces({' + ('a' * 80) + '}' + ('a' * 120) + ')' - def some_method(self): - return False + def some_method(self): + return False - def test_long_repr(): - obj = LongReprWithBraces() - assert obj.some_method() - """ + def test_long_repr(): + obj = LongReprWithBraces() + assert obj.some_method() + """ ) - result = pytester.runpytest() - result.stdout.no_fnmatch_line("*unbalanced braces*") + assert msg is not None + assert "unbalanced braces" not in msg + assert msg.splitlines()[:2] == ["assert False", " + where False = some_method()"] class TestIssue925: - def test_simple_case(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_simple_case(self) -> None: + msg = getmsg_src( + """ + def test_ternary_display(): + assert (False == False) == False """ - def test_ternary_display(): - assert (False == False) == False - """ ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*E*assert (False == False) == False"]) + assert msg == "assert (False == False) == False" - def test_long_case(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_long_case(self) -> None: + msg = getmsg_src( + """ + def test_ternary_display(): + assert False == (False == True) == True """ - def test_ternary_display(): - assert False == (False == True) == True - """ ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*E*assert (False == True) == True"]) + assert msg == "assert (False == True) == True" - def test_many_brackets(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_many_brackets(self) -> None: + msg = getmsg_src( """ def test_ternary_display(): assert True == ((False == True) == True) """ ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*E*assert True == ((False == True) == True)"]) + assert msg == "assert True == ((False == True) == True)" class TestIssue2121: + # rewriter: import hook -- the subject is which files a ``python_files`` + # pattern with subdirectories causes to be rewritten on import. def test_rewrite_python_files_contain_subdirs(self, pytester: Pytester) -> None: pytester.makepyfile( **{ @@ -1578,8 +1606,8 @@ def test_simple_failure(): class TestAssertionRewriteWalrusOperator: """See #10743""" - def test_assertion_walrus_operator(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_assertion_walrus_operator(self) -> None: + getmsg_src( """ def my_func(before, after): return before == after @@ -1591,13 +1619,12 @@ def test_walrus_conversion(): a = "Hello" assert not my_func(a, a := change_value(a)) assert a == "hello" - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 - def test_assertion_walrus_operator_dont_rewrite(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_assertion_walrus_operator_dont_rewrite(self) -> None: + getmsg_src( """ 'PYTEST_DONT_REWRITE' def my_func(before, after): @@ -1610,13 +1637,12 @@ def test_walrus_conversion_dont_rewrite(): a = "Hello" assert not my_func(a, a := change_value(a)) assert a == "hello" - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 - def test_assertion_inline_walrus_operator(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_assertion_inline_walrus_operator(self) -> None: + getmsg_src( """ def my_func(before, after): return before == after @@ -1625,13 +1651,12 @@ def test_walrus_conversion_inline(): a = "Hello" assert not my_func(a, a := a.lower()) assert a == "hello" - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 - def test_assertion_inline_walrus_operator_reverse(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_assertion_inline_walrus_operator_reverse(self) -> None: + getmsg_src( """ def my_func(before, after): return before == after @@ -1640,97 +1665,83 @@ def test_walrus_conversion_reverse(): a = "Hello" assert my_func(a := a.lower(), a) assert a == 'hello' - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 - def test_assertion_walrus_no_variable_name_conflict( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( + def test_assertion_walrus_no_variable_name_conflict(self) -> None: + msg = getmsg_src( """ def test_walrus_conversion_no_conflict(): a = "Hello" assert a == (b := a.lower()) - """ + """ ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines(["*AssertionError: assert 'Hello' == 'hello'"]) + assert msg is not None + # remaining lines are the string diff `util._reprcompare` appends + assert msg.splitlines()[0] == "assert 'Hello' == 'hello'" def test_assertion_walrus_operator_true_assertion_and_changes_variable_value( - self, pytester: Pytester + self, ) -> None: - pytester.makepyfile( + getmsg_src( """ def test_walrus_conversion_succeed(): a = "Hello" assert a != (a := a.lower()) assert a == 'hello' - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 - def test_assertion_walrus_operator_fail_assertion(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_assertion_walrus_operator_fail_assertion(self) -> None: + msg = getmsg_src( """ def test_walrus_conversion_fails(): a = "Hello" assert a == (a := a.lower()) - """ + """ ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines(["*AssertionError: assert 'Hello' == 'hello'"]) + assert msg is not None + assert msg.splitlines()[0] == "assert 'Hello' == 'hello'" - def test_assertion_walrus_operator_boolean_composite( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( + def test_assertion_walrus_operator_boolean_composite(self) -> None: + getmsg_src( """ def test_walrus_operator_change_boolean_value(): a = True assert a and True and ((a := False) is False) and (a is False) and ((a := None) is None) assert a is None - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 - def test_assertion_walrus_operator_compare_boolean_fails( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( + def test_assertion_walrus_operator_compare_boolean_fails(self) -> None: + msg = getmsg_src( """ def test_walrus_operator_change_boolean_value(): a = True assert not (a and ((a := False) is False)) - """ + """ ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines(["*assert not (True and False is False)"]) + assert msg == "assert not (True and False is False)" - def test_assertion_walrus_operator_boolean_none_fails( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( + def test_assertion_walrus_operator_boolean_none_fails(self) -> None: + msg = getmsg_src( """ def test_walrus_operator_change_boolean_value(): a = True assert not (a and ((a := None) is None)) - """ + """ ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines(["*assert not (True and None is None)"]) + assert msg == "assert not (True and None is None)" def test_assertion_walrus_operator_value_changes_cleared_after_each_test( - self, pytester: Pytester + self, ) -> None: - pytester.makepyfile( + # ``getmsg_src`` calls both ``test_*`` functions in the same namespace, + # which is what makes the leak from the first into the second visible. + getmsg_src( """ def test_walrus_operator_change_value(): a = True @@ -1739,15 +1750,12 @@ def test_walrus_operator_change_value(): def test_walrus_operator_not_override_value(): a = True assert a is True - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 - def test_assertion_namedexpr_compare_left_overwrite( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( + def test_assertion_namedexpr_compare_left_overwrite(self) -> None: + msg = getmsg_src( """ def test_namedexpr_compare_left_overwrite(): a = "Hello" @@ -1756,105 +1764,91 @@ def test_namedexpr_compare_left_overwrite(): assert (a := b) == c and (a := "Test") == "Test" """ ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines(["*assert ('World' == 'Test'*"]) + assert msg is not None + assert msg.splitlines()[0] == "assert ('World' == 'Test'" class TestIssue11028: - def test_assertion_walrus_operator_in_operand(self, pytester: Pytester) -> None: - pytester.makepyfile( + def test_assertion_walrus_operator_in_operand(self) -> None: + getmsg_src( """ def test_in_string(): assert (obj := "foo") in obj - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 - def test_assertion_walrus_operator_in_operand_json_dumps( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( + def test_assertion_walrus_operator_in_operand_json_dumps(self) -> None: + getmsg_src( """ import json def test_json_encoder(): assert (obj := "foo") in json.dumps(obj) - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 - def test_assertion_walrus_operator_equals_operand_function( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( + def test_assertion_walrus_operator_equals_operand_function(self) -> None: + getmsg_src( """ def f(a): return a def test_call_other_function_arg(): assert (obj := "foo") == f(obj) - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 def test_assertion_walrus_operator_equals_operand_function_keyword_arg( - self, pytester: Pytester + self, ) -> None: - pytester.makepyfile( + getmsg_src( """ def f(a='test'): return a def test_call_other_function_k_arg(): assert (obj := "foo") == f(a=obj) - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 def test_assertion_walrus_operator_equals_operand_function_arg_as_function( - self, pytester: Pytester + self, ) -> None: - pytester.makepyfile( + getmsg_src( """ def f(a='test'): return a def test_function_of_function(): assert (obj := "foo") == f(f(obj)) - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 - def test_assertion_walrus_operator_gt_operand_function( - self, pytester: Pytester - ) -> None: - pytester.makepyfile( + def test_assertion_walrus_operator_gt_operand_function(self) -> None: + msg = getmsg_src( """ def add_one(a): return a + 1 def test_gt(): assert (obj := 4) > add_one(obj) - """ + """ ) - result = pytester.runpytest() - assert result.ret == 1 - result.stdout.fnmatch_lines(["*assert 4 > 5", "*where 5 = add_one(4)"]) + assert msg == "assert 4 > 5\n + where 5 = add_one(4)" class TestIssue11239: - def test_assertion_walrus_different_test_cases(self, pytester: Pytester) -> None: + def test_assertion_walrus_different_test_cases(self) -> None: """Regression for (#11239) Walrus operator rewriting would leak to separate test cases if they used the same variables. """ - pytester.makepyfile( + getmsg_src( """ def test_1(): state = {"x": 2}.get("x") @@ -1863,16 +1857,16 @@ def test_1(): def test_2(): db = {"x": 2} assert (state := db.get("x")) is not None - """ + """, + must_pass=True, ) - result = pytester.runpytest() - assert result.ret == 0 @pytest.mark.skipif( sys.maxsize <= (2**31 - 1), reason="Causes OverflowError on 32bit systems" ) @pytest.mark.parametrize("offset", [-1, +1]) +# rewriter: .pyc caching -- the mtime written into the pyc header. def test_source_mtime_long_long(pytester: Pytester, offset) -> None: """Support modification dates after 2038 in rewritten files (#4903). @@ -1895,6 +1889,7 @@ def test(): pass assert result.ret == 0 +# rewriter: import hook -- reentrancy of the hook while writing a pyc. def test_rewrite_infinite_recursion( pytester: Pytester, pytestconfig, monkeypatch ) -> None: @@ -1930,6 +1925,7 @@ def spy_write_pyc(*args, **kwargs): class TestEarlyRewriteBailout: + # rewriter: import hook -- which modules find_spec is even called for. @pytest.fixture def hook( self, pytestconfig, monkeypatch, pytester: Pytester @@ -2045,6 +2041,9 @@ def test(): class TestAssertionPass: + # rewriter: config plumbing -- the `enable_assertion_pass_hook` ini has to + # reach the rewriter, and a registered pytest_assertion_pass hookimpl has to + # reach util._assertion_pass. Both are session state, not rewriter output. def test_option_default(self, pytester: Pytester) -> None: config = pytester.parseconfig() assert config.getini("enable_assertion_pass_hook") is False @@ -2304,6 +2303,7 @@ def test_get_cache_dir(self, monkeypatch, prefix, source, expected) -> None: assert get_cache_dir(Path(source)) == Path(expected) + # rewriter: .pyc caching -- where the rewritten pyc lands on disk. def test_sys_pycache_prefix_integration( self, tmp_path, monkeypatch, pytester: Pytester ) -> None: @@ -2367,6 +2367,8 @@ def get_verbosity(self, verbosity_type: str | None = None) -> int: def test_get_maxsize_for_saferepr_no_config(self) -> None: assert _get_maxsize_for_saferepr(None) == DEFAULT_REPR_MAX_SIZE + # rewriter: config plumbing -- the three tests below check that -v/-vv reach + # `util._config`, which is what `_saferepr` reads its maxsize from. def create_test_file(self, pytester: Pytester, size: int) -> None: pytester.makepyfile( f""" @@ -2393,17 +2395,13 @@ def test_max_increased_verbosity(self, pytester: Pytester) -> None: class TestIssue11140: - def test_constant_not_picked_as_module_docstring(self, pytester: Pytester) -> None: - pytester.makepyfile( - """\ - 0 - - def test_foo(): - pass - """ - ) - result = pytester.runpytest() - assert result.ret == 0 + def test_constant_not_picked_as_module_docstring(self) -> None: + src = "0\n\ndef test_foo():\n pass\n" + # A non-string leading constant is not a docstring, so the rewriter must + # neither read it as one nor place its imports after it. + m = rewrite(src) + assert isinstance(m.body[0], ast.Import) + getmsg_src(src, must_pass=True) class TestSafereprUnbounded: @@ -2429,6 +2427,7 @@ def test_saferepr_unbounded(self): ) +# rewriter: plugin machinery -- the subject is a full run with a plugin disabled. def test_assertion_failure_when_terminalreporter_is_disabled( pytester: Pytester, ) -> None: From 40f173c06da85e19173d780850ae5480407425c6 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 11:27:01 +0200 Subject: [PATCH 23/30] testing: skip traceback rendering in terminal ensemble tests An ensemble item's code object belongs to the *host* test file, so a failing item's `repr_failure` runs `getstatementrange_ast` over all 4000-odd lines of `test_terminal.py` -- `inspect.findsource` hands back whole-file lines, and the AST cache lives on `FormattedExcinfo`, i.e. per failure, so it never hits. That is ~40ms per failing item. A pytester temp file is ten lines, so the same repr costs ~0.2ms there. This is why the ported tests came out *slower* than the pytester originals they replaced: the port trades ~10ms of config build for ~37ms of host-file reparse on every failure. `Function.repr_failure` only reaches `_getentrysource` for tbstyle `short`/`long`, and `--tb=no` still produces `longrepr.reprcrash`, so short summary lines (`FAILED ... - assert 0`, `-rfE`, `-rx`) render unchanged. Pass it at the 16 call sites -- 20 test functions -- whose assertions never touch traceback text, each annotated with what it does assert on instead. Host-file parses drop from 109 to 18, and the file from 20.15s to 16.70s. The remaining 18 are tests that genuinely assert on traceback output and must keep paying. Levers that did not pay, measured rather than assumed: dropping `capture_output` is worth 0.9ms and only 7 of 74 ported tests can (the rest assert on `record.output`); `collect_tests` over `run_tests` is worth 0.5ms, since collection is nearly the whole config cost; batching rounds into one `Ensemble` saves only config builds, and the cost here is per failing *item* -- `test_teardown_many` did 20 reparses inside a single run. Co-Authored-By: Claude Opus 5 (1M context) --- testing/test_terminal.py | 77 ++++++++++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 5a264fb06d4..9441899d38a 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -963,11 +963,13 @@ def test_fail(): def test_skip(): pytest.skip("dontshow") + # `--tb=no`: only the absence of a skip summary is asserted on; see + # TestProgressWithTeardown for why rendering the traceback is costly. record = run_tests( test_ok, test_fail, test_skip, - rootpath=tmp_path, + spec=ConfigSpec(rootpath=tmp_path, args=("--tb=no",)), name="test_no_skip_summary_if_failure", capture_output=True, ) @@ -1233,8 +1235,11 @@ class TestClass: def test_skip(self): pytest.skip("hello") + # `--tb=no`: the `-v` per-test lines are the point; see + # TestProgressWithTeardown for why rendering the traceback is costly. spec = ConfigSpec( - rootpath=tmp_path, args=("-v", "-Walways::pytest.PytestWarning") + rootpath=tmp_path, + args=("-v", "--tb=no", "-Walways::pytest.PytestWarning"), ) record = run_tests( test_fail, @@ -1328,7 +1333,9 @@ def test_summary_f_alias(self, tmp_path: Path) -> None: def test(): assert False - spec = ConfigSpec(rootpath=tmp_path, args=("-rfF",)) + # `--tb=no`: the asserted line is the short summary one, built from + # `longrepr.reprcrash`, which `--tb=no` still produces. + spec = ConfigSpec(rootpath=tmp_path, args=("-rfF", "--tb=no")) record = run_tests( test, spec=spec, name="test_summary_f_alias", capture_output=True ) @@ -1414,16 +1421,19 @@ def test_fail_extra_reporting( def test_this(): assert 0, "this_failed" * 100 + # `--tb=no`: the short summary line is what is asserted on, and it is built + # from `longrepr.reprcrash`, which `--tb=no` still produces. Rendering the + # traceback would re-parse this whole file (see TestProgressWithTeardown). record = run_tests( test_this, - spec=ConfigSpec(rootpath=tmp_path, args=("-rN",)), + spec=ConfigSpec(rootpath=tmp_path, args=("-rN", "--tb=no")), name="test_fail_extra_reporting", capture_output=True, ) record.stdout.no_fnmatch_line("*short test summary*") record = run_tests( test_this, - rootpath=tmp_path, + spec=ConfigSpec(rootpath=tmp_path, args=("--tb=no",)), name="test_fail_extra_reporting", capture_output=True, ) @@ -1473,7 +1483,7 @@ def test_this(): record = run_tests( test_this, - spec=ConfigSpec(rootpath=tmp_path, args=("-rp",)), + spec=ConfigSpec(rootpath=tmp_path, args=("-rp", "--tb=no")), capture_output=True, ) record.stdout.no_fnmatch_line("*short test summary*") @@ -2120,9 +2130,12 @@ def test_failure(): warnings.warn("warning_from_" + "test") assert 0 + # `--tb=no` renders identically here - `--no-summary` already suppresses + # the FAILURES section - but skips the failure repr, which would re-parse + # this whole file (see TestProgressWithTeardown). spec = ConfigSpec( rootpath=tmp_path, - args=("--no-summary",), + args=("--no-summary", "--tb=no"), inicfg={"filterwarnings": ["default"]}, ) record = run_tests( @@ -2349,8 +2362,12 @@ def test_three_3(): @staticmethod def _run(tmp_path: Path, test_files, *args: str): + # `--tb=no`: only the progress and summary lines are asserted on, and + # rendering the tracebacks of the two failures would re-parse the whole + # of this file (see TestProgressWithTeardown). spec = ConfigSpec( - rootpath=tmp_path, args=("-o", "console_output_style=classic", *args) + rootpath=tmp_path, + args=("-o", "console_output_style=classic", "--tb=no", *args), ) return run_tests(*test_files, spec=spec, capture_output=True) @@ -2529,7 +2546,10 @@ def test_foobar(i): # The host suite turns warnings into errors; the point here is the # yellow progress indicator a *recorded* warning produces. inicfg: dict[str, object] = {"filterwarnings": ["always"]} - record = self._run(tmp_path, sources, inicfg=inicfg) + # `--tb=no`: the progress indicators are the point; rendering the five + # ValueError tracebacks would re-parse this file five times over (see + # TestProgressWithTeardown). + record = self._run(tmp_path, sources, "--tb=no", inicfg=inicfg) record.stdout.re_match_lines( color_mapping.format_for_rematch( [ @@ -2543,7 +2563,7 @@ def test_foobar(i): record.assert_outcomes(passed=15, failed=5, xfailed=1, warnings=5) # Only xfail should have yellow progress indicator. - record = self._run(tmp_path, (axfail_module,)) + record = self._run(tmp_path, (axfail_module,), "--tb=no") record.stdout.re_match_lines( color_mapping.format_for_rematch( [ @@ -2772,7 +2792,16 @@ def test_capture_no_progress_enabled( class TestProgressWithTeardown: - """Ensure we show the correct percentages for tests that fail during teardown (#3088)""" + """Ensure we show the correct percentages for tests that fail during teardown (#3088) + + ``--tb=no`` where no traceback is asserted on: an ensemble item's code + object belongs to *this* file, so every failure repr re-parses the whole + of ``test_terminal.py`` to find its statement range (~40ms each, and + ``test_teardown_many`` produces twenty of them). Suppressing the + traceback skips ``Node.repr_failure``'s source lookup entirely; the + progress column, the short summary and the report objects these tests do + assert on are all unaffected. + """ @pytest.fixture def teardown_fixture_plugin(self) -> object: @@ -2838,7 +2867,9 @@ def test_foo(fail_teardown): record = run_tests( test_foo, spec=ConfigSpec( - rootpath=tmp_path, extra_plugins=(teardown_fixture_plugin,) + rootpath=tmp_path, + args=("--tb=no",), + extra_plugins=(teardown_fixture_plugin,), ), name="test_teardown_simple", capture_output=True, @@ -2857,7 +2888,7 @@ def test_foo(fail_teardown): test_foo, spec=ConfigSpec( rootpath=tmp_path, - args=("-rfE",), + args=("-rfE", "--tb=no"), extra_plugins=(teardown_fixture_plugin,), ), name="test_teardown_with_test_also_failing", @@ -2878,7 +2909,9 @@ def test_teardown_many( record = run_tests( *many_sources, spec=ConfigSpec( - rootpath=tmp_path, extra_plugins=(teardown_fixture_plugin,) + rootpath=tmp_path, + args=("--tb=no",), + extra_plugins=(teardown_fixture_plugin,), ), capture_output=True, ) @@ -2894,7 +2927,7 @@ def test_teardown_many_verbose( *many_sources, spec=ConfigSpec( rootpath=tmp_path, - args=("-v",), + args=("-v", "--tb=no"), extra_plugins=(teardown_fixture_plugin,), ), capture_output=True, @@ -3489,10 +3522,15 @@ def test_skip(): def _run( module: ModuleType, tmp_path: Path, verbosity: int, *args: str ) -> RunRecord: - """Run *module* with ``verbosity_test_cases`` set, capturing output.""" + """Run *module* with ``verbosity_test_cases`` set, capturing output. + + ``--tb=no``: every case here asserts a consecutive block of collect and + progress lines, never a traceback, and rendering the one failure would + re-parse this whole file (see TestProgressWithTeardown). + """ spec = ConfigSpec( rootpath=tmp_path, - args=args, + args=("--tb=no", *args), inicfg={"verbosity_test_cases": str(verbosity)}, ) return run_tests(module, spec=spec, capture_output=True) @@ -3728,10 +3766,13 @@ def test_xfail(): def test_xfail_reason(): assert False + # `--tb=no` renders identically here - XFAILURES is already suppressed + # without `--xfail-tb` - but skips the failure repr, which would re-parse + # this whole file twice (see TestProgressWithTeardown). record = run_tests( test_xfail, test_xfail_reason, - spec=ConfigSpec(rootpath=tmp_path, args=("-rx",)), + spec=ConfigSpec(rootpath=tmp_path, args=("-rx", "--tb=no")), name="test_summary_xfail_reason", capture_output=True, ) From b5471acb920a4f4b30221ad5c7f061f24803573c Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 13:50:48 +0200 Subject: [PATCH 24/30] code: parse the enclosing block, not the whole file, to find a statement Locating the failing statement of a traceback entry means parsing source into an AST, and `getsource` handed `getstatementrange_ast` the entire file. The AST cache lives on `FormattedExcinfo`, i.e. one per rendered failure, so a large test module reparsed all of itself for every entry of every failure -- ~40ms per entry for a 4000-line file, against ~0.2ms for the ten-line temp file the same test would use under pytester. Narrow the parse to the block the frame belongs to. Full suite goes from 304.45s to 249.88s. `inspect.getblock` rather than the code object's own line extent: `co_lines()` can undershoot, because a trailing `), "msg"` line may carry no bytecode, and cutting it would change the rendered source. Two guards, both needed and both pinned by tests: * `getblock` only walks an indented suite for def/class/decorated code; for anything else `BlockFinder` stops at the first logical line. A module-level frame therefore gets a block that is not the module -- which rendered `???` in place of the conftest line in `test_better_reporting_on_conftest_load_failure`. Falling back whenever the reported line is outside the block covers that, and covers exec'd or generated code whose lines do not match the file. * tokenizing can fail on sources that do not correspond to the frame. The astcache key carries the parse offset rather than the frame's first line: whether narrowing applied depends on the line being reported, so two entries in the same function can disagree, and a cached tree must never be paired with linenos it was not parsed from. Not used here, for the record: `co_positions()` gives the failing *expression* (lines 9-11 of a five-line assert whose statement is 8-12), and `ast.stmt.end_lineno` spans a compound statement's whole suite where pytest shows only its header -- across `src/_pytest` it differs from the current result on 30.6% of lines, 99.5% of those compound. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/14809.improvement.rst | 6 +++ src/_pytest/_code/code.py | 34 ++++++++++--- testing/code/test_code.py | 87 +++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 changelog/14809.improvement.rst diff --git a/changelog/14809.improvement.rst b/changelog/14809.improvement.rst new file mode 100644 index 00000000000..09cbb4ba1b8 --- /dev/null +++ b/changelog/14809.improvement.rst @@ -0,0 +1,6 @@ +Rendering a traceback entry now parses only the enclosing block rather than the whole source file. + +Locating the failing statement requires parsing source into an AST, and that was done for the +entire file once per rendered entry -- so a failure in a large test module paid for parsing every +line of it, repeatedly. The parse is now restricted to the block the frame belongs to, falling +back to the full file where the block cannot be determined. diff --git a/src/_pytest/_code/code.py b/src/_pytest/_code/code.py index e37a1324c67..dbcc5b0b6ef 100644 --- a/src/_pytest/_code/code.py +++ b/src/_pytest/_code/code.py @@ -13,6 +13,7 @@ from pathlib import Path import re import sys +import tokenize from traceback import extract_tb from traceback import format_exception from traceback import format_exception_only @@ -284,7 +285,7 @@ def getfirstlinesource(self) -> int: return self.frame.code.firstlineno def getsource( - self, astcache: dict[str | Path, ast.AST] | None = None + self, astcache: dict[tuple[str | Path, int], ast.AST] | None = None ) -> Source | None: """Return failing source code.""" # we use the passed in astcache to not reparse asttrees @@ -292,19 +293,40 @@ def getsource( source = self.frame.code.fullsource if source is None: return None + start = self.getfirstlinesource() + # Narrow the parse to the enclosing block: locating one statement is + # otherwise O(file), and it is paid once per rendered traceback entry. + block, offset = source, 0 + try: + candidate = Source(inspect.getblock(source.raw_lines[start:])) + except (OSError, IndentationError, tokenize.TokenError, SyntaxError): + pass + else: + # The block can fall short of the frame: getblock only walks an + # indented suite for def/class/decorated code and otherwise stops + # at the first logical line, so a module-level frame lands here, + # as does exec'd or generated code whose lines do not match. + # Only narrow when the reported line is actually inside. + if start <= self.lineno < start + len(candidate.lines): + block, offset = candidate, start + # The key carries the offset, not `start`: whether the block was + # narrowed depends on the line being reported, so two entries in the + # same function can disagree, and a cached tree must never be paired + # with linenos it was not parsed from. key = astnode = None if astcache is not None: - key = self.frame.code.path - if key is not None: + path = self.frame.code.path + if path is not None: + key = (path, offset) astnode = astcache.get(key, None) - start = self.getfirstlinesource() try: astnode, _, end = getstatementrange_ast( - self.lineno, source, astnode=astnode + self.lineno - offset, block, astnode=astnode ) except SyntaxError: end = self.lineno + 1 else: + end += offset if key is not None and astcache is not None: astcache[key] = astnode return source[start:end] @@ -893,7 +915,7 @@ class ExceptionInfoFormatter: truncate_args: bool = True chain: bool = True - astcache: dict[str | Path, ast.AST] = dataclasses.field( + astcache: dict[tuple[str | Path, int], ast.AST] = dataclasses.field( default_factory=dict, init=False, repr=False ) diff --git a/testing/code/test_code.py b/testing/code/test_code.py index 6947320a9ce..aebf8987a59 100644 --- a/testing/code/test_code.py +++ b/testing/code/test_code.py @@ -1,9 +1,13 @@ # mypy: allow-untyped-defs from __future__ import annotations +import ast +import linecache +from pathlib import Path import re import sys from types import FrameType +from typing import Any from unittest import mock from _pytest._code import Code @@ -225,3 +229,86 @@ def test_ExceptionChainRepr(): assert isinstance(repr1, ExceptionChainRepr) assert hash(repr1) != hash(repr2) assert repr1 is not excinfo1.getrepr() + + +class TestGetSourceNarrowing: + """``TracebackEntry.getsource`` parses the enclosing block, not the file. + + Locating the failing statement means parsing source into an AST. Doing + that for the whole file costs O(file) per rendered traceback entry, which + is why the block is parsed instead -- with a fallback for the frames whose + block cannot be determined or does not contain the reported line. + """ + + def test_function_frame_parses_only_the_block(self) -> None: + astcache: dict[tuple[str | Path, int], ast.AST] = {} + excinfo = pytest.raises(ValueError, self._boom) + entry = excinfo.traceback[-1] + source = entry.getsource(astcache) + assert source is not None + assert str(source).endswith('raise ValueError("boom")') + + # The cached tree is the method, not this whole file. + (cached,) = astcache.values() + assert isinstance(cached, ast.Module) + (node,) = cached.body + assert isinstance(node, ast.FunctionDef) + assert node.name == "_boom" + + def _boom(self) -> None: + raise ValueError("boom") + + def test_module_frame_parses_the_whole_file(self) -> None: + """A module frame has no enclosing block, so it parses the whole file. + + ``getblock`` walks an indented suite only for def/class/decorated + code; for anything else it stops at the first logical line, which is + never the whole module. + """ + astcache: dict[tuple[str | Path, int], ast.AST] = {} + filename = "" + lines = ["if 1:\n", " raise ValueError('boom')\n", "x = 2\n"] + code = compile("".join(lines), filename, "exec") + with mock.patch.dict(linecache.cache, {filename: (1, None, lines, filename)}): + excinfo = pytest.raises(ValueError, exec, code, {}) + entry = excinfo.traceback[-1] + assert entry.frame.code.raw.co_name == "" + source = entry.getsource(astcache) + assert source is not None + assert str(source) == "if 1:\n raise ValueError('boom')" + + # The whole file, so the statement after the block is in the tree. + (cached,) = astcache.values() + assert isinstance(cached, ast.Module) + assert len(cached.body) == 2 + + def test_line_outside_the_block_falls_back(self) -> None: + """The block can fall short of the frame -- exec'd or generated code, + a decorator returning a differently shaped callable.""" + filename = "" + # What linecache reports disagrees with what was compiled: the block + # starting at line 1 is a single line, but the frame reports line 3. + shown = ["def g(): pass\n", "x = 1\n", "y = 2\n"] + code = compile( + "def g():\n x = 1\n raise ValueError('boom')\n", filename, "exec" + ) + ns: dict[str, Any] = {} + exec(code, ns) + with mock.patch.dict(linecache.cache, {filename: (1, None, shown, filename)}): + excinfo = pytest.raises(ValueError, ns["g"]) + source = excinfo.traceback[-1].getsource() + assert source is not None + # Narrowing would have cut the file off after line 1. + assert "y = 2" in str(source) + + def test_unparseable_block_falls_back(self) -> None: + """``inspect.getblock`` tokenizes, and tokenizing can fail.""" + filename = "" + shown = ["def g():\n", ' """\n'] + code = compile("def g():\n raise ValueError('boom')\n", filename, "exec") + ns: dict[str, Any] = {} + exec(code, ns) + with mock.patch.dict(linecache.cache, {filename: (1, None, shown, filename)}): + excinfo = pytest.raises(ValueError, ns["g"]) + source = excinfo.traceback[-1].getsource() + assert source is not None From 8ae718bd5bd4b2509fe128af385f01fc69864ab5 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 19:59:26 +0200 Subject: [PATCH 25/30] ensemble: make tmpdir opt-in with a caller-supplied factory A `TempPathFactory` built from a config allocates a numbered directory under the global `pytest-of-` root. That scans the root and every sibling run's leftovers, takes a cleanup lock, and leaves a directory behind -- per ensemble. Ensembles are built in the hundreds, so the `tmpdir` plugin cannot be a default. Drop it from DEFAULT_PLUGINS and load it only when `ConfigSpec.tmp_path_factory` supplies a factory, bound over the one the plugin builds by a trylast `pytest_configure`, so the fixtures, retention handling and sessionfinish cleanup all stay the plugin's own. `make_tmp_path_factory(basetemp)` builds one that takes the `--basetemp` path rather than allocating; the `ensemble_tmp_path_factory` fixture roots it in the host test's own `tmp_path`, so it needs no allocation and is cleaned up with the host. Five tests in testing/python/fixtures.py genuinely use `tmp_path` inside the ensemble and now opt in; nothing else in the suite did, which is the point -- ~1000 ensemble configs were registering a plugin they never used. Note this does not move `find_prefixed`: `getbasetemp()` is lazy, so those ensembles never allocated a base temp to begin with. The 19s of directory scanning is pytester's own `tmp_path`, not the ensemble's. Co-Authored-By: Claude Opus 5 (1M context) --- src/_pytest/ensemble/__init__.py | 2 ++ src/_pytest/ensemble/config.py | 60 +++++++++++++++++++++++++++++++- testing/conftest.py | 14 ++++++++ testing/python/fixtures.py | 36 +++++++++++++------ testing/test_ensemble.py | 58 ++++++++++++++++++++++++++++++ 5 files changed, 159 insertions(+), 11 deletions(-) diff --git a/src/_pytest/ensemble/__init__.py b/src/_pytest/ensemble/__init__.py index fcf99973129..082d1c520db 100644 --- a/src/_pytest/ensemble/__init__.py +++ b/src/_pytest/ensemble/__init__.py @@ -70,6 +70,7 @@ from _pytest.ensemble.config import ConfigSpec from _pytest.ensemble.config import configured from _pytest.ensemble.config import DEFAULT_PLUGINS +from _pytest.ensemble.config import make_tmp_path_factory from _pytest.ensemble.results import ensure_recorder from _pytest.ensemble.results import ItemRecord from _pytest.ensemble.results import run_items @@ -99,6 +100,7 @@ "collect_sources", "collect_tests", "configured", + "make_tmp_path_factory", "module_from_path", "run_items", "run_tests", diff --git a/src/_pytest/ensemble/config.py b/src/_pytest/ensemble/config.py index 69901251493..d65bc0e81a1 100644 --- a/src/_pytest/ensemble/config.py +++ b/src/_pytest/ensemble/config.py @@ -14,11 +14,13 @@ from _pytest.config import Config from _pytest.config import essential_plugins +from _pytest.config import hookimpl from _pytest.config import PytestPluginManager from _pytest.config.findpaths import ConfigValue from _pytest.config.findpaths import parse_override_ini from _pytest.stash import StashKey from _pytest.terminal import terminal_file_key +from _pytest.tmpdir import TempPathFactory #: Warnings raised while the ensemble config was being configured or @@ -33,6 +35,12 @@ #: excluding everything that renders output, captures io, or installs #: process-global state (terminal, capture, cacheprovider, assertion, #: debugging, faulthandler, logging, threadexception, unraisableexception, ...). +#: +#: ``tmpdir`` is not here either: it would allocate the ensemble its own +#: numbered base temp directory under the global ``pytest-of-`` root, +#: which means scanning that root once per ensemble and leaving a directory +#: behind for every one ever configured. It is loaded only when +#: :attr:`ConfigSpec.tmp_path_factory` supplies a preconfigured factory. DEFAULT_PLUGINS: Final[tuple[str, ...]] = ( *essential_plugins, # mark, main, runner, fixtures, helpconfig "python", @@ -42,7 +50,6 @@ "unittest", "monkeypatch", "recwarn", - "tmpdir", # Not for the rewriting - that is installed from Config._preparse, which # an ensemble never runs - but for the failure *explanation*. Without # this plugin ``assertion.util._reprcompare`` stays bound to whatever the @@ -86,6 +93,14 @@ class ConfigSpec: #: Not supported yet; ensemble configs never load conftest files. load_conftests: bool = False + #: Preconfigured temp path factory. Supplying one loads the ``tmpdir`` + #: plugin and binds this factory instead of the one it would build from + #: the config, so an ensemble's ``tmp_path`` lives wherever the caller + #: decided - normally inside the *host* test's own ``tmp_path``, which + #: costs no root scan and is cleaned up with the host. Build one with + #: :func:`make_tmp_path_factory`. + tmp_path_factory: TempPathFactory | None = None + #: Stream the terminal plugin writes to, when it is loaded at all. An #: ensemble must never be given the stdout of whatever is running it, #: so this is bound at construction rather than redirected around it. @@ -116,6 +131,46 @@ def without_plugins(self, *names: str) -> ConfigSpec: return self.replace(plugins=tuple(p for p in self.plugins if p not in names)) +def make_tmp_path_factory(basetemp: pathlib.Path) -> TempPathFactory: + """Build a :class:`TempPathFactory` for an ensemble, rooted at *basetemp*. + + *basetemp* is treated the way ``--basetemp`` is: it is removed if it + already exists, then created. Pass a path that is yours to destroy - a + subdirectory of the host test's ``tmp_path`` is the intended use. + + The point is what this does *not* do. A factory built from a config + allocates a numbered directory under the global ``pytest-of-`` + root, which scans that root and every sibling run's leftovers, registers + a cleanup lock, and leaves the directory behind afterwards. Ensembles are + built in the hundreds, so paying that per ensemble is not viable. + """ + return TempPathFactory( + given_basetemp=basetemp, + trace=lambda *args, **kwargs: None, + retention_count=0, + retention_policy="all", + _ispytest=True, + ) + + +class _BindTmpPathFactory: + """Bind a caller-supplied factory over the one ``tmpdir`` builds. + + The ``tmpdir`` plugin creates its factory in ``pytest_configure``; this + runs last and replaces it, so the fixtures, the retention handling and + the ``pytest_sessionfinish`` cleanup all stay the plugin's own. + """ + + __pytest_no_fixtures__ = True + + def __init__(self, factory: TempPathFactory) -> None: + self._factory = factory + + @hookimpl(trylast=True) + def pytest_configure(self, config: Config) -> None: + config._tmp_path_factory = self._factory # type: ignore[attr-defined] + + def _own(value: object) -> ConfigValue: """Wrap a spec's ini value in a ConfigValue the config may own. @@ -176,6 +231,9 @@ def configured(spec: ConfigSpec) -> Iterator[Config]: try: for name in spec.plugins: pluginmanager.import_plugin(name) + if spec.tmp_path_factory is not None: + pluginmanager.import_plugin("tmpdir") + pluginmanager.register(_BindTmpPathFactory(spec.tmp_path_factory)) for plugin in spec.extra_plugins: if isinstance(plugin, str): pluginmanager.import_plugin(plugin) diff --git a/testing/conftest.py b/testing/conftest.py index 663c9d80b3e..153770bf0fc 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -3,13 +3,16 @@ from collections.abc import Generator import importlib.metadata +from pathlib import Path import re import sys from packaging.version import Version +from _pytest.ensemble import make_tmp_path_factory from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester +from _pytest.tmpdir import TempPathFactory import pytest @@ -91,6 +94,17 @@ def pytest_collection_modifyitems(items) -> Generator[None]: return (yield) +@pytest.fixture +def ensemble_tmp_path_factory(tmp_path: Path) -> TempPathFactory: + """A ``TempPathFactory`` for an ensemble, rooted in this test's tmp_path. + + Pass it as ``ConfigSpec.tmp_path_factory`` to give the ensemble a working + ``tmp_path`` fixture. The directory belongs to the host test, so it needs + no base temp allocation of its own and is cleaned up with the host's. + """ + return make_tmp_path_factory(tmp_path / "ensemble-tmp") + + @pytest.fixture def tw_mock(): """Returns a mock terminal writer""" diff --git a/testing/python/fixtures.py b/testing/python/fixtures.py index fc014d9daf5..6d49a383c97 100644 --- a/testing/python/fixtures.py +++ b/testing/python/fixtures.py @@ -25,6 +25,7 @@ from _pytest.pytester import LineMatcher from _pytest.pytester import Pytester from _pytest.python import Function +from _pytest.tmpdir import TempPathFactory import pytest @@ -845,7 +846,7 @@ def test_func(resource): record.assert_outcomes(passed=2) def test_getfixturevalue_teardown_previously_requested_does_not_warn( - self, tmp_path: Path + self, tmp_path: Path, ensemble_tmp_path_factory: TempPathFactory ) -> None: """Test that requesting a fixture during teardown that was previously requested is OK (#12882). @@ -863,7 +864,11 @@ def test_it(fix): pass # -Werror of the original: any warning would fail the run. - spec = ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["error"]}) + spec = ConfigSpec( + rootpath=tmp_path, + inicfg={"filterwarnings": ["error"]}, + tmp_path_factory=ensemble_tmp_path_factory, + ) record = run_tests(fix, test_it, spec=spec) record.assert_outcomes(passed=1, warnings=0) @@ -897,7 +902,7 @@ def test_it(fix): record.assert_outcomes(passed=1) def test_getfixturevalue_teardown_new_inactive_fixture_errors( - self, tmp_path: Path + self, tmp_path: Path, ensemble_tmp_path_factory: TempPathFactory ) -> None: """Test that requesting a fixture during teardown that was not previously requested raises an error (#12882).""" @@ -910,7 +915,8 @@ def fix(request): def test_it(fix): pass - record = run_tests(fix, test_it, rootpath=tmp_path) + spec = ConfigSpec(rootpath=tmp_path, tmp_path_factory=ensemble_tmp_path_factory) + record = run_tests(fix, test_it, spec=spec) # The call phase passes; the teardown error is a separate report. record.assert_outcomes(passed=1, errors=1) teardown = record["test_it"].teardown @@ -921,7 +927,7 @@ def test_it(fix): ) def test_getfixturevalue_teardown_new_inactive_fixture_errors_top_request( - self, tmp_path: Path + self, tmp_path: Path, ensemble_tmp_path_factory: TempPathFactory ) -> None: """Test that requesting a fixture during teardown that was not previously requested raises an error (tricky case) (#12882).""" @@ -929,7 +935,8 @@ def test_getfixturevalue_teardown_new_inactive_fixture_errors_top_request( def test_it(request): request.addfinalizer(lambda: request.getfixturevalue("tmp_path")) - record = run_tests(test_it, rootpath=tmp_path) + spec = ConfigSpec(rootpath=tmp_path, tmp_path_factory=ensemble_tmp_path_factory) + record = run_tests(test_it, spec=spec) record.assert_outcomes(passed=1, errors=1) teardown = record["test_it"].teardown assert teardown is not None @@ -1121,7 +1128,9 @@ def test_somefunc(): ... req = TopRequest(item, _ispytest=True) assert req.path == modcol.path - def test_request_fixturenames(self, tmp_path: Path) -> None: + def test_request_fixturenames( + self, tmp_path: Path, ensemble_tmp_path_factory: TempPathFactory + ) -> None: @pytest.fixture def arg1(): pass @@ -1144,7 +1153,8 @@ def test_function(request, farg): "tmp_path_factory", } - record = run_tests(arg1, farg, sarg, test_function, rootpath=tmp_path) + spec = ConfigSpec(rootpath=tmp_path, tmp_path_factory=ensemble_tmp_path_factory) + record = run_tests(arg1, farg, sarg, test_function, spec=spec) record.assert_outcomes(passed=1) def test_request_fixturenames_dynamic_fixture(self) -> None: @@ -2085,7 +2095,9 @@ def item(request): return pytester @pytest.fixture - def spec(self, tmp_path: Path) -> ConfigSpec: + def spec( + self, tmp_path: Path, ensemble_tmp_path_factory: TempPathFactory + ) -> ConfigSpec: """The rootdir conftest of this class, as a plugin object.""" class ConftestPlugin: @@ -2109,7 +2121,11 @@ def fm(self, request): def item(self, request): return request._pyfuncitem - return ConfigSpec(rootpath=tmp_path, extra_plugins=(ConftestPlugin(),)) + return ConfigSpec( + rootpath=tmp_path, + extra_plugins=(ConftestPlugin(),), + tmp_path_factory=ensemble_tmp_path_factory, + ) def test_parsefactories_conftest(self, spec: ConfigSpec) -> None: def test_check_setup(item, fm): diff --git a/testing/test_ensemble.py b/testing/test_ensemble.py index 552386541de..bc361dc6949 100644 --- a/testing/test_ensemble.py +++ b/testing/test_ensemble.py @@ -22,11 +22,13 @@ from _pytest.ensemble import configured from _pytest.ensemble import Ensemble from _pytest.ensemble import EnsembleModule +from _pytest.ensemble import make_tmp_path_factory from _pytest.ensemble import module_from_path from _pytest.ensemble import run_tests from _pytest.ensemble import running_session from _pytest.nodes import Collector from _pytest.pytester import Pytester +from _pytest.tmpdir import TempPathFactory import pytest @@ -933,3 +935,59 @@ def test_inner(): ) result = pytester.runpytest_inprocess() result.assert_outcomes(passed=1) + + +class TestTmpPath: + """``tmpdir`` is opt-in, and only with a factory the caller controls.""" + + def test_not_available_by_default(self, tmp_path: Path) -> None: + def test_it(tmp_path: Path) -> None: + pass + + record = run_tests(test_it, rootpath=tmp_path) + record.assert_outcomes(errors=1) + setup = record["test_it"].setup + assert setup is not None + assert "fixture 'tmp_path' not found" in setup.longreprtext + + def test_factory_binds_and_is_used( + self, tmp_path: Path, ensemble_tmp_path_factory: TempPathFactory + ) -> None: + seen: list[Path] = [] + + def test_it(tmp_path: Path) -> None: + seen.append(tmp_path) + + spec = ConfigSpec(rootpath=tmp_path, tmp_path_factory=ensemble_tmp_path_factory) + run_tests(test_it, spec=spec).assert_outcomes(passed=1) + + (path,) = seen + assert path.is_dir() + # Inside the host's own tmp_path, not a base temp of the ensemble's own. + assert path.is_relative_to(tmp_path) + + def test_no_basetemp_of_its_own(self, tmp_path: Path) -> None: + """The whole point: no numbered dir under ``pytest-of-``.""" + factory = make_tmp_path_factory(tmp_path / "ensemble-tmp") + + def test_it(tmp_path: Path) -> None: + pass + + spec = ConfigSpec(rootpath=tmp_path, tmp_path_factory=factory) + run_tests(test_it, spec=spec).assert_outcomes(passed=1) + assert factory.getbasetemp() == (tmp_path / "ensemble-tmp").resolve() + + def test_factory_is_shared_across_runs( + self, tmp_path: Path, ensemble_tmp_path_factory: TempPathFactory + ) -> None: + """A reused factory keeps handing out fresh directories.""" + seen: list[Path] = [] + + def test_it(tmp_path: Path) -> None: + seen.append(tmp_path) + + spec = ConfigSpec(rootpath=tmp_path, tmp_path_factory=ensemble_tmp_path_factory) + for _ in range(2): + run_tests(test_it, spec=spec).assert_outcomes(passed=1) + + assert len(set(seen)) == 2 From 82e26cb6b4beab3b8dd72a1dcecb269120bbb462 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 20:08:28 +0200 Subject: [PATCH 26/30] ensemble: opt in to unraisableexception, without collecting by default `unraisableexception` was already absent from DEFAULT_PLUGINS, but an ensemble that opted into it would inherit the default gc policy and run a full `gc.collect()` at both the cleanup and the unconfigure site. The heap that walks is the *host* process's - measured at ~243ms a pass in pytest's own suite - so it costs whatever the host happens to be holding rather than anything the ensemble owns. `ConfigSpec.gc_collect_iterations` seeds the stash key the plugin reads, defaulting to none: opting the plugin in gets the unraisable *reporting* without the collection, and a test that needs finalizers flushed for an exception to surface raises it deliberately. pytester keeps its own policy unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- src/_pytest/ensemble/config.py | 13 ++++++++++ testing/test_ensemble.py | 46 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/_pytest/ensemble/config.py b/src/_pytest/ensemble/config.py index d65bc0e81a1..26e5f1a1682 100644 --- a/src/_pytest/ensemble/config.py +++ b/src/_pytest/ensemble/config.py @@ -21,6 +21,7 @@ from _pytest.stash import StashKey from _pytest.terminal import terminal_file_key from _pytest.tmpdir import TempPathFactory +from _pytest.unraisableexception import gc_collect_iterations_key #: Warnings raised while the ensemble config was being configured or @@ -101,6 +102,15 @@ class ConfigSpec: #: :func:`make_tmp_path_factory`. tmp_path_factory: TempPathFactory | None = None + #: How many ``gc.collect()`` passes ``unraisableexception`` makes, when + #: that plugin is opted into at all. It is not in :data:`DEFAULT_PLUGINS`, + #: and even when loaded an ensemble does not collect by default: the heap + #: it would walk is the *host* process's, so a full pass costs whatever + #: the host happens to be holding rather than anything the ensemble owns. + #: Raise it only for a test that needs finalizers flushed before an + #: unraisable exception can surface. + gc_collect_iterations: int = 0 + #: Stream the terminal plugin writes to, when it is loaded at all. An #: ensemble must never be given the stdout of whatever is running it, #: so this is bound at construction rather than redirected around it. @@ -267,6 +277,9 @@ def configured(spec: ConfigSpec) -> Iterator[Config]: config._inicache.clear() config._finalize_parse(args, decide_args=False) + # Read by ``unraisableexception`` at configure, cleanup and + # unconfigure time; harmless when that plugin was not opted into. + config.stash[gc_collect_iterations_key] = spec.gc_collect_iterations if spec.output is not None: # Must be stashed before configure: the terminal reporter binds # its stream when it is constructed, and must never bind ours. diff --git a/testing/test_ensemble.py b/testing/test_ensemble.py index bc361dc6949..b161562e39d 100644 --- a/testing/test_ensemble.py +++ b/testing/test_ensemble.py @@ -2,13 +2,16 @@ from __future__ import annotations +from collections.abc import Callable from collections.abc import Generator from contextlib import ExitStack +import gc import os from pathlib import Path import sys import types import unittest +from unittest import mock import warnings from _pytest._io import TerminalWriter @@ -20,6 +23,7 @@ from _pytest.ensemble import collect_tests from _pytest.ensemble import ConfigSpec from _pytest.ensemble import configured +from _pytest.ensemble import DEFAULT_PLUGINS from _pytest.ensemble import Ensemble from _pytest.ensemble import EnsembleModule from _pytest.ensemble import make_tmp_path_factory @@ -991,3 +995,45 @@ def test_it(tmp_path: Path) -> None: run_tests(test_it, spec=spec).assert_outcomes(passed=1) assert len(set(seen)) == 2 + + +class TestUnraisable: + """``unraisableexception`` is opt-in, and never collects by default.""" + + @staticmethod + def _counting_collect(counter: list[int]) -> Callable[..., int]: + real_collect = gc.collect + + def counting(*args: object, **kwargs: object) -> int: + counter[0] += 1 + return real_collect(*args, **kwargs) # type: ignore[arg-type] + + return counting + + def test_not_loaded_by_default(self) -> None: + assert "unraisableexception" not in DEFAULT_PLUGINS + + def test_opted_in_does_not_collect_by_default(self, tmp_path: Path) -> None: + """Loading the plugin must not make the ensemble walk the host heap.""" + + def test_it() -> None: + pass + + counter = [0] + spec = ConfigSpec(rootpath=tmp_path).with_plugins("unraisableexception") + with mock.patch.object(gc, "collect", self._counting_collect(counter)): + run_tests(test_it, spec=spec).assert_outcomes(passed=1) + assert counter[0] == 0 + + def test_iterations_are_honoured(self, tmp_path: Path) -> None: + def test_it() -> None: + pass + + counter = [0] + spec = ConfigSpec(rootpath=tmp_path, gc_collect_iterations=2).with_plugins( + "unraisableexception" + ) + with mock.patch.object(gc, "collect", self._counting_collect(counter)): + run_tests(test_it, spec=spec).assert_outcomes(passed=1) + # Two passes, at both the unconfigure and the cleanup site. + assert counter[0] == 4 From 3bfb5b4443760e8ca780a924a7379747560c5c55 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 23:13:36 +0200 Subject: [PATCH 27/30] testing: port the node warning tests to _pytest.ensemble `pytester.getitems` builds a config through `parseconfigure`, which does not get pytester's `gc_collect_iterations = 0` the way `inline_run` does -- so each of these three paid two full `gc.collect()` passes over the host suite's heap, ~243ms each, for two lines of assertion. `warn` needs a live config for the hook call, so the parametrized one collects inside an `Ensemble` block; the type-enforcement one only needs the item, so `collect_tests` is enough. Co-Authored-By: Claude Opus 5 (1M context) --- testing/test_nodes.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/testing/test_nodes.py b/testing/test_nodes.py index e976b9e6f11..0fc87c9f6a1 100644 --- a/testing/test_nodes.py +++ b/testing/test_nodes.py @@ -6,6 +6,8 @@ import warnings from _pytest import nodes +from _pytest.ensemble import collect_tests +from _pytest.ensemble import Ensemble from _pytest.outcomes import OutcomeException from _pytest.pytester import Pytester from _pytest.warning_types import PytestWarning @@ -66,25 +68,22 @@ def runtest(self): "warn_type, msg", [(DeprecationWarning, "deprecated"), (PytestWarning, "pytest")] ) def test_node_warn_is_no_longer_only_pytest_warnings( - pytester: Pytester, warn_type: type[Warning], msg: str + tmp_path: Path, warn_type: type[Warning], msg: str ) -> None: - items = pytester.getitems( - """ - def test(): - pass - """ - ) - with pytest.warns(warn_type, match=msg): - items[0].warn(warn_type(msg)) + def test() -> None: + pass + with Ensemble(test, rootpath=tmp_path) as ensemble: + items = ensemble.collect() + with pytest.warns(warn_type, match=msg): + items[0].warn(warn_type(msg)) -def test_node_warning_enforces_warning_types(pytester: Pytester) -> None: - items = pytester.getitems( - """ - def test(): - pass - """ - ) + +def test_node_warning_enforces_warning_types(tmp_path: Path) -> None: + def test() -> None: + pass + + items = collect_tests(test, rootpath=tmp_path) with pytest.raises( ValueError, match="warning must be an instance of Warning or subclass" ): From c725d0ddf95876ff91589105ae5c788eeb2210da Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 23:13:39 +0200 Subject: [PATCH 28/30] testing: port two collection tests to _pytest.ensemble `test_check_equality` and `test_unpacked_marks_added_to_keywords` are about node identity and mark unpacking, not about collecting a filesystem -- which is what the rest of this file tests and what an ensemble deliberately does not do. Note `Node` defines no `__eq__`: equality is identity, and the original passed because `collect_by_name` returned pytester's cached node both times. The port looks the name up twice in one collection rather than collecting twice, which would have compared two distinct objects with equal node ids and failed. `build_module` carries the module-level `pytestmark` as a keyword member, so the class and method marks nest exactly as in the source file. Co-Authored-By: Claude Opus 5 (1M context) --- testing/test_collection.py | 62 ++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/testing/test_collection.py b/testing/test_collection.py index 093162ddec4..f9ba1ceda66 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -13,6 +13,8 @@ from _pytest.compat import running_on_ci from _pytest.config import ExitCode +from _pytest.ensemble import build_module +from _pytest.ensemble import collect_tests from _pytest.fixtures import FixtureRequest from _pytest.main import _in_venv from _pytest.main import Session @@ -39,23 +41,32 @@ def test_collect_versus_item(self) -> None: assert not issubclass(Collector, Item) assert not issubclass(Item, Collector) - def test_check_equality(self, pytester: Pytester) -> None: - modcol = pytester.getmodulecol( - """ - def test_pass(): pass - def test_fail(): assert 0 - """ - ) - fn1 = pytester.collect_by_name(modcol, "test_pass") + def test_check_equality(self, tmp_path: Path) -> None: + def test_pass() -> None: + pass + + def test_fail() -> None: + assert 0 + + # ``Node`` defines no ``__eq__``, so equality is identity: looking a + # name up twice must yield the one node, as pytester's collection + # cache did for the original. + by_name = { + item.name: item + for item in collect_tests(test_pass, test_fail, rootpath=tmp_path) + } + fn1 = by_name["test_pass"] + fn2 = by_name["test_pass"] + modcol = fn1.parent assert isinstance(fn1, pytest.Function) - fn2 = pytester.collect_by_name(modcol, "test_pass") assert isinstance(fn2, pytest.Function) + assert isinstance(modcol, pytest.Module) assert fn1 == fn2 - assert fn1 != modcol + assert fn1 != modcol # type: ignore[comparison-overlap] assert hash(fn1) == hash(fn2) - fn3 = pytester.collect_by_name(modcol, "test_fail") + fn3 = by_name["test_fail"] assert isinstance(fn3, pytest.Function) assert not (fn1 == fn3) assert fn1 != fn3 @@ -63,12 +74,12 @@ def test_fail(): assert 0 for fn in fn1, fn2, fn3: assert isinstance(fn, pytest.Function) assert fn != 3 # type: ignore[comparison-overlap] - assert fn != modcol + assert fn != modcol # type: ignore[comparison-overlap] assert fn != [1, 2, 3] # type: ignore[comparison-overlap] assert [1, 2, 3] != fn # type: ignore[comparison-overlap] - assert modcol != fn + assert modcol != fn # type: ignore[comparison-overlap] - assert pytester.collect_by_name(modcol, "doesnotexist") is None + assert "doesnotexist" not in by_name def test_getparent_and_accessors(self, pytester: Pytester) -> None: modcol = pytester.getmodulecol( @@ -943,18 +954,17 @@ def test_method(self): pass assert item.keywords["kw"] == "method" assert len(item.keywords) == len(set(item.keywords)) - def test_unpacked_marks_added_to_keywords(self, pytester: Pytester) -> None: - item = pytester.getitem( - """ - import pytest - pytestmark = pytest.mark.foo - class TestClass: - pytestmark = pytest.mark.bar - def test_method(self): pass - test_method.pytestmark = pytest.mark.baz - """, - "test_method", - ) + def test_unpacked_marks_added_to_keywords(self, tmp_path: Path) -> None: + class TestClass: + pytestmark = pytest.mark.bar + + def test_method(self) -> None: + pass + + test_method.pytestmark = pytest.mark.baz # type: ignore[attr-defined] + + module = build_module("test_marks", TestClass, pytestmark=pytest.mark.foo) + (item,) = collect_tests(module, rootpath=tmp_path) assert isinstance(item, pytest.Function) cls = item.getparent(pytest.Class) assert cls is not None From c1a4a937c88030e23314afaa8f696ca9275b77ae Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 23:13:41 +0200 Subject: [PATCH 29/30] testing: port test_plugin_already_exists to _pytest.ensemble `configured()` runs configure and unconfigure around the block, which is what the original drove by hand. The terminal plugin is opted in so that `-p terminal` names one that is loaded already, and given a private buffer -- an ensemble's terminal reporter must never bind the outer test's stdout. Co-Authored-By: Claude Opus 5 (1M context) --- testing/test_session.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/testing/test_session.py b/testing/test_session.py index 3184dc272cd..e7b69420829 100644 --- a/testing/test_session.py +++ b/testing/test_session.py @@ -1,11 +1,13 @@ # mypy: allow-untyped-defs from __future__ import annotations +import io from pathlib import Path from _pytest.config import ExitCode from _pytest.ensemble import build_module from _pytest.ensemble import ConfigSpec +from _pytest.ensemble import configured from _pytest.ensemble import Ensemble from _pytest.ensemble import run_tests from _pytest.ensemble import RunRecord @@ -326,11 +328,15 @@ def test_plugin_specify(pytester: Pytester) -> None: # ensemble: same as above - ``-p`` is never consumed by an ensemble config. -def test_plugin_already_exists(pytester: Pytester) -> None: - config = pytester.parseconfig("-p", "terminal") - assert config.option.plugins == ["terminal"] - config._do_configure() - config._ensure_unconfigure() +def test_plugin_already_exists(tmp_path: Path) -> None: + # ``-p terminal`` names a plugin that is loaded already; configure and + # unconfigure must both survive it. The stream is a private buffer, since + # a loaded terminal plugin would otherwise bind the outer test's stdout. + spec = ConfigSpec( + rootpath=tmp_path, args=("-p", "terminal"), output=io.StringIO() + ).with_plugins("terminal") + with configured(spec) as config: + assert config.option.plugins == ["terminal"] # ensemble: --ignore excludes filesystem paths from a directory walk. From 5f8cadc8856a0dd06afb40a6e8314b0f2868e135 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 14 Aug 2026 23:13:43 +0200 Subject: [PATCH 30/30] testing: port test_fixturerequest_getmodulepath to _pytest.ensemble The subject is that `TopRequest.path` and `.fspath` agree with the module's, so it needs `legacypath` opted in but nothing from the filesystem. Co-Authored-By: Claude Opus 5 (1M context) --- testing/test_legacypath.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/testing/test_legacypath.py b/testing/test_legacypath.py index 3e71e6b190c..9846ac8eec4 100644 --- a/testing/test_legacypath.py +++ b/testing/test_legacypath.py @@ -4,6 +4,8 @@ from pathlib import Path from _pytest.compat import LEGACY_PATH +from _pytest.ensemble import collect_tests +from _pytest.ensemble import ConfigSpec from _pytest.fixtures import TopRequest from _pytest.legacypath import TempdirFactory from _pytest.legacypath import Testdir @@ -91,10 +93,15 @@ def test_cache_makedir(cache: pytest.Cache) -> None: dir.remove() -def test_fixturerequest_getmodulepath(pytester: pytest.Pytester) -> None: - modcol = pytester.getmodulecol("def test_somefunc(): pass") - (item,) = pytester.genitems([modcol]) +def test_fixturerequest_getmodulepath(tmp_path: Path) -> None: + def test_somefunc() -> None: + pass + + spec = ConfigSpec(rootpath=tmp_path).with_plugins("legacypath") + (item,) = collect_tests(test_somefunc, spec=spec) assert isinstance(item, pytest.Function) + modcol = item.parent + assert modcol is not None req = TopRequest(item, _ispytest=True) assert req.path == modcol.path assert req.fspath == modcol.fspath # type: ignore[attr-defined]