Experiment: take 864 of 1145 tests in 13 files off pytester

bench/ensemble_vs_pytester_examples.py

@@ -0,0 +1,159 @@
+"""Compare pytester and ``_pytest.ensemble`` on the *same real files*.
+
+``bench/ensemble_vs_pytester.py`` compares the two harnesses on generated
+sources, which measures the harness but says nothing about the usual shape
+of a pytester test: take a script that already exists under
+``testing/example_scripts``, put it somewhere, and run it.
+
+The two ways of doing that are:
+
+``copy`` + run
+    What ``pytester.copy_example`` does: put the file in the pytester
+    tmpdir, then run it with ``runpytest_inprocess``/``inline_run``/
+    ``runpytest_subprocess``. The script is imported from the copy, so its
+    reported paths are the copy's. The copy is done with ``shutil.copy``
+    rather than ``copy_example`` itself, which picks one fixed destination
+    and so cannot be called in a loop.
+``module_from_path``
+    ``module_from_path(path)`` imports the script where it lives - without
+    registering it in ``sys.modules`` or writing bytecode beside it - and
+    ``run_tests`` collects the resulting module. The script is never
+    copied, and its items report their real paths.
+
+Both run the same code, so the difference is the harness, not the work.
+
+Run with::
+
+    python bench/ensemble_vs_pytester_examples.py
+    pytest bench/ensemble_vs_pytester_examples.py -s    # equivalent
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+import contextlib
+import io
+from pathlib import Path
+import shutil
+import time
+
+from _pytest.ensemble import module_from_path
+from _pytest.ensemble import run_tests
+from _pytest.pytester import Pytester
+
+
+#: Real example scripts, relative to ``testing/example_scripts``. Examples
+#: that deliberately fail at import time are not benchmark subjects - the
+#: two harnesses would not be doing the same work.
+EXAMPLES = [
+    "fixtures/fill_fixtures/test_funcarg_basic.py",
+    "fixtures/fill_fixtures/test_funcarg_lookup_modulelevel.py",
+    "fixtures/fill_fixtures/test_funcarg_lookup_classlevel.py",
+    "fixtures/fill_fixtures/test_extend_fixture_module_class.py",
+    "unittest/test_setup_skip.py",
+    "unittest/test_setup_skip_class.py",
+    "unittest/test_setup_skip_module.py",
+]
+
+ITERATIONS = {"subprocess": 3}
+DEFAULT_ITERATIONS = 10
+
+EXAMPLE_ROOT = Path(__file__).parent.parent / "testing" / "example_scripts"
+
+
+def _files(path: Path) -> int:
+    """Files below *path*, ignoring bytecode caches."""
+    return sum(
+        1
+        for p in path.rglob("*")
+        if p.is_file() and "__pycache__" not in p.parts and p.suffix != ".pyc"
+    )
+
+
+def _measure(
+    run: Callable[[int], None], iterations: int, watched: Path
+) -> tuple[float, float]:
+    """Return (seconds, files created below *watched*) per iteration."""
+    with contextlib.redirect_stdout(io.StringIO()):
+        run(0)  # warm up
+        before = _files(watched)
+        start = time.perf_counter()
+        for i in range(1, iterations + 1):
+            run(i)
+        elapsed = time.perf_counter() - start
+    return elapsed / iterations, (_files(watched) - before) / iterations
+
+
+def _arms(pytester: Pytester, rel: str) -> dict[str, Callable[[int], None]]:
+    source = EXAMPLE_ROOT / rel
+
+    def copy(i: int) -> Path:
+        # Each iteration gets its own directory, which is what a real
+        # pytester test gets too.
+        target = pytester.path / f"run{i}"
+        target.mkdir(exist_ok=True)
+        dest = target / source.name
+        shutil.copy(source, dest)
+        return dest
+
+    def subprocess_(i: int) -> None:
+        pytester.runpytest_subprocess(copy(i))
+
+    def inprocess(i: int) -> None:
+        pytester.runpytest_inprocess(copy(i))
+
+    def inline(i: int) -> None:
+        pytester.inline_run(copy(i))
+
+    def copy_only(i: int) -> None:
+        copy(i)
+
+    def ensemble(i: int) -> None:
+        run_tests(module_from_path(source), rootpath=source.parent)
+
+    def import_only(i: int) -> None:
+        module_from_path(source)
+
+    return {
+        "subprocess": subprocess_,
+        "inprocess": inprocess,
+        "inline": inline,
+        "ensemble": ensemble,
+        "copy only": copy_only,
+        "import only": import_only,
+    }
+
+
+def test_ensemble_vs_pytester_on_examples(pytester: Pytester) -> None:
+    """Not an assertion test - run with ``-s`` and read the table."""
+    print()
+    totals: dict[str, float] = {}
+    for rel in EXAMPLES:
+        results = {}
+        for name, run in _arms(pytester, rel).items():
+            iterations = ITERATIONS.get(name, DEFAULT_ITERATIONS)
+            results[name] = _measure(run, iterations, pytester.path)
+            totals[name] = totals.get(name, 0.0) + results[name][0]
+
+        baseline = results["ensemble"][0]
+        print(f"\n{rel}")
+        print(f"  {'arm':<12} {'per run':>10} {'files':>8} {'vs ensemble':>13}")
+        for name, (seconds, files) in results.items():
+            ratio = f"{seconds / baseline:.1f}x" if baseline else "n/a"
+            print(f"  {name:<12} {seconds * 1000:>8.2f}ms {files:>8.1f} {ratio:>13}")
+
+    print(f"\n{'=' * 56}\nacross all {len(EXAMPLES)} examples")
+    base = totals["ensemble"]
+    print(f"  {'arm':<12} {'total':>10} {'vs ensemble':>13}")
+    for name, seconds in totals.items():
+        print(f"  {name:<12} {seconds * 1000:>8.2f}ms {seconds / base:>12.1f}x")
+    print(
+        "\n(files counted below the pytester tmpdir, excluding __pycache__;\n"
+        " the ensemble arm writes nothing and never leaves the source tree)"
+    )
+
+
+if __name__ == "__main__":
+    import pytest
+
+    raise SystemExit(pytest.main([__file__, "-s", "-q", "-p", "no:randomly"]))

changelog/14809.improvement.rst

@@ -0,0 +1,6 @@
+Rendering a traceback entry now parses only the enclosing block rather than the whole source file.
+
+Locating the failing statement requires parsing source into an AST, and that was done for the
+entire file once per rendered entry -- so a failure in a large test module paid for parsing every
+line of it, repeatedly. The parse is now restricted to the block the frame belongs to, falling
+back to the full file where the block cannot be determined.

pyproject.toml

@@ -369,7 +369,7 @@ disable = [
 ]
 
 [tool.codespell]
-ignore-words-list = "afile,asend,asser,assertio,feld,hove,ned,noes,notin,paramete,parth,tesults,varius,wil"
+ignore-words-list = "afile,asend,asser,assertio,feld,hellow,hove,ned,noes,notin,paramete,parth,tesults,varius,wil"
 skip = "AUTHORS,*/plugin_list.rst"
 write-changes = true
 

src/_pytest/_code/code.py

@@ -13,6 +13,7 @@
 from pathlib import Path
 import re
 import sys
+import tokenize
 from traceback import extract_tb
 from traceback import format_exception
 from traceback import format_exception_only
@@ -284,27 +285,48 @@ def getfirstlinesource(self) -> int:
         return self.frame.code.firstlineno
 
     def getsource(
-        self, astcache: dict[str | Path, ast.AST] | None = None
+        self, astcache: dict[tuple[str | Path, int], ast.AST] | None = None
     ) -> Source | None:
         """Return failing source code."""
         # we use the passed in astcache to not reparse asttrees
         # within exception info printing
         source = self.frame.code.fullsource
         if source is None:
             return None
+        start = self.getfirstlinesource()
+        # Narrow the parse to the enclosing block: locating one statement is
+        # otherwise O(file), and it is paid once per rendered traceback entry.
+        block, offset = source, 0
+        try:
+            candidate = Source(inspect.getblock(source.raw_lines[start:]))
+        except (OSError, IndentationError, tokenize.TokenError, SyntaxError):
+            pass
+        else:
+            # The block can fall short of the frame: getblock only walks an
+            # indented suite for def/class/decorated code and otherwise stops
+            # at the first logical line, so a module-level frame lands here,
+            # as does exec'd or generated code whose lines do not match.
+            # Only narrow when the reported line is actually inside.
+            if start <= self.lineno < start + len(candidate.lines):
+                block, offset = candidate, start
+        # The key carries the offset, not `start`: whether the block was
+        # narrowed depends on the line being reported, so two entries in the
+        # same function can disagree, and a cached tree must never be paired
+        # with linenos it was not parsed from.
         key = astnode = None
         if astcache is not None:
-            key = self.frame.code.path
-            if key is not None:
+            path = self.frame.code.path
+            if path is not None:
+                key = (path, offset)
                 astnode = astcache.get(key, None)
-        start = self.getfirstlinesource()
         try:
             astnode, _, end = getstatementrange_ast(
-                self.lineno, source, astnode=astnode
+                self.lineno - offset, block, astnode=astnode
             )
         except SyntaxError:
             end = self.lineno + 1
         else:
+            end += offset
             if key is not None and astcache is not None:
                 astcache[key] = astnode
         return source[start:end]
@@ -893,7 +915,7 @@ class ExceptionInfoFormatter:
     truncate_args: bool = True
     chain: bool = True
 
-    astcache: dict[str | Path, ast.AST] = dataclasses.field(
+    astcache: dict[tuple[str | Path, int], ast.AST] = dataclasses.field(
         default_factory=dict, init=False, repr=False
     )
 

src/_pytest/ensemble/__init__.py

@@ -70,6 +70,7 @@
 from _pytest.ensemble.config import ConfigSpec
 from _pytest.ensemble.config import configured
 from _pytest.ensemble.config import DEFAULT_PLUGINS
+from _pytest.ensemble.config import make_tmp_path_factory
 from _pytest.ensemble.results import ensure_recorder
 from _pytest.ensemble.results import ItemRecord
 from _pytest.ensemble.results import run_items
@@ -99,6 +100,7 @@
     "collect_sources",
     "collect_tests",
     "configured",
+    "make_tmp_path_factory",
     "module_from_path",
     "run_items",
     "run_tests",

src/_pytest/ensemble/config.py

@@ -14,11 +14,14 @@
 
 from _pytest.config import Config
 from _pytest.config import essential_plugins
+from _pytest.config import hookimpl
 from _pytest.config import PytestPluginManager
 from _pytest.config.findpaths import ConfigValue
 from _pytest.config.findpaths import parse_override_ini
 from _pytest.stash import StashKey
 from _pytest.terminal import terminal_file_key
+from _pytest.tmpdir import TempPathFactory
+from _pytest.unraisableexception import gc_collect_iterations_key
 
 
 #: Warnings raised while the ensemble config was being configured or
@@ -33,6 +36,12 @@
 #: excluding everything that renders output, captures io, or installs
 #: process-global state (terminal, capture, cacheprovider, assertion,
 #: debugging, faulthandler, logging, threadexception, unraisableexception, ...).
+#:
+#: ``tmpdir`` is not here either: it would allocate the ensemble its own
+#: numbered base temp directory under the global ``pytest-of-<user>`` root,
+#: which means scanning that root once per ensemble and leaving a directory
+#: behind for every one ever configured. It is loaded only when
+#: :attr:`ConfigSpec.tmp_path_factory` supplies a preconfigured factory.
 DEFAULT_PLUGINS: Final[tuple[str, ...]] = (
     *essential_plugins,  # mark, main, runner, fixtures, helpconfig
     "python",
@@ -42,7 +51,6 @@
     "unittest",
     "monkeypatch",
     "recwarn",
-    "tmpdir",
     # Not for the rewriting - that is installed from Config._preparse, which
     # an ensemble never runs - but for the failure *explanation*. Without
     # this plugin ``assertion.util._reprcompare`` stays bound to whatever the
@@ -86,6 +94,23 @@ class ConfigSpec:
     #: Not supported yet; ensemble configs never load conftest files.
     load_conftests: bool = False
 
+    #: Preconfigured temp path factory. Supplying one loads the ``tmpdir``
+    #: plugin and binds this factory instead of the one it would build from
+    #: the config, so an ensemble's ``tmp_path`` lives wherever the caller
+    #: decided - normally inside the *host* test's own ``tmp_path``, which
+    #: costs no root scan and is cleaned up with the host. Build one with
+    #: :func:`make_tmp_path_factory`.
+    tmp_path_factory: TempPathFactory | None = None
+
+    #: How many ``gc.collect()`` passes ``unraisableexception`` makes, when
+    #: that plugin is opted into at all. It is not in :data:`DEFAULT_PLUGINS`,
+    #: and even when loaded an ensemble does not collect by default: the heap
+    #: it would walk is the *host* process's, so a full pass costs whatever
+    #: the host happens to be holding rather than anything the ensemble owns.
+    #: Raise it only for a test that needs finalizers flushed before an
+    #: unraisable exception can surface.
+    gc_collect_iterations: int = 0
+
     #: Stream the terminal plugin writes to, when it is loaded at all. An
     #: ensemble must never be given the stdout of whatever is running it,
     #: so this is bound at construction rather than redirected around it.
@@ -116,6 +141,46 @@ def without_plugins(self, *names: str) -> ConfigSpec:
         return self.replace(plugins=tuple(p for p in self.plugins if p not in names))
 
 
+def make_tmp_path_factory(basetemp: pathlib.Path) -> TempPathFactory:
+    """Build a :class:`TempPathFactory` for an ensemble, rooted at *basetemp*.
+
+    *basetemp* is treated the way ``--basetemp`` is: it is removed if it
+    already exists, then created. Pass a path that is yours to destroy - a
+    subdirectory of the host test's ``tmp_path`` is the intended use.
+
+    The point is what this does *not* do. A factory built from a config
+    allocates a numbered directory under the global ``pytest-of-<user>``
+    root, which scans that root and every sibling run's leftovers, registers
+    a cleanup lock, and leaves the directory behind afterwards. Ensembles are
+    built in the hundreds, so paying that per ensemble is not viable.
+    """
+    return TempPathFactory(
+        given_basetemp=basetemp,
+        trace=lambda *args, **kwargs: None,
+        retention_count=0,
+        retention_policy="all",
+        _ispytest=True,
+    )
+
+
+class _BindTmpPathFactory:
+    """Bind a caller-supplied factory over the one ``tmpdir`` builds.
+
+    The ``tmpdir`` plugin creates its factory in ``pytest_configure``; this
+    runs last and replaces it, so the fixtures, the retention handling and
+    the ``pytest_sessionfinish`` cleanup all stay the plugin's own.
+    """
+
+    __pytest_no_fixtures__ = True
+
+    def __init__(self, factory: TempPathFactory) -> None:
+        self._factory = factory
+
+    @hookimpl(trylast=True)
+    def pytest_configure(self, config: Config) -> None:
+        config._tmp_path_factory = self._factory  # type: ignore[attr-defined]
+
+
 def _own(value: object) -> ConfigValue:
     """Wrap a spec's ini value in a ConfigValue the config may own.
 
@@ -176,6 +241,9 @@ def configured(spec: ConfigSpec) -> Iterator[Config]:
     try:
         for name in spec.plugins:
             pluginmanager.import_plugin(name)
+        if spec.tmp_path_factory is not None:
+            pluginmanager.import_plugin("tmpdir")
+            pluginmanager.register(_BindTmpPathFactory(spec.tmp_path_factory))
         for plugin in spec.extra_plugins:
             if isinstance(plugin, str):
                 pluginmanager.import_plugin(plugin)
@@ -209,6 +277,9 @@ def configured(spec: ConfigSpec) -> Iterator[Config]:
             config._inicache.clear()
 
         config._finalize_parse(args, decide_args=False)
+        # Read by ``unraisableexception`` at configure, cleanup and
+        # unconfigure time; harmless when that plugin was not opted into.
+        config.stash[gc_collect_iterations_key] = spec.gc_collect_iterations
         if spec.output is not None:
             # Must be stashed before configure: the terminal reporter binds
             # its stream when it is constructed, and must never bind ours.