From 64e686f28e3b06bd48b69015fd77b08e4a07d8dd Mon Sep 17 00:00:00 2001 From: majin0824 Date: Tue, 25 Aug 2026 04:21:42 -0700 Subject: [PATCH] Fix: honor disabled xdist in L2 dispatcher Use pytest plugin-manager state instead of package importability. An explicit plugin disable keeps L2 children serial. Resource-phase concurrency remains available. Tests cover both command paths and the docs define the CLI contract. --- conftest.py | 32 ++++------ docs/testing.md | 5 ++ tests/ut/py/test_l2_dispatch.py | 101 ++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 tests/ut/py/test_l2_dispatch.py diff --git a/conftest.py b/conftest.py index 003963c8a4..69a8c0cc14 100644 --- a/conftest.py +++ b/conftest.py @@ -1057,26 +1057,20 @@ def _on_done(res): # ----- Phase 2: L2 per-runtime subprocess ----- l2_runtimes = _collect_st_runtimes(session.items, level=2) l2_failed = False - # When we have more than one device, enable pytest-xdist so the L2 phase - # spreads classes across devices. Each xdist worker slices --device 0-7 - # down to one id in its own pytest_configure (above) and the st_worker - # fixture is session-scoped inside the worker — one ChipWorker per (runtime, - # device), reused across every class assigned to that worker. - xdist_available = False - if max_parallel > 1: - try: - import xdist # noqa: F401,PLC0415 - - xdist_available = True - except ImportError: - print( - "\n[warning] -j > 1 but pytest-xdist not installed; " - "falling back to serial L2 phase. pip install pytest-xdist to enable.\n", - flush=True, - ) + # An active pytest-xdist plugin spreads L2 classes across devices. Each + # worker slices --device 0-7 down to one id in pytest_configure (above), + # and the session-scoped st_worker fixture reuses one ChipWorker per + # (runtime, device). + xdist_active = max_parallel > 1 and cfg.pluginmanager.hasplugin("xdist") + if max_parallel > 1 and not xdist_active and not cfg.pluginmanager.is_blocked("xdist"): + print( + "\n[warning] --max-parallel > 1 but the pytest-xdist plugin is not active; " + "falling back to serial L2 phase. Install or enable pytest-xdist to use L2 parallelism.\n", + flush=True, + ) for rt in l2_runtimes: cmd = base_args + ["--runtime", rt, "--level", "2"] - if xdist_available: + if xdist_active: cmd += ["-n", str(max_parallel), "--dist", "loadfile"] # Per-runtime sink for the in-process poison guards (issue #1110). Each # xdist worker appends the classes it poison-skips; we re-run them in a @@ -1091,7 +1085,7 @@ def _on_done(res): # need to buffer their stdout — we can stream it directly through # the group markers. ``::group::`` on its own line before the run # opens the fold; ``::endgroup::`` after closes it. - label = f"L2 {rt}" + (f" [-n {max_parallel}]" if xdist_available else "") + label = f"L2 {rt}" + (f" [-n {max_parallel}]" if xdist_active else "") start = time.monotonic() print(f"::group::{label}", flush=True) result = subprocess.run(cmd, check=False, cwd=cwd, env=run_env) diff --git a/docs/testing.md b/docs/testing.md index 36f5f4cc37..9c0a7570a4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -852,6 +852,11 @@ pytest --runtime --level 2 --device 8-11 -n 4 --dist loadfile `pytest-xdist` starts 4 workers (`gw0`..`gw3`). Each worker's `pytest_configure` slices `--device 8-11` down to a single id (`gw0` → `8`, `gw1` → `9`, ...), and `st_worker` is session-scoped, so the worker initializes exactly one `ChipWorker(device=N)` and reuses it for every L2 class routed to it. `--dist loadfile` keeps all cases from one test file on the same worker, amortizing any file-level setup cost. +An explicit `-p no:xdist` is authoritative for the L2 children inherited from +the top-level invocation. The dispatcher omits `-n` and `--dist`, so each L2 +runtime subprocess executes serially. Resource-phase subprocess concurrency +continues to follow `--max-parallel` because it does not use pytest-xdist. + ### L2 phase — standalone fanout Standalone (`python test_*.py -d 8-11`) uses the same scheduler module: classes are round-robin assigned to `len(device_ids)` chunks, one subprocess per chunk launched with a single device and explicit `--case ClassName::` selectors. diff --git a/tests/ut/py/test_l2_dispatch.py b/tests/ut/py/test_l2_dispatch.py new file mode 100644 index 0000000000..69bb86eb5b --- /dev/null +++ b/tests/ut/py/test_l2_dispatch.py @@ -0,0 +1,101 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Tests for L2 dispatcher child-command construction.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +_ROOT = Path(__file__).resolve().parents[3] + + +def _load_root_conftest(): + spec = importlib.util.spec_from_file_location("_root_conftest_l2_dispatch", _ROOT / "conftest.py") + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class _Config: + def __init__(self, *, xdist_active: bool, xdist_blocked: bool, disable_arg: bool = False): + args = ["first.py", "second.py"] + if disable_arg: + args.extend(["-p", "no:xdist"]) + args.extend(["--max-parallel", "2"]) + self.invocation_params = SimpleNamespace( + args=tuple(args), + dir=_ROOT, + ) + self.pluginmanager = SimpleNamespace( + hasplugin=lambda name: name == "xdist" and xdist_active, + is_blocked=lambda name: name == "xdist" and xdist_blocked, + ) + + def getoption(self, name, default=None): + return { + "--device": "0-1", + "--exitfirst": False, + "--platform": "a2a3sim", + "--manual": "include", + "--max-parallel": "2", + }.get(name, default) + + +def test_l2_dispatch_does_not_add_xdist_options_when_plugin_is_disabled(monkeypatch): + cf = _load_root_conftest() + commands = [] + + def fake_run(command, **_kwargs): + commands.append(command) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(cf.subprocess, "run", fake_run) + config = _Config(xdist_active=False, xdist_blocked=True, disable_arg=True) + items = [ + SimpleNamespace(cls=SimpleNamespace(_st_runtime="host_build_graph", _st_level=2)), + SimpleNamespace(cls=SimpleNamespace(_st_runtime="tensormap_and_ringbuffer", _st_level=2)), + ] + session = SimpleNamespace(config=config, items=items, testsfailed=0, testscollected=0) + + assert cf._dispatch_test_phases(session, []) is True + assert len(commands) == 2 + for command in commands: + assert "no:xdist" in command + assert "-n" not in command + assert "--dist" not in command + + +def test_l2_dispatch_adds_xdist_options_when_plugin_is_active(monkeypatch): + cf = _load_root_conftest() + commands = [] + + def fake_run(command, **_kwargs): + commands.append(command) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(cf.subprocess, "run", fake_run) + config = _Config(xdist_active=True, xdist_blocked=False) + items = [SimpleNamespace(cls=SimpleNamespace(_st_runtime="host_build_graph", _st_level=2))] + session = SimpleNamespace(config=config, items=items, testsfailed=0, testscollected=0) + + assert cf._dispatch_test_phases(session, []) is True + assert len(commands) == 1 + assert commands[0][-8:] == [ + "--runtime", + "host_build_graph", + "--level", + "2", + "-n", + "2", + "--dist", + "loadfile", + ]