Fix: address scheduler profiling and xdist follow-ups

conftest.py

@@ -901,7 +901,7 @@ def _base_pytest_argv(session, *, strip_options=()):
 
 
 def _resolve_max_parallel(cfg, platform: str, device_ids: list[int]) -> int:
-    """Parse the -j/--max-parallel CLI value; 'auto' → platform-aware default."""
+    """Parse --max-parallel; 'auto' selects a platform-aware default."""
     raw = cfg.getoption("--max-parallel", default="auto")
     if raw in (None, "", "auto"):
         return _ps.default_max_parallel(platform or "", device_ids)
@@ -914,6 +914,56 @@ def _resolve_max_parallel(cfg, platform: str, device_ids: list[int]) -> int:
     return val
 
 
+def _invocation_has_option(cfg, options):
+    """Return whether the original command line contains one of ``options``."""
+    for arg in cfg.invocation_params.args:
+        text = str(arg)
+        if text in options or any(text.startswith(f"{option}=") for option in options):
+            return True
+    return False
+
+
+def _l2_xdist_options(cfg, max_parallel: int):
+    """Return whether L2 uses xdist, child defaults, and the worker label.
+
+    ``numprocesses`` arrives here as ``None`` or ``0``. A top-level ``-n N``
+    with N > 0 puts xdist in distribution mode, and its ``pytest_runtestloop``
+    runs the whole session before this dispatcher's own hook is reached, so a
+    positive worker count is only ever seen from a direct caller.
+
+    ``--pdb`` is serial whatever ``numprocesses`` says: xdist zeroes the worker
+    count only for ``-n auto`` / ``-n logical``, so a bare ``--pdb`` arrives
+    with it unset — and the L2 child inherits ``--pdb``, which xdist rejects
+    with a usage error as soon as ``-n`` puts the child in distribution mode.
+    """
+    plugin_active = cfg.pluginmanager.hasplugin("xdist")
+    if not plugin_active or cfg.getoption("usepdb", default=False):
+        return False, [], None
+
+    requested_workers = cfg.getoption("numprocesses", default=None)
+    if requested_workers == 0:
+        return False, [], None
+
+    if requested_workers is None and max_parallel <= 1:
+        return False, [], None
+
+    options = []
+    if requested_workers is None:
+        options.extend(["-n", str(max_parallel)])
+    # xdist rewrites its effective default from ``no`` to ``load`` as soon as
+    # ``-n`` is active. Inspect the original command line so that derived
+    # ``load`` does not masquerade as an explicit user choice. ``-d`` is
+    # xdist's shortcut for ``--dist=load`` and overrides a ``--dist`` passed
+    # alongside it, so it is an explicit choice too. The scan sees the
+    # invocation argv only: a ``--dist`` coming from ini ``addopts`` reads as
+    # unset here, and ``addopts`` is prepended to the child's argv, so the
+    # appended default wins over it.
+    if not _invocation_has_option(cfg, {"--dist", "-d"}):
+        options.extend(["--dist", "loadfile"])
+    worker_label = requested_workers if requested_workers is not None else max_parallel
+    return True, options, worker_label
+
+
 def _emit_group(header: str, body: str) -> None:
     """Print a GitHub Actions collapsible group around ``body``.
 
@@ -1061,8 +1111,8 @@ def _on_done(res):
     # worker slices --device 0-7 down to one id in pytest_configure (above),
     # and the session-scoped st_worker fixture reuses one ChipWorker per
     # (runtime, device).
-    xdist_active = max_parallel > 1 and cfg.pluginmanager.hasplugin("xdist")
-    if max_parallel > 1 and not xdist_active and not cfg.pluginmanager.is_blocked("xdist"):
+    xdist_active, xdist_options, xdist_workers = _l2_xdist_options(cfg, max_parallel)
+    if max_parallel > 1 and not cfg.pluginmanager.hasplugin("xdist") and not cfg.pluginmanager.is_blocked("xdist"):
         print(
             "\n[warning] --max-parallel > 1 but the pytest-xdist plugin is not active; "
             "falling back to serial L2 phase. Install or enable pytest-xdist to use L2 parallelism.\n",
@@ -1071,7 +1121,7 @@ def _on_done(res):
     for rt in l2_runtimes:
         cmd = base_args + ["--runtime", rt, "--level", "2"]
         if xdist_active:
-            cmd += ["-n", str(max_parallel), "--dist", "loadfile"]
+            cmd += xdist_options
         # Per-runtime sink for the in-process poison guards (issue #1110). Each
         # xdist worker appends the classes it poison-skips; we re-run them in a
         # fresh subprocess after this one exits so they don't silently lose
@@ -1085,7 +1135,7 @@ def _on_done(res):
         # need to buffer their stdout — we can stream it directly through
         # the group markers. ``::group::`` on its own line before the run
         # opens the fold; ``::endgroup::`` after closes it.
-        label = f"L2 {rt}" + (f" [-n {max_parallel}]" if xdist_active else "")
+        label = f"L2 {rt}" + (f" [-n {xdist_workers}]" if xdist_active else "")
         start = time.monotonic()
         print(f"::group::{label}", flush=True)
         result = subprocess.run(cmd, check=False, cwd=cwd, env=run_env)

docs/dfx/chip-swimlane-profiling.md

@@ -50,10 +50,11 @@ available.
   `release` / `dummy` / `early_dispatch` / `drain` / `graph_prepare`), plus
   nested phases.
   In `tensormap_and_ringbuffer`, `resolve` is nested within `complete` or
-  `dummy`; in `host_build_graph`, `resolve`, `async_poll`, and `dummy` are
-  standalone, mutually exclusive phases on the dedicated P thread. HBG
-  `resolve` uses that P thread's main scheduler lane; TMR's nested `resolve`
-  uses a sibling scheduler sub-lane. The drain sub-phases are nested within
+  `dummy`; in `host_build_graph`, `resolve_standalone`, `async_poll`, and
+  `dummy` are standalone, mutually exclusive phases on the dedicated P
+  thread. The converter renders `resolve_standalone` as `resolve` on that P
+  thread's main scheduler lane; TMR's nested `resolve` uses a sibling
+  scheduler sub-lane. The drain sub-phases are nested within
   their `drain` bar,
   and two **separate-lane**
   phases (`dummy_task` and `predicated_skip`, sampled immediately before
@@ -268,7 +269,8 @@ field but render differently in Perfetto:
 | `early_dispatch` | outer | sched | blocks staged by speculative early-dispatch this pass |
 | `drain` | outer | sched | blocks staged by this thread's global sync-start drain pass |
 | `graph_prepare` | outer | sched | Graph Definition nodes expanded this pass |
-| `resolve` | inner (TMR); P-thread outer (HBG) | TMR sched sub-lane; HBG P sched lane | consumers visited in `on_task_complete` (TMR); completed SPSC slots (HBG) |
+| `resolve` | inner (TMR) | TMR sched sub-lane | consumers visited in `on_task_complete` |
+| `resolve_standalone` | P-thread outer (HBG); rendered as `resolve` | HBG P sched lane | completed SPSC slots |
 | `drain_prepare` | inner | sched, nested in `drain` | subtasks prepared for global sync-start publication |
 | `drain_publish` | inner | sched, nested in `drain` | subtasks published during global sync-start staging |
 | `dummy_task` | separate-lane | Worker View AICPU_N (pid=4) | one dummy entering `on_task_complete()`; full identity is in `task_id` |
@@ -709,7 +711,7 @@ Both architectures use split phase streams:
   several (e.g. Complete, AsyncPoll, Dispatch, Release, plus Resolve).
   `ChipSwimlaneSchedPhaseKind` spans the outer phases
   (Complete, Dispatch, Release, Dummy, EarlyDispatch, AsyncPoll, Drain,
-  GraphPrepare), runtime-specific Resolve, the inner drain phases
+  GraphPrepare, ResolveStandalone), TMR's inner Resolve, the inner drain phases
   (DrainPrepare, DrainPublish), and
   the separate-lane markers (DummyTask, PredicatedSkip) — see §3.2 for how
   each is rendered. Carries loop_iter + tasks_processed + pop_hit /

docs/dfx/sched-overhead-model.md

@@ -113,9 +113,14 @@ python -m simpler_setup.tools.sched_overhead_analysis \
 
 For TMR captures, Resolve is nested in Complete or Dummy and is excluded from
 the phase total to avoid double counting. For HBG captures, Resolve is
-standalone work on the P thread and is included. Empty HBG async polling is
-reported as compact `AsyncPoll(0)` bars, so its measured CPU cost contributes
-to the scheduler budget instead of being reconstructed as idle. HBG's S
+standalone work on the P thread and is included. The two are told apart by the
+`resolve_standalone` phase discriminator the HBG P thread emits, not by
+timestamp containment; a capture predating that discriminator falls back to
+containment, where a Resolve ending exactly at its Complete or Dummy parent's
+end counts as standalone. Both spellings report under the `resolve` label.
+Empty HBG async polling is reported as compact `AsyncPoll(0)` bars, so its
+measured CPU cost contributes to the scheduler budget instead of being
+reconstructed as idle. HBG's S
 threads detect AICore FIN and dispatch work, while its P thread resolves
 completion state and dependencies. Part 5 reports their loop rates separately;
 the Tail-OH-to-loop comparison uses only S-thread loops because Tail OH ends at

docs/testing.md

@@ -852,10 +852,27 @@ pytest --runtime <rt> --level 2 --device 8-11 -n 4 --dist loadfile
 
 `pytest-xdist` starts 4 workers (`gw0`..`gw3`). Each worker's `pytest_configure` slices `--device 8-11` down to a single id (`gw0` → `8`, `gw1` → `9`, ...), and `st_worker` is session-scoped, so the worker initializes exactly one `ChipWorker(device=N)` and reuses it for every L2 class routed to it. `--dist loadfile` keeps all cases from one test file on the same worker, amortizing any file-level setup cost.
 
-An explicit `-p no:xdist` is authoritative for the L2 children inherited from
-the top-level invocation. The dispatcher omits `-n` and `--dist`, so each L2
-runtime subprocess executes serially. Resource-phase subprocess concurrency
-continues to follow `--max-parallel` because it does not use pytest-xdist.
+An explicit `-p no:xdist` or `-n 0` is authoritative for the L2 children
+inherited from the top-level invocation. The dispatcher does not append xdist
+options in either case, so each L2 runtime subprocess executes serially. `--pdb`
+reaches the same state, and reaches it on its own gate rather than through the
+worker count: xdist zeroes that count only for `-n auto` / `-n logical`, while
+a bare `--pdb` leaves it unset — and since the L2 child inherits `--pdb`, an
+appended `-n` would make xdist reject the child with
+`--pdb is incompatible with distributing tests`. An explicit top-level `--dist`
+(or its `-d` shortcut) is preserved too; the dispatcher adds only the options
+the invocation left unset.
+
+A top-level `-n N` with N > 0 is a different case: it puts xdist in
+distribution mode, whose `pytest_runtestloop` claims the session before this
+dispatcher's hook runs, so the phase split does not happen at all and every
+collected level runs in one flat xdist fanout. Size L2 parallelism with
+`--max-parallel`, not with a top-level `-n`.
+
+Resource-phase subprocess concurrency continues to follow `--max-parallel`
+because it does not use pytest-xdist. If plugin autoloading is disabled without
+blocking xdist explicitly, the dispatcher warns before falling back to serial
+L2 execution.
 
 ### L2 phase — standalone fanout
 

simpler_setup/scene_test.py

@@ -2211,17 +2211,17 @@ def run_module(module_name):  # noqa: PLR0912, PLR0915 -- CLI parsing + dispatch
         # slot but the dispatcher doesn't actually run tests here.
         args.device = device_ids[0]
 
-        # Resolve -j (max parallel) — 'auto' is CPU-aware on sim, device-count on hardware.
+        # Resolve --max-parallel; 'auto' is CPU-aware on sim and device-count on hardware.
         if args.max_parallel in (None, "", "auto"):
             args.max_parallel = default_max_parallel(args.platform, device_ids)
         else:
             try:
                 args.max_parallel = int(args.max_parallel)
             except (TypeError, ValueError):
-                print(f"ERROR: -j must be 'auto' or an integer, got {args.max_parallel!r}", file=sys.stderr)
+                print(f"ERROR: --max-parallel must be 'auto' or an integer, got {args.max_parallel!r}", file=sys.stderr)
                 sys.exit(2)
             if args.max_parallel < 1:
-                print(f"ERROR: -j must be >= 1, got {args.max_parallel}", file=sys.stderr)
+                print(f"ERROR: --max-parallel must be >= 1, got {args.max_parallel}", file=sys.stderr)
                 sys.exit(2)
         # Profiling + parallelism is safe: each test case sets its own
         # `output_prefix` on CallConfig (see run_class_cases) so diagnostic
@@ -2448,9 +2448,9 @@ def _on_done(res):
     l2_failed = False
     for rt in sorted(l2_by_runtime):
         classes = l2_by_runtime[rt]
-        # Chunk count = min(-j, number of classes). We intentionally do NOT
+        # Chunk count = min(--max-parallel, number of classes). We intentionally do NOT
         # include len(device_ids) here: each chunk uses 1 device and at most
-        # max_parallel chunks run concurrently, so a pool bigger than -j just
+        # max_parallel chunks run concurrently, so a larger device pool just
         # leaves unused ids. Fewer, larger chunks also amortize ChipWorker
         # init (layer-4 reuse) over more cases.
         n = min(args.max_parallel, len(classes))

simpler_setup/tools/sched_overhead_analysis.py

@@ -35,15 +35,13 @@
 from collections import defaultdict
 from pathlib import Path
 
-_SCHED_OUTER_PHASES = (
-    "complete",
-    "async_poll",
-    "dispatch",
-    "release",
-    "dummy",
-    "early_dispatch",
-    "drain",
-    "graph_prepare",
+from simpler_setup.tools.scheduler_phase_records import (
+    SCHED_OUTER_PHASES as _SCHED_OUTER_PHASES,
+)
+from simpler_setup.tools.scheduler_phase_records import (
+    canonical_sched_phase,
+    nested_resolve_record_ids,
+    scheduler_thread_role,
 )
 
 
@@ -229,23 +227,12 @@ def parse_scheduler_from_json_phases(data):  # noqa: PLR0912
         # thread. Count only the latter so the P thread is not dropped without
         # double-counting TMR's nested bars.
         outer_recs = [r for r in records if r.get("phase") in _SCHED_OUTER_PHASES]
-        resolve_parents = sorted(
-            (
-                (r.get("start_time_us", 0), r.get("end_time_us", 0))
-                for r in outer_recs
-                if r.get("phase") in ("complete", "dummy")
-            ),
-            key=lambda interval: interval[0],
-        )
-        resolve_parent_starts = [interval[0] for interval in resolve_parents]
-
-        def is_nested_resolve(rec):
-            start = rec.get("start_time_us", 0)
-            end = rec.get("end_time_us", 0)
-            parent_idx = bisect.bisect_right(resolve_parent_starts, start) - 1
-            return parent_idx >= 0 and end <= resolve_parents[parent_idx][1]
-
-        standalone_resolve = [r for r in records if r.get("phase") == "resolve" and not is_nested_resolve(r)]
+        nested_resolve_ids = nested_resolve_record_ids(records)
+        standalone_resolve = [
+            record
+            for record in records
+            if canonical_sched_phase(record.get("phase")) == "resolve" and id(record) not in nested_resolve_ids
+        ]
         work_recs = sorted(
             outer_recs + standalone_resolve,
             key=lambda r: r.get("start_time_us", 0),
@@ -261,7 +248,7 @@ def is_nested_resolve(rec):
         prev_end = None
 
         for rec in work_recs:
-            phase = rec["phase"]
+            phase = canonical_sched_phase(rec["phase"])
             start = rec.get("start_time_us", 0)
             end = rec.get("end_time_us", 0)
             # Idle = wall-clock gap between this record and the previous
@@ -297,16 +284,10 @@ def is_nested_resolve(rec):
         finishes_per_loop = total_finishes / loops if loops > 0 else 0.0
         pop_total = pop_hit + pop_miss
         pop_hit_rate = pop_hit / pop_total * 100 if pop_total > 0 else 0.0
-        phases_seen = {rec["phase"] for rec in work_recs}
+        phases_seen = {canonical_sched_phase(rec["phase"]) for rec in work_recs}
         if phase_us["idle"] > 0:
             phases_seen.add("idle")
-        scheduler_only_phases = {"complete", "dispatch", "release", "early_dispatch", "drain", "graph_prepare"}
-        has_scheduler_work = bool(phases_seen & scheduler_only_phases)
-        has_resolution_work = bool(phases_seen & {"resolve", "async_poll", "dummy"})
-        is_unassigned_thread = bool(assigned_thread_indices) and tid not in assigned_thread_indices
-        is_resolution_thread = (
-            has_resolution_work and not has_scheduler_work and (bool(standalone_resolve) or is_unassigned_thread)
-        )
+        role = scheduler_thread_role(records, assigned_thread_indices, tid, nested_resolve_ids)
 
         t = {
             # `completed` remains the legacy logical-task field used by the
@@ -322,7 +303,7 @@ def is_nested_resolve(rec):
             "pop_miss": pop_miss,
             "pop_hit_rate": pop_hit_rate,
             "format": "json_phase",
-            "role": "resolution" if is_resolution_thread else "scheduler",
+            "role": role,
             "phases_seen": phases_seen,
         }
         for p, us in phase_us.items():