diff --git a/l2-orchestrator-standalone/.gitignore b/l2-orchestrator-standalone/.gitignore new file mode 100644 index 0000000000..aafe4c4648 --- /dev/null +++ b/l2-orchestrator-standalone/.gitignore @@ -0,0 +1,4 @@ +# Every build tree this package makes. scripts/profile_l2.sh alone creates +# three (build, build-prof, build-tm), and the root .gitignore's `build/` rule +# matches only the first. +build*/ diff --git a/l2-orchestrator-standalone/CMakeLists.txt b/l2-orchestrator-standalone/CMakeLists.txt new file mode 100644 index 0000000000..7055f88d8a --- /dev/null +++ b/l2-orchestrator-standalone/CMakeLists.txt @@ -0,0 +1,156 @@ +# Standalone L2 orchestrator package: bench + profile for the four-line +# orchestration sequence in runtime_maker.cpp:538-541. Host-only, no CANN, +# no Ascend SDK, no NPU. +# +# The TU list mirrors the `host` target of +# src/a2a3/runtime/host_build_graph/build_config.py, minus `host/` itself +# (that is where runtime_maker and its CANN dependencies live): +# +# BUILD_CONFIG["host"]["source_dirs"] = +# ["host", "runtime/orchestrator_core", "runtime/shared", "orchestration"] +# +# scripts/check_extraction.sh asserts this list still matches the source repo. + +cmake_minimum_required(VERSION 3.15) +# C is required, and not incidentally: qwen3_dynamic_tensormap.h is C and cannot +# be compiled as C++ (C99 compound-literal lvalues; see esl_shim/esl_c_abi.h). +project(l2_orchestrator CXX C) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) + +if(WIN32) + message(FATAL_ERROR "This package requires POSIX. Build on Linux.") +endif() + +# Neither simpler nor this package sets a build type by default, which would +# leave the bench measuring an -O0 engine. Pin it: an unoptimised orchestrator +# is not the thing anyone wants a number for. +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) +endif() + +set(ROOT ${CMAKE_CURRENT_SOURCE_DIR}) +# Engine TUs are compiled straight out of the enclosing checkout, not copied: +# the bench measures the orchestrator of THIS revision, and an edit to +# ../src is picked up by the next build with nothing to keep in sync. +set(S ${ROOT}/../src) +set(R ${S}/a2a3/runtime/host_build_graph) + +# SIMPLER_ORCH_PROFILING gates the engine's own per-STEP cycle counters inside +# submit_task_common (Level 3 of the profile report). It requires SIMPLER_DFX=1, +# and both add work to the measured path — so the default is off and the +# throughput numbers come from the clean build. +option(L2_ORCH_PROFILING "Build the engine's own per-STEP cycle counters (Level 3)" OFF) +# SIMPLER_TENSORMAP_PROFILING adds the engine's own TensorMap lookup counters +# (bucket chain length, overlap checks/hits, insert count) — Level 4 of the +# report. profiling_config.h #errors unless SIMPLER_ORCH_PROFILING is also on, +# so this implies it. +option(L2_TENSORMAP_PROFILING "Build the engine's TensorMap lookup counters (Level 4)" OFF) + +add_library(l2_engine STATIC + ${R}/runtime/orchestrator_core/pto_orchestrator.cpp + ${R}/runtime/orchestrator_core/pto_ring_buffer.cpp + ${R}/runtime/orchestrator_core/pto_runtime2.cpp + ${R}/runtime/shared/pto_runtime2_init.cpp + ${R}/runtime/shared/pto_shared_memory.cpp + ${R}/runtime/shared/pto_tensormap.cpp + ${R}/runtime/shared/runtime.cpp + ${R}/orchestration/common.cpp + ${ROOT}/src/host_shim/host_shim.cpp +) +target_include_directories(l2_engine PUBLIC + ${R}/runtime + ${R}/common + ${R}/orchestration + ${S}/a2a3/runtime + ${S} + ${S}/common + ${S}/common/log/include + ${S}/common/task_interface + ${S}/common/platform/include + ${S}/a2a3/platform/include +) +if(L2_TENSORMAP_PROFILING) + set(L2_ORCH_PROFILING ON) +endif() +if(L2_ORCH_PROFILING) + target_compile_definitions(l2_engine PUBLIC SIMPLER_DFX=1 SIMPLER_ORCH_PROFILING=1) + message(STATUS "SIMPLER_ORCH_PROFILING=1 — Level 3 built; this build is NOT the one to quote throughput from") +endif() +if(L2_TENSORMAP_PROFILING) + target_compile_definitions(l2_engine PUBLIC SIMPLER_TENSORMAP_PROFILING=1) + message(STATUS "SIMPLER_TENSORMAP_PROFILING=1 — Level 4 (TensorMap chain stats) built") +endif() + +# Where qwen3_dynamic_tensormap.h lives. Defaults to this package's own +# directory; override to point elsewhere. +set(L2_QWEN3_DYN_CASE_DIR "${ROOT}" CACHE PATH + "Directory containing qwen3_dynamic_tensormap.h") +if(NOT EXISTS "${L2_QWEN3_DYN_CASE_DIR}/qwen3_dynamic_tensormap.h") + message(FATAL_ERROR + "qwen3_dynamic_tensormap.h not found in ${L2_QWEN3_DYN_CASE_DIR}.\n" + "Pass -DL2_QWEN3_DYN_CASE_DIR= to point at it.") +endif() + +# The payload is separate from the engine so a bench-side change cannot reach +# the engine TUs. +add_library(l2_payloads STATIC + ${ROOT}/bench/payload_qwen3_dyn.cpp + ${ROOT}/bench/payload_qwen3_dyn_case.c # C, on purpose — see below + ${ROOT}/bench/esl_shim/esl_shim_impl.cpp +) +target_include_directories(l2_payloads PUBLIC ${ROOT}/bench) +target_link_libraries(l2_payloads PUBLIC l2_engine) + +# esl_shim/ is deliberately NOT on any target-wide include path: it provides +# headers named `mem_pool.h` and `tensormap.h`, and only the two TUs below may +# see them. Everything else in the package must keep resolving those names the +# way it does today (i.e. not at all). +set_source_files_properties(${ROOT}/bench/esl_shim/esl_shim_impl.cpp PROPERTIES + INCLUDE_DIRECTORIES "${ROOT}/bench/esl_shim" +) + +# The case's SPMD tier is a compile-time knob of the case itself: +# +# #ifndef QWEN3_SPMD_TIER +# #define QWEN3_SPMD_TIER 4 +# #endif +# +# and it changes the DAG, not just its scheduling. `qwen3_blocks_per_task()` +# returns min(total_chunks, {1,2,4,8,1<<30}[tier]), so the tier decides how many +# SPMD chunks are folded into ONE task. This package pins tier 0 — one chunk per +# task, 3096 tasks / 3096 subtasks, the most task-dense variant and therefore the +# one that puts the most pressure on the orchestrator per unit of device work. +# The #ifndef guard in the case exists precisely so the build can set it, which +# is why pinning it here does not modify the case file. +# +# The entry symbol is renamed at the same time so the case's C-linkage +# `aicpu_orchestration_entry` cannot collide with the engine-side name. +set_source_files_properties(${ROOT}/bench/payload_qwen3_dyn_case.c PROPERTIES + INCLUDE_DIRECTORIES "${L2_QWEN3_DYN_CASE_DIR};${ROOT}/bench/esl_shim" + COMPILE_DEFINITIONS "aicpu_orchestration_entry=qwen3_dyn_orchestration_entry;QWEN3_SPMD_TIER=0" +) + +add_executable(l2_bench ${ROOT}/bench/l2_bench.cpp ${ROOT}/bench/l2_profile.cpp) +target_link_libraries(l2_bench PRIVATE l2_payloads) + +add_executable(l2_orch_main ${ROOT}/apps/l2_orch_main.cpp) +target_include_directories(l2_orch_main PRIVATE ${ROOT}/bench) +target_link_libraries(l2_orch_main PRIVATE l2_payloads) + +enable_testing() +add_test(NAME smoke COMMAND l2_orch_main) +# Both entry points are covered because the throughput path and the profile path +# drive the entry differently (the latter through the replacement ops table). +add_test(NAME bench COMMAND l2_bench --mode=throughput --repeat=1) +add_test(NAME profile COMMAND l2_bench --mode=profile) +# The prefault diagnostics touch the SM and arena before the engine initialises +# them, so a regression there would corrupt the run rather than just skew a +# number. Cover both, and assert the graph still comes out the same size. +add_test(NAME prefault COMMAND l2_bench --mode=throughput --repeat=1 --prefault-all) +set_tests_properties(prefault PROPERTIES + PASS_REGULAR_EXPRESSION "kernel submits +3096 +framework allocs +779") diff --git a/l2-orchestrator-standalone/HANDOFF.md b/l2-orchestrator-standalone/HANDOFF.md new file mode 100644 index 0000000000..a3d27f658c --- /dev/null +++ b/l2-orchestrator-standalone/HANDOFF.md @@ -0,0 +1,295 @@ +# Handoff — l2-orchestrator-standalone + +Context summary for resuming work after a session reset. For how to *use* the +package, read `README.md` instead; this file records **why it is the way it is** +and what is not done. + +## What this package is + +A host-only bench + profile harness for **one sequence** — the four lines that +build an L2 task graph, at +`simpler-main/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp:538-541`: + +```c +rt_scope_begin(rt); +entry_points->entry(orch_l2); +rt_scope_end(rt); +rt_orchestration_done(rt); +``` + +Engine under test: **L2 `PTO2OrchestratorState`** (a2a3 / `host_build_graph`). +Not the L3 `Orchestrator` — that is the sibling `../l3-orchestrator-standalone`, +a different engine at a different level. + +## How it came about (the question chain that produced it) + +1. "What are l3-orchestrator-standalone's inputs/outputs?" → its input is C++ API + calls, not a file; its output is DAG/scheduling behaviour, not numbers. +2. "How does it use simpler's `runtime_maker`?" → **it does not**, zero + references. `runtime_maker` is L2 glue driving `PTO2OrchestratorState`. +3. "Is that four-line snippet L2 or L3?" → **L2**. `host_build_graph` runs the + *same L2 orchestrator* on the host; execution site changed, level did not. +4. "Expand those four lines fully" → 1 stack push + T×(6-step submit) + A×alloc + + 1 pop + 1 release store. +5. "Build a package to bench/profile them, like the L3 one" → this package. +6. "Run it with the repo-root `qwen3_dynamic_tensormap.h` case" → the + `qwen3-dyn` payload. +7. "Wasn't that case 3096 tasks?" → yes, **at tier 0**. The build now pins tier + 0; tier 4 (522 tasks) was the case's own default. +8. "Only tier 0 of qwen3-dyn is useful" → the synth and qwen3 payloads, the + other four tiers and the sweep mode were removed. The package builds one + workload. + +## Non-obvious decisions, and why + +**Why it can run on a host at all.** `host_build_graph` already runs the L2 +orchestrator on the host into a host SM mirror, then H2Ds the image +(`run_host_orchestration`, runtime_maker.cpp:487-499). This package stops before +the H2D. Three substitutions, all in `bench/l2_harness.h`: GM heap → host +`aligned_alloc` (orchestrator only does address arithmetic on it, never +dereferences); host SM → same thing the real path already uses; +`entry_points->bind` → direct `framework_bind_runtime` (same function, dlopen +only exists to cross the .so boundary). + +**The extraction boundary was free.** `build_config.py`'s `host` target already +compiles exactly `runtime/orchestrator_core` + `runtime/shared` + +`orchestration`. Dropping `host/` (= runtime_maker + CANN) leaves 8 TUs that +compiled on the first try. Only 9 symbols were missing, all diagnostics → +`src/host_shim/host_shim.cpp`. + +**`pto_orchestration_api.h` and `pto_runtime2.h` cannot share a TU.** Both define +`PTO2Runtime` / `PTO2RuntimeOps` (by design: .so side vs runtime side). The +driver is the runtime side (like runtime_maker); the payloads are the .so side. +Do not "fix" this by including both. + +**The Level-2 profiling seam is the ops table.** `rt->ops` is a plain +`const PTO2RuntimeOps *` and the orchestration API reaches the engine +*exclusively* through it. Copy the table, wrap each entry with a clock read, +repoint `rt->ops`. Intercepts 100% of entry→engine traffic, needs no engine +cooperation, cannot miss a call site. Strictly better than L3's `set_test_hook`. + +**Two Level-3 traps, both handled.** The engine's `g_orch_*_cycle` counters are +process-global accumulators that **reset on read**, so they are snapshotted right +after the clean run (reading at report time folds both runs and doubles +everything). And the cycle→time divisor is `cntfrq_el0` (100 MHz here), **not** +`PLATFORM_PROF_SYS_CNT_FREQ` (50 MHz, the device counter) — using the latter made +STEP 5 report 73 ms inside a 21 ms run. + +## The `qwen3-dyn` payload — the subtle parts + +The repo-root case is compiled **unmodified and in place** (not copied, not +translated). `scripts/check_extraction.sh` asserts no in-package duplicate exists. + +**It is compiled as C, and must be.** Its shapes are C99 compound literals +(`(uint32_t[]){90, 5120}`), which are lvalues in C but temporaries in C++. g++ +rejects the decay outright. Verified across `-std=c++17 / gnu++17 / gnu++11 / +gnu++03`, with and without `-fpermissive` — **no flag accepts it**, and there is +no clang on this box. Hence `bench/payload_qwen3_dyn_case.c` + the C ABI in +`bench/esl_shim/esl_c_abi.h`. + +**`Tensor` on the C side is an opaque 128-byte blob.** The case never reads a +single `Tensor` member (verified by grep), and `ChipTensor` is exactly 128 bytes / +64-byte aligned / standard-layout / trivially copyable — all four +`static_assert`ed in `esl_shim_impl.cpp`. A blob assumes only size+alignment, so +unlike a field-by-field mirror it cannot silently rot. Note `Tensor` is *already* +a distinct struct in `src/common/task_interface/buffer.h`, so aliasing that name +in C++ is a hard error — that is why the C++ side uses `EslTensor` throughout. + +**The tag mapping, read from esl_proxy's source** (the `esl_proxy` checkout): +the `_ro` variants call only `add_tensor_addr()` and push **nothing** onto the +pending list `tm_submit` looks up/inserts. So `_ro` = "pass to kernel, create no +dependency", not a narrower access grant: + +| esl_proxy | L2 tag | +| --- | --- | +| `tm_in` | `INPUT` | +| `tm_out` | `OUTPUT_EXISTING` (**not** `OUTPUT` — see below) | +| `tm_inout` | `INOUT` | +| `tm_*_ro` | `NO_DEP` | + +`tm_out` → `OUTPUT_EXISTING` is load-bearing: only `OUTPUT_EXISTING`/`INOUT` get +registered in the TensorMap, and a runtime-created `OUTPUT` is explicitly +skipped. Mapping it to `OUTPUT` would allocate a second buffer, register no +producer, and **silently erase every RaW edge in the case**. + +**This disagrees with the L3 package.** `bench/qwen3_l3_replay.h` maps +`tm_in_ro -> INPUT` and `tm_out_ro -> OUTPUT_EXISTING`, giving `_ro` args real +edges. Against esl_proxy's source that is wrong and inflates the L3 replay's edge +count. 2874 of this case's tensor args are `_ro` at tier 0. **The L3 package has +not been corrected.** + +**`NO_DEP` still does creator retention** (L2 Step A), which esl_proxy has no +equivalent of. Every `_ro` arg here is an external or a view of one, and +externals have no creator — so it contributes nothing. `assert_no_creator()` +enforces this at every `_ro` call rather than assuming it. + +**`QWEN3_SPMD_TIER` changes the DAG, not just scheduling.** The subtask total is +tier-invariant (3096); the task count is not. Default here is **0**. + +| tier | kernel submits | subtasks | engine tasks | +| --- | --- | --- | --- | +| **0** (default) | **3096** | 3096 | **3875** | +| 1 / 2 / 3 | 1602 / 864 / 678 | 3096 | 2381 / 1643 / 1457 | +| 4 (case's own default) | 522 | 3096 | 1301 | + +The tier is exported from the case's own macro +(`payload_qwen3_dyn_case.c: const int qwen3_dyn_spmd_tier = QWEN3_SPMD_TIER;`) so +the report cannot drift from the binary. + +## Cross-validation (the strongest evidence the shim is faithful) + +The unmodified case through this shim vs the L3 package's **independently +hand-translated** replay, at every tier: + +| tier | L2 kernel submits | L3 `task_cnt` | +| --- | --- | --- | +| 0 | 3096 | 3096 | +| 1 | 1602 | 1602 | +| 2 | 864 | 864 | +| 3 | 678 | 678 | +| 4 | 522 | 522 | + +At tier 0 five numbers match exactly: 3096 submits, 779 allocs, 3096 subtasks, +3875 engine tasks, and the `DUR_*` sum 110168.700 us == L3's +`timeline.busy_ns = 110168700`. Tier 4 agrees too (522 / 779 / 3096 / 12572220). + +## Findings worth remembering + +**CORRECTION — an earlier reading in this file and in conversation was wrong.** +I originally reported "dependency inference is not the cost; writing the +descriptor is (STEP 5 = 66-86%)". That was a **measurement artifact of this +harness**, not a property of the engine. STEP 5's cold cost is ~96% **first touch +of the shared memory**. Anyone quoting the cold per-STEP percentages as engine +cost is quoting paging. `--prefault-sm` / `--prefault-arena` / `--prefault-all` +exist to separate the three effects; Level 4 (`-DL2_TENSORMAP_PROFILING=1`) +explains what is left. + +Steady-state (prefaulted) per-STEP shares, and they differ **by payload**: + +| step | `qwen3-dyn` tier 0 | `qwen3` (40 layers) | +| --- | --- | --- | +| STEP 1 prepare_task (ring slot + heap) | 11.0% | **43.2%** | +| STEP 3 TensorMap lookup | **74.4%** | 24.7% | +| STEP 5 payload/descriptor write | 9.9% | 26.5% | +| STEP 4 register outputs | 4.1% | 1.7% | +| cold → warm total | 9.75 → 2.97 ms | 19.68 → 1.71 ms | + +- **The two payloads have different bottlenecks, and Level 4 says why.** + `qwen3-dyn` writes many SPMD sub-views of shared buffers; the TensorMap hashes + on `buffer.addr` **alone** (`pto_tensormap.h:522`), so they collapse into one + bucket — avg chain 13.2, **MAX 120** (q_proj's 20 chunks × 6 tiles), **67.5% of + overlap checks find nothing**. `qwen3` allocates whole buffers, so its **MAX + chain is 1 and 0% is wasted**; its cost is the allocator instead. +- **STEP 3 is memory-latency bound, not compute bound.** 2.201 ms / 88784 walked + entries = **24.8 ns per entry** ≈ a cache miss. `check_overlap` already + fast-rejects in O(1) on a byte range (`pto_tensormap.h:240-249`), so the cost is + chasing `next_in_bucket` through an entry pool allocated in submit order. +- **Chain length scales with SPMD width — a superlinear term.** tier 0 → 4: + lookups fall 5.5× but entries walked fall **27×**. +- **The per-run SM allocation is a real production cost, not just a harness one.** + `runtime_maker.cpp:487` does `new uint8_t[sm_size]` **every run** and memsets + only the header. At the production default `PTO2_TASK_WINDOW_SIZE = 16384` that + is a fresh **~77.6 MB**, so large-block new/free hands pages back to the OS and + the next run faults them in again. ~6.2 ms per run at this graph size. + `sizeof(PTO2TaskPayload) = 4864 B`, of which `tensors[32] × 128 B` = 4096 B. +- **Submit cost tracks tensor-arg count, not graph size.** Established with + synth sweeps that held task count constant while moving one knob (`depth` + 8→128 and `width` 1→32 flat at ~2.0 us/task; `tensors` 1→24 rising 1.81→2.39 + us then saturating; `scalars` 1→16 flat). That payload is gone, so the claim + is no longer reproducible here — what remains is the profile's SUBMIT LATENCY + BY TENSOR-ARG COUNT table, which shows the same effect confounded with which + TensorMap bucket each arg set hashes into. +- **The four scope/done calls are free.** `entry` is ~99.97% of the block; + `on_scope_end` is literally `{}` in `host_build_graph` + (`scheduler/pto_scheduler.h:866`). + +## Optimization candidates (measured, none implemented) + +Ranked by headroom the data supports. This package measures; it does not patch +the engine. Checked `simpler-main/docs/investigations/` first — the existing +entries cover scheduler dispatch and host worker dispatch, **none touches the +orchestrator submit path**, so none of these was previously rejected. + +1. **Dense per-buffer entry scan — ≈1.5 ms, 51% of `qwen3-dyn`'s steady-state.** + Not "cheaper compares" (the L1 reject is already O(1)) but less pointer + chasing: a compact parallel array of `(start_offset, extent, version)` scanned + linearly before touching any full entry, or keep each buffer's chain **sorted + by `start_offset`** and stop once `entry.start >= in_end`. Sorted insertion is + O(chain) but there are 4002 inserts against 88784 walks. Upper bound: walking + only the 28260 real overlaps costs 0.70 ms instead of 2.20 ms. +2. **Pool the host SM mirror across runs — ≈6.2 ms per run.** Reusing a dirty + buffer is safe: init-on-write is already the engine's invariant. The device + SM and arena are already pooled (`acquire_pooled_gm_sm`); the host mirror is + not, which reads as an omission rather than a decision. +3. **Shrink `PTO2TaskPayload`.** 4096 of 4864 B is `tensors[32]`, but this case + peaks at 12 args and `qwen3` at 17. Cuts SM footprint (hence #2's residual) + and slot-to-slot locality. It is a host↔device wire struct so it must stay POD + and contiguous — variable-length slots with offset indices are what codestyle + rule 8 endorses, and the same rule warns against worst-case fixed arrays. ABI + change; measure the true max across all examples first. +4. **Raise the SPMD tier** — workload knob, zero code change, 11.6 → 3.3 ms cold. + Changes device-side SPMD granularity and load balance, so it is a trade. + +**Bound on candidate 1:** nothing is reclaimed here, so chain length only grows. +Production wraps the ring and reclaims entries at the watermark, so steady-state +chains should be shorter and 1.5 ms is an **upper bound at this graph size**. +Measuring the steady state needs the reclaim paths this harness never enters. + +## The caveat that bounds every number + +**Nothing is ever reclaimed.** No scheduler, no completions → `last_task_alive` +never advances → the ring's watermark reclaim never fires. Therefore: + +- `--task-window` must exceed the payload's **total** task count and `--heap-mb` + its **total** allocation. Not sized per steady state. +- Every repetition rebuilds the whole runtime; reusing one would measure + back-pressure. +- **The reclaim paths are never exercised** — `sync_tensormap`'s eviction, + `ensure_tensormap_capacity`'s back-pressure spin, the 500 ms deadlock backstop. + This bench measures the fast path only. + +A payload that exhausts either resource latches a fatal, after which every submit +is a silent no-op. The driver checks `orch_error_code` after every run and refuses +to print numbers when it is set — without that check, exhaustion looks like a +small, very fast graph. + +## Verification status (all re-run at tier 0) + +| check | result | +| --- | --- | +| clean build, default flags | 0 warnings, 0 errors | +| `-Wall -Wextra` | warnings only in the repo's engine code; 0 in hand-written code | +| ASAN + UBSAN, both modes | **clean** (LeakSanitizer cannot run in this container — needs ptrace) | +| `ctest` | 4/4 pass (incl. a prefault test asserting the graph size is unchanged) | +| `check_extraction.sh` | 3/3 pass; **negative-tested** — an edited engine file is reported and TU-list drift exits 1 | +| `ldd` | no CANN (libc / libstdc++ / libm only) | + +The tier sweep and the multi-payload ASAN matrix were verified before those +payloads and tiers were removed; only tier 0 of qwen3-dyn is buildable now. + +## Not done / open + +1. **Not committed.** `l2-orchestrator-standalone/` is untracked (`??`). 91 files + would be added, no build artifacts (verified after the `.gitignore` fix below). +2. **`.gitignore` gap found and fixed** in this session: `build/` and `**/build/` + do not match `build-prof/`, which `scripts/profile_l2.sh` creates and whose + `l2_bench` is an extension-less executable no other rule caught. Added + `build-*/` and `**/build-*/`. **This fix is also untracked.** +3. **No back-pressure test.** The only real coverage gap: deliberately undersize + `--task-window` and assert it fails with the expected `orch_error_code` rather + than crashing. Currently such a run is a bench *failure*, not a test case. This + is also what blocks measuring the reclaim paths, and therefore what bounds the + optimization estimates above. +6. **ASAN/UBSAN and `-Wall -Wextra` are not wired into any script or CI.** They + were run by hand this session (both clean). Reproduce with: + `cmake -B /tmp/san -S . -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -g" -DCMAKE_C_FLAGS="-fsanitize=address,undefined -g" -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined"` + then run with `ASAN_OPTIONS=detect_leaks=0`. +4. **The L3 package's `_ro` tag mapping is still wrong** (see above) and its edge + counts are still inflated. Not corrected — it would change + `docs/qwen3-l3-equivalence-report.md` and the `reports/sim-*.json` evidence. +5. **`tensormap_and_ringbuffer` not covered.** The repo-root case's *native* + runtime is TRB (device AICPU), whose four lines are at + `a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp:707-709` and + `:810`. Harder to extract: that orchestrator runs on-device, its + `on_scope_end` does real work (`release_producer_scope` per task), and there + is no existing host-side run path. diff --git a/l2-orchestrator-standalone/README.md b/l2-orchestrator-standalone/README.md new file mode 100644 index 0000000000..8cfdea1bdb --- /dev/null +++ b/l2-orchestrator-standalone/README.md @@ -0,0 +1,477 @@ +# L2 Orchestrator standalone package + +**In-repo, one payload.** This package lives inside the simpler checkout and +compiles the engine **straight out of `../src`** — there is no copy of the +runtime here, so an edit to `../src` is measured by the next build with nothing +to keep in sync. The only engine-side file the package owns is +`src/host_shim/host_shim.cpp`, which defines the 9 symbols the AICPU binary +would otherwise provide. + +It measures exactly one workload: `qwen3_dynamic_tensormap.h` at +`QWEN3_SPMD_TIER=0`, which lives in this directory. Results for the engine as of +`perf/hbg-orch` @ 72c3163d are in +`reports/perf-report-perf-hbg-orch-qwen3-dyn.md`. + +Bench and profile for **one sequence** — the four lines that build an L2 task +graph, at `src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp:538-541`: + +```c +rt_scope_begin(rt); +entry_points->entry(orch_l2); +rt_scope_end(rt); +rt_orchestration_done(rt); +``` + +The engine under measurement is **L2 `PTO2OrchestratorState`** (a2a3 / +`host_build_graph`). This is *not* the L3 `Orchestrator` — that one is in +`../l3-orchestrator-standalone`. The two are different engines at different +levels, not two views of one thing. + +No CANN, no Ascend SDK, no NPU, no `.so` to dlopen. + +## Why this can run on a host at all + +`host_build_graph` already runs the L2 orchestrator on the host: it builds the +graph into a **host SM mirror** and only then H2Ds the populated image to the +device, which boots scheduler-only (`run_host_orchestration`, runtime_maker.cpp:487-499). +This package stops before the H2D. Three substitutions, all documented in +`bench/l2_harness.h`: + +| simpler | here | why it is sound | +| --- | --- | --- | +| device GM heap | host `aligned_alloc` | the orchestrator only does address arithmetic on it — the AICore would dereference it, and there is no AICore here | +| host SM mirror | same, a plain buffer | not a substitution: the host-orch path already uses one | +| `entry_points->bind` + dlsym'd entry | direct call, linked in | same `framework_bind_runtime` function (orchestration/common.cpp:42); dlopen only exists to cross the .so boundary | + +**What is measured is graph CONSTRUCTION.** Nothing executes. There is no +scheduler, no completion, no H2D, no pointer relocation. + +## Build and run + +```bash +cmake -B build -S . +cmake --build build --parallel 8 +ctest --test-dir build --output-on-failure # 4 tests +./build/l2_orch_main # smoke: the four lines, 3875 tasks +``` + +If a binary dies with ``version `GLIBCXX_3.4.30' not found``, an older +`libstdc++.so.6` is ahead of the system one on the loader path: + +```bash +export LD_LIBRARY_PATH="$(dirname "$(readlink -f "$(g++ -print-file-name=libstdc++.so)")"):$LD_LIBRARY_PATH" +``` + +Everything at once: + +```bash +./scripts/profile_l2.sh # throughput + per-step profile + cold/warm attribution +./scripts/check_extraction.sh # assert the harness still matches the repo runtime +``` + +## The payload + +`qwen3_dynamic_tensormap.h` — the esl_proxy case, compiled **unmodified**, via +the C-ABI shim in `bench/esl_shim/`. 3096 kernel submits + 779 framework allocs += 3875 engine tasks. + +```bash +./build/l2_bench --mode=throughput --repeat=5 +./build-prof/l2_bench --mode=profile +``` + +### `QWEN3_SPMD_TIER` is pinned to 0, and it changes the DAG + +The case guards its tier with `#ifndef`, so the build sets it — `CMakeLists.txt` +compiles the case with `-DQWEN3_SPMD_TIER=0` and offers no knob for the rest. + +`qwen3_blocks_per_task()` returns `min(total_chunks, {1,2,4,8,1<<30}[tier])`, i.e. +the tier decides how many SPMD chunks fold into **one** task. The subtask total +is tier-invariant; the task count is not — tier 0 gives 3096 kernel submits, +tier 4 (the case's own default) gives 522, both for the same 3096 subtasks. + +Tier 0 is pinned because one chunk per task is the most task-dense variant, so +it puts the most orchestration pressure on the engine per unit of device work. +**A task count quoted without its tier is meaningless** — the report prints +`QWEN3_SPMD_TIER=N` on the accounting line for that reason, read from the macro +as the case itself saw it (`payload_qwen3_dyn_case.c` exports it) so the report +cannot drift from the binary, and `check_extraction.sh` asserts the pin. + +This is the natural home for that case: it is an **L2** case (`tm_in` / `tm_out` +/ `tm_submit`, `aicpu_orchestration_entry`), and L2's TensorMap does the +**view-overlap detection** the case's SPMD sub-view writes are built around +(`pto_tensormap.h:23`). The sibling L3 package had to disclaim exactly this — L3 +keys on whole-buffer identity with no byte-range refinement, so its edge set is +coarser by construction. No such disclaimer is needed here. + +**The case file is not translated.** It lives in this directory and is +`#include`d as it stands; `check_extraction.sh` asserts exactly one copy exists, +so no stale duplicate can be picked up instead. Three build-level +accommodations, none of which touch the file: + +1. **It is compiled as C**, because it is C. Its shapes are C99 compound + literals — `tensor_from_base_layout(orch_args + 0, (uint32_t[]){90, 5120}, 2, BFLOAT16)` + — which are lvalues in C but temporaries in C++, and g++ rejects the decay + ("taking address of temporary array"). No dialect or permissiveness flag + accepts it: checked across `-std=c++17`, `gnu++17`, `gnu++11`, `gnu++03`, + with and without `-fpermissive`. +2. `-Daicpu_orchestration_entry=qwen3_dyn_orchestration_entry`, so the case's + C-linkage entry cannot collide with the engine-side name of the same symbol. +3. `Tensor` on the C side is an **opaque 128-byte / 64-byte-aligned blob**, which + the C++ shim reinterprets as `ChipTensor`. The case never reads a single + `Tensor` member (verified by grep), and `ChipTensor` is exactly 128 bytes, + 64-byte aligned, standard-layout and trivially copyable — all four + `static_assert`ed in `esl_shim_impl.cpp`. A blob assumes only size and + alignment, so unlike a field-by-field mirror it cannot silently rot. + +### The tag mapping, and where the L3 package got it wrong + +Read from esl_proxy's source, not inferred. In its `tensormap.h` the `_ro` +variants push **nothing** onto the pending list that `tm_submit` later looks up +and inserts: + +```c +tm_in_ptr: add_tensor_addr(); tm_pending_push(t, TM_PEND_IN) +tm_in_ro_ptr: add_tensor_addr() /* no push */ +tm_out_ro_ptr: add_tensor_addr() /* no push */ +``` + +So `_ro` means "hand this tensor to the kernel, create **no** dependency" — it is +not a narrower access grant. Hence: + +| esl_proxy | dependency role | L2 tag | +| --- | --- | --- | +| `tm_in` | TensorMap lookup (RaW) | `INPUT` | +| `tm_out` | TensorMap insert | `OUTPUT_EXISTING` | +| `tm_inout` | lookup + insert | `INOUT` | +| `tm_*_ro` | none | `NO_DEP` | + +L2's `NO_DEP` exists for precisely this case ("skips OverlapMap lookup, depends +on creator only"). Two details that are easy to get wrong and are handled: + +- `tm_out` must map to `OUTPUT_EXISTING`, **not** `OUTPUT`. The case allocates + its buffers up front, and only `OUTPUT_EXISTING`/`INOUT` get registered in the + TensorMap — a runtime-created `OUTPUT` is explicitly skipped. Mapping `tm_out` + to `OUTPUT` would allocate a second buffer and register no producer, silently + erasing every RaW edge in the case. +- `NO_DEP` still performs L2's Step-A creator retention, which esl_proxy has no + equivalent of. Every `_ro` arg in this case is an entry external or a view of + one, and externals have no creator, so Step A contributes nothing — and that + invariant is **asserted at every `_ro` call**, not assumed. + +This **disagrees with the sibling L3 package's table** in +`bench/qwen3_l3_replay.h`, which maps `tm_in_ro -> INPUT` and +`tm_out_ro -> OUTPUT_EXISTING`, i.e. gives the `_ro` args real edges. Against +esl_proxy's source that is wrong, and it inflates the L3 replay's edge count. +804 of this case's 2,766 tensor args are `_ro`. + +### Measured at tier 0, and cross-validated against the L3 replay + +``` +payload=qwen3-dyn tasks=3875 +four-line block median 11.340 ms per task 2.93 us 341,702 tasks/s + QWEN3_SPMD_TIER=0 + kernel submits 3096 framework allocs 779 (engine tasks = 3875) + SPMD subtasks 3096 scalar args 7326 + dep-tracked args 9162 _ro (NO_DEP) args 2874 + sum of the case's DUR_*: 110168.700 us (virtual AICore time, NOT measured) +``` + +**Five** numbers match the L3 package's independently hand-translated replay of +the same case at the same tier, exactly — `reports/lat-t0.json` there reports +`task_cnt=3096`, `alloc_cnt=779`, `subtask_cnt=3096`, `task+alloc=3875`, and +`timeline.busy_ns=110168700`. Two unrelated paths (a hand translation to L3, and +the unmodified case through this shim) agreeing to the digit is the strongest +available evidence that the shim reproduces the case's structure rather than +approximating it. The agreement holds at tier 4 as well (522 / 779 / 3096 / +12572220). + +`tasks=3875` is `3096 + 779` because on L2 an `alloc_tensors` **is** a real task — +it claims a ring slot, cuts the GM heap and registers the buffer — whereas in +esl_proxy it is a pool-tail bump. That difference is visible in the profile +rather than hidden. + +### Where the time actually goes — separate allocation cost from compute first + +**A raw Level-3 reading is misleading, and this is the most important thing on +this page.** With a cold SM, STEP 5 appears to be 66% of submit cost. It is not: +96% of that is the **first touch of the shared memory**, not the engine writing +descriptors. `--prefault-sm` / `--prefault-arena` isolate the three effects: + +| step | cold (raw) | +prefault SM | +prefault arena | +both | +| --- | --- | --- | --- | --- | +| STEP 1 prepare_task | 0.501 ms | 0.334 | 0.469 | **0.333** | +| STEP 3 infer deps: TensorMap lookup | 2.552 ms | 2.231 | 2.558 | **2.201** | +| STEP 4 register outputs | 0.296 ms | 0.310 | **0.122** | **0.122** | +| STEP 5 payload/descriptor GM write | **6.461 ms** | **0.275** | 6.300 | **0.296** | + +Read the rows, not the totals: + +- **STEP 5** collapses 24× on `--prefault-sm` and does not move on + `--prefault-arena`. So it is SM first-touch. The engine's actual descriptor + + payload write is **0.28 ms**, not 6.5 ms. +- **STEP 4** halves on `--prefault-arena` only — the TensorMap buckets and entry + pool live in the arena. +- **STEP 3** barely moves (−14%). It is **real CPU work**, and once the + allocation effects are removed it is **74%** of submit cost. + +So the steady-state breakdown at tier 0 is: + +| step | time | share | +| --- | --- | --- | +| **STEP 3 TensorMap lookup** | **2.201 ms** | **74%** | +| STEP 1 prepare_task | 0.333 ms | 11% | +| STEP 5 payload/descriptor write | 0.296 ms | 10% | +| STEP 4 register outputs | 0.122 ms | 4% | +| STEP 2 / 6 | 0.017 ms | 1% | + +**Do not quote the cold numbers as engine cost.** They are dominated by a +per-run allocation that production shares (see below), which is a different +problem with a different fix. + +### Why STEP 3 costs what it does — Level 4 + +`-DL2_TENSORMAP_PROFILING=1` turns on the engine's own lookup counters: + +``` +lookups 6612 +bucket entries walked 88784 +avg chain length 13.43 +MAX chain length 120 +overlap checks 86970 +overlap hits 28260 (32.5% of checks) +wasted walk 67.5% +``` + +Root cause is at `pto_tensormap.h:522` — the map **hashes on `buffer.addr` +alone**, so every SPMD sub-view of one buffer shares a bucket and each lookup +walks the whole chain. `MAX chain = 120` is exactly q_proj's 20 chunks × 6 tiles. + +The arithmetic says where the cost sits: + +``` +2.201 ms / 6612 lookups = 333 ns per lookup +2.201 ms / 88784 walked = 24.8 ns per chain entry +``` + +24.8 ns is about a cache miss. `check_overlap` already fast-rejects in O(1) on a +byte-range test (`pto_tensormap.h:240-249`), so the cost is **not** the +comparison — it is chasing `next_in_bucket` through an entry pool that is +allocated in submit order and therefore scattered. + +The tier makes this visible as a superlinear term: + +| | tier 0 | tier 4 | ratio | +| --- | --- | --- | --- | +| lookups | 6612 | 1194 | 5.5× | +| **entries walked** | **88784** | **3215** | **27×** | +| MAX chain | 120 | 17 | 7× | + +Entries walked grows 27× while lookups grow 5.5×: chain length itself scales +with SPMD width. + +### End-to-end effect of the two levers + +| | cold | prefaulted | +| --- | --- | --- | +| tier 0 | 11.606 ms | 5.030 ms | +| tier 4 | 3.259 ms | 1.129 ms | + +Together: 11.6 ms → 1.13 ms, a **90%** reduction — but the two levers are not +equivalent. Prefaulting is a *diagnostic*; the production fix is to pool the +buffer (below). The tier is a *workload* knob that changes device-side SPMD +granularity, so it is a trade, not a free win. + +### Optimization candidates, with measured headroom + +Ranked by what the numbers above support. None of these is implemented here — +this package measures, it does not patch the engine. + +1. **Make the per-buffer entry scan dense (≈1.5 ms, 51% of steady-state cost).** + Not "make each compare cheaper" — the L1 reject is already O(1). Reduce the + pointer chasing: either keep a compact parallel array of each entry's + `(start_offset, extent, version)` so the scan is a linear sweep before any + full-entry touch, or keep each buffer's chain **sorted by `start_offset`** and + stop once `entry.start >= in_end`. Sorted insertion is O(chain) but there are + 4002 inserts against 88784 walks. Upper bound: walking only the 28260 real + overlaps costs 0.70 ms instead of 2.20 ms. + +2. **Pool the host SM mirror across runs (≈6.2 ms per run).** + `runtime_maker.cpp:487` does `new uint8_t[sm_size]` **every run** and memsets + only the header segment. At the production default + `PTO2_TASK_WINDOW_SIZE = 16384` that is a fresh **~77.6 MB** allocation, so + large-block `new`/`free` returns the pages to the OS and the next run faults + them in again. Safe to reuse a dirty buffer: init-on-write is already the + engine's invariant. The device-side SM and arena are already pooled + (`acquire_pooled_gm_sm`); the host mirror is not, which reads as an omission. + +3. **Shrink `PTO2TaskPayload` (4864 B/slot, of which `tensors[32] × 128 B` = 4096 B).** + This case peaks at 12 tensor args (the `qwen3` payload at 17), yet every slot + reserves 32. Costs SM footprint — hence #2's residual — and slot-to-slot + locality (a submit writes ~512 B but consecutive slots' hot regions are 4864 B + apart). It is a host↔device wire struct, so it must stay POD and contiguous — + but variable-length slots with offset indices are exactly what the project's + own codestyle rule 8 endorses, and the same rule warns against sizing fixed + arrays to a worst case. ABI change; measure the real maximum across all + examples first. + +4. **Raise the SPMD tier (11.6 → 3.3 ms cold, zero code change).** + A workload knob, not an engine fix: folding chunks cuts the number of + TensorMap lookups but changes device-side SPMD granularity and load balance. + +**Caveat that bounds candidate 1's estimate:** nothing is reclaimed here, so +chain length only ever grows. Production wraps the ring and reclaims entries at +the watermark, so steady-state chains should be shorter and 1.5 ms is an **upper +bound at this graph size**, not a production figure. Measuring the steady state +requires the reclaim paths, which this harness never enters. + +## Modes + +```bash +./build/l2_bench --mode=throughput --repeat=5 +./build-prof/l2_bench --mode=profile +# cold/warm attribution — needed before reading any Level-3 percentage +./build-tm/l2_bench --mode=profile --prefault-all +./build/l2_bench --help +``` + +- `throughput` — N clean repetitions, wall clock only. **Quote these.** +- `profile` — one clean run then one instrumented run, so the instrument's own + cost is a reported number rather than folded in silently. **Pass + `--prefault-all` before reading any per-STEP percentage** (see below). + +## The four profile levels + +No engine source is modified. Each level has a different seam: + +**Level 1 — the four lines.** Driver-bracketed `steady_clock`. Free of +instrument cost. + +**Level 2 — every `entry` → engine call.** `rt->ops` is a plain +`const PTO2RuntimeOps *` and the orchestration API reaches the engine +*exclusively* through it (`pto_orchestration_api.h:175, 125, 245, 254, 262` — +every one is `rt->ops->…`). So the profiler copies that table, wraps each entry +with a clock read, and repoints `rt->ops` at the copy. This intercepts **100%** +of the traffic, needs no cooperation from the engine, and cannot miss a call +site — a strictly better seam than L3's `set_test_hook`. + +**Level 3 — the engine's own per-STEP counters.** `submit_task_common` already +laps `CYCLE_COUNT_LAP` into `g_orch_*_cycle` at each of its six STEP boundaries, +and `orchestrator_get_profiling()` returns them. Compiled out unless +`SIMPLER_ORCH_PROFILING=1`: + +```bash +cmake -B build-prof -S . -DL2_ORCH_PROFILING=1 && cmake --build build-prof --parallel 8 +``` + +Two traps this package handles, both of which produce plausible-looking garbage +if missed: + +- The counters are **process-global accumulators that reset on read**, so they + are snapshotted right after the clean run — reading at report time would fold + both runs together and double every number. +- The cycle→time divisor is **`cntfrq_el0` (100 MHz here), not + `PLATFORM_PROF_SYS_CNT_FREQ` (50 MHz)**. The latter describes the device + counter, not the host clock the shim reads, and using it makes every Level-3 + duration 2× too large. + +**Level 4 — the engine's own TensorMap lookup counters.** Bucket chain length +(avg and max), overlap checks vs hits, insert count. This is what turns +"STEP 3 is 74%" into a root cause. Implies Level 3: + +```bash +cmake -B build-tm -S . -DL2_TENSORMAP_PROFILING=1 && cmake --build build-tm --parallel 8 +``` + +**Cold/warm attribution — `--prefault-sm` / `--prefault-arena` / `--prefault-all`.** +Diagnostics, not optimizations: they pre-touch the SM and/or the runtime arena so +first-touch cost lands outside the measured window instead of being attributed to +whichever step happened to touch the page. Without them, STEP 5 reads as 66-86% +of submit cost and the real bottleneck is invisible. Semantically safe — every +byte the engine reads from either region is written by the init phases first. + +## Measured, this host (Kunpeng-920 aarch64, `-O3`, taskset 0-3) + +3875 engine tasks, `--repeat=20`, median: + +| condition | four-line block | per task | throughput | +| --- | --- | --- | --- | +| cold | 11.41 ms | 2.94 us | 340 k tasks/s | +| `--prefault-all` | 4.71 – 4.90 ms | 1.22 – 1.26 us | 790 – 820 k tasks/s | + +Level 1 shows where the block's time is: `entry` **99.95%**, +`rt_orchestration_done` 0.03%, `rt_scope_begin` and `rt_scope_end` together +under 0.02%. The two scope calls are O(1) stack pushes and, in +`host_build_graph`, `on_scope_end` is literally `{}` +(`scheduler/pto_scheduler.h:866`) — so the outer scope pair costs nothing and +all cost is inside `entry`. + +Level 3 — **cold vs prefaulted**, because the raw reading is dominated by SM +first-touch and says almost nothing about the engine: + +| step | cold | prefaulted | prefaulted share | first-touch share of cold | +| --- | --- | --- | --- | --- | +| STEP 5 payload/descriptor GM write | 6.523 ms | 0.294 ms | 9.9% | 95% | +| **STEP 3 infer deps: TensorMap lookup** | 2.554 ms | **2.213 ms** | **74.7%** | 13% | +| STEP 1 prepare_task (slot + heap) | 0.506 ms | 0.314 ms | 10.6% | 38% | +| STEP 4 register outputs | 0.299 ms | 0.120 ms | 4.0% | 60% | +| STEP 2 sync_tensormap | 0.030 ms | 0.012 ms | 0.4% | 60% | +| STEP 6 publish fanin_count | 0.014 ms | 0.008 ms | 0.3% | 43% | + +**Cold and warm answer different questions.** Cold, STEP 5 looks like the +bottleneck at 66% of submit cost — but 95% of that is the kernel faulting in SM +pages, a cost paid once per runtime in production rather than per orchestration. +Prefaulted, the engine bottleneck is **STEP 3, the TensorMap dependency lookup, +at 75%**. Level 4 explains why: this case writes many SPMD sub-views of shared +buffers, the map hashes on `buffer.addr` alone, and they collapse into a handful +of buckets — max chain 120, 67.5% of overlap checks find nothing. + +That is specific to workloads shaped like this one. A workload that allocates +whole buffers instead keeps its max chain at 1 and pays in the ring/heap +allocator (STEP 1) rather than in the lookup, so a fix aimed at the chain would +do nothing for it. + +## Caveat that bounds every number here + +**Nothing is ever reclaimed.** There is no scheduler and no completion, so +`last_task_alive` never advances and the ring's watermark reclaim never fires. +Consequences: + +- `--task-window` must exceed the payload's **total** task count, and + `--heap-mb` its **total** allocation. The defaults (8192 slots, 2048 MiB) hold + this graph with headroom. +- Every repetition tears down and rebuilds the whole runtime. Reusing one would + measure back-pressure, not steady-state submit. +- The reclaim paths (`sync_tensormap`'s eviction, `ensure_tensormap_capacity`'s + back-pressure spin, the 500 ms deadlock backstop) are **never exercised**. + This bench measures the fast path only. + +A payload that exhausts either resource latches a fatal, after which every +submit is a silent no-op — which would otherwise look like a small, very fast +graph. The driver checks `orch_error_code` after every run and refuses to print +numbers when it is set. + +## What is in the box + +| path | role | +| --- | --- | +| `../src/a2a3/runtime/host_build_graph/` | the L2 engine — 8 TUs, compiled in place, not copied | +| `src/host_shim/host_shim.cpp` | the only hand-written engine-side file: 9 symbols (5 log sinks, 4 scope-stats stubs) plus `get_sys_cnt_aicpu` | +| `qwen3_dynamic_tensormap.h` | the case, compiled unmodified as C | +| `bench/l2_harness.h` | host-only runtime assembly | +| `bench/l2_profile.{h,cpp}` | the 4-level profiler and the ops-table interceptor | +| `bench/payload_qwen3_dyn*.{cpp,c}` | the payload: entry adapter plus the case's TU | +| `bench/esl_shim/` | the esl_proxy C ABI, implemented on the L2 orchestration API | +| `apps/l2_orch_main.cpp` | smoke driver: the four lines, nothing else | +| `scripts/` | `profile_l2.sh`, `check_extraction.sh`, `cold_warm_table.py` | + +The TU list is the `host` target of +`src/a2a3/runtime/host_build_graph/build_config.py` minus `host/` itself, which +is where `runtime_maker` and its CANN dependencies live. `check_extraction.sh` +asserts that correspondence. + +## Confirm there is no CANN + +```bash +ldd build/l2_bench | grep -Ei 'ascend|hcom|runtime|acl' # must print nothing +``` diff --git a/l2-orchestrator-standalone/apps/l2_orch_main.cpp b/l2-orchestrator-standalone/apps/l2_orch_main.cpp new file mode 100644 index 0000000000..bd90a521cd --- /dev/null +++ b/l2-orchestrator-standalone/apps/l2_orch_main.cpp @@ -0,0 +1,63 @@ +/* + * Smoke driver: the four lines, nothing else. + * + * rt_scope_begin(rt); + * entry_points->entry(orch_l2); + * rt_scope_end(rt); + * rt_orchestration_done(rt); + * + * Same shape as runtime_maker.cpp:538-541, with the dlopen'd entry replaced by + * a linked-in one. Prints the task count the orchestrator actually claimed and + * whether mark_done published its signal — enough to tell "the engine ran" from + * "the engine latched a fatal on the first submit and no-op'd the rest", which + * is otherwise indistinguishable from a very fast run. + */ + +#include +#include + +#include "l2_harness.h" +#include "payload.h" +#include "pto_runtime2.h" + +using namespace l2_bench; + +int main() { + try { + // Same graph the bench builds, so a smoke failure here is a real + // failure there: nothing is reclaimed, so both must hold in full. + HarnessConfig hc; + hc.task_window = 8192; + hc.heap_bytes = 2048ULL << 20; + Harness h(hc); + + PayloadArgs args; + qwen3_dyn_build_args(args); + + PTO2Runtime *rt = h.rt(); + rt_scope_begin(rt); + qwen3_dyn_entry(args.args); + rt_scope_end(rt); + rt_orchestration_done(rt); + + if (h.fatal()) { + std::fprintf(stderr, "l2_orch_main: orchestrator latched fatal, error_code=%d\n", h.error_code()); + return 1; + } + const int32_t tasks = h.active_task_count(); + if (tasks <= 0) { + std::fprintf(stderr, "l2_orch_main: no tasks were claimed\n"); + return 1; + } + if (!h.orchestration_done()) { + std::fprintf(stderr, "l2_orch_main: mark_done did not publish orchestrator_done\n"); + return 1; + } + std::printf("l2_orch_main: ok tasks=%d orchestrator_done=1\n", tasks); + std::printf("l2_orch_main: this is graph CONSTRUCTION only — no task is executed, no H2D happens.\n"); + return 0; + } catch (const std::exception &e) { + std::fprintf(stderr, "l2_orch_main: failed: %s\n", e.what()); + return 1; + } +} diff --git a/l2-orchestrator-standalone/bench/esl_shim/esl_c_abi.h b/l2-orchestrator-standalone/bench/esl_shim/esl_c_abi.h new file mode 100644 index 0000000000..578343ff20 --- /dev/null +++ b/l2-orchestrator-standalone/bench/esl_shim/esl_c_abi.h @@ -0,0 +1,191 @@ +/* + * esl_proxy C ABI, reimplemented over the L2 orchestrator. + * + * This header is included from BOTH sides: + * - the C translation unit that compiles qwen3_dynamic_tensormap.h unmodified + * - the C++ translation unit that implements these functions on L2 + * + * WHY THE CASE IS COMPILED AS C, NOT C++ + * -------------------------------------- + * The case passes its tensor shapes as C99 compound literals: + * + * tensor_from_base_layout(orch_args + 0, (uint32_t[]){90, 5120}, 2, BFLOAT16) + * + * In C a compound literal is an lvalue with enclosing-block lifetime, so + * array-to-pointer decay is well defined. In C++ it is a temporary, and g++ + * rejects the decay outright ("taking address of temporary array"). No dialect + * or permissiveness flag accepts it — checked across -std=c++17 / gnu++17 / + * gnu++11 / gnu++03, with and without -fpermissive. The file is C, so it is + * compiled as C, and this header is the boundary. + * + * WHY `Tensor` IS AN OPAQUE BLOB + * ------------------------------ + * The C side needs `Tensor` to be a complete type (the case declares locals, + * copies them, and takes their address) but it never reads a single member — + * verified by grep: no `.shapes`, `.buffer`, or any other field access in the + * whole case. So the C side gets a blob, and the C++ side reinterprets it as + * ChipTensor. Both are 128 bytes at 64-byte alignment, and ChipTensor is + * standard-layout and trivially copyable, which is what makes the case's + * by-value copies (`Tensor v = view(t, ...)`) mean exactly what they mean for a + * ChipTensor. esl_shim_impl.cpp static_asserts all of it rather than trusting + * this comment. + * + * A blob is deliberately stronger than mirroring esl_proxy's struct field by + * field: it assumes only size and alignment, so it cannot silently rot if a + * ChipTensor field is reordered. + */ + +#ifndef ESL_SHIM_C_ABI_H +#define ESL_SHIM_C_ABI_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* --- Tensor -------------------------------------------------------------- */ + +typedef struct EslTensor { + unsigned char opaque[128]; +} __attribute__((aligned(64))) EslTensor; + +#ifndef __cplusplus +/* The case spells it `Tensor`. The C++ side must NOT: `Tensor` is already a + * distinct wire struct in src/common/task_interface/buffer.h, and aliasing the + * name there is a hard redefinition error. */ +typedef EslTensor Tensor; +#endif + +/* esl_proxy's dtype_t IS the element width in bytes (its tensor.h:16-20). + * FLOAT32 and INT32 are both 4 and therefore indistinguishable — carried over + * as-is, since width is all the orchestrator uses a dtype for. */ +typedef enum { + BFLOAT16 = 2, + FLOAT32 = 4, + INT32 = 4 +} dtype_t; + +/* --- task.h ------------------------------------------------------------- */ + +typedef enum { + TASK_TYPE_CUBE = 0, + TASK_TYPE_VECTOR = 1, + /* esl_proxy really defines MIX == VECTOR == 1 (task.h:19-20), so the case's + * one TASK_TYPE_MIX task is indistinguishable from a VECTOR one on the + * esl_proxy side too. Preserved, not "fixed". */ + TASK_TYPE_MIX = 1, + TASK_TYPE_CNT = 3 +} task_type_t; + +typedef enum { + ORG_MODE_SINGLE = 0, + ORG_MODE_GROUP = 1, + ORG_MODE_SPMD_SYNC = 2, + ORG_MODE_SPMD_ASYNC = 3 +} org_mode_t; + +#define RING_SIZE 4096 +#define RING_MASK (RING_SIZE - 1) + +/* --- ring_buf.h ---------------------------------------------------------- */ + +/* + * Only the fields the case itself touches. The case defines two static helpers + * (set_task_type / set_block_num) that write .type / .mode / .count directly; + * neither is called anywhere in it, but both must type-check, which is the only + * reason this struct is visible at all. + * + * It is NOT the pending-task state. That lives in esl_shim_impl.cpp as a + * CoreTaskArgs per ring slot, because a C++ Arg cannot appear in a C header. + * new_task() writes both. Consequence, stated plainly: if a future case DID + * call set_block_num(), it would update this struct and not the CoreTaskArgs, + * so the block count would not reach the engine. esl_shim_impl.cpp's submit + * cross-checks the two and aborts on disagreement rather than silently + * submitting the wrong SPMD width. + */ +struct esl_task_desc { + uint32_t id; + task_type_t type; + org_mode_t mode; + uint32_t index; + uint32_t count; + uint32_t duration; + uint32_t tensor_cnt; + uint32_t scalar_cnt; +}; + +extern struct esl_task_desc g_basic_buf[RING_SIZE]; +extern uint32_t g_task_id; + +int new_task(uint32_t task_id, uint32_t type, uint32_t count, uint32_t duration); +void add_scalar(uint32_t task_id, int64_t value); + +/* --- tensor.h ----------------------------------------------------------- */ + +EslTensor tensor_from_base_layout(uint64_t base, const uint32_t shapes[], uint32_t ndims, dtype_t dtype); +EslTensor esl_view_at(const EslTensor *t, uint32_t off0, uint32_t off1, uint32_t n0, uint32_t n1); + +#ifndef __cplusplus +/* esl_proxy's own spelling: a macro so the call site keeps by-value syntax + * while the source Tensor is passed by address (its tensor.h:139). + * + * C-side ONLY. `view` is also a member function of ChipTensor + * (src/common/task_interface/tensor.h:281), and an object-like macro named + * `view` mangles that declaration for every C++ TU that sees both headers. + * The C++ implementation calls esl_view_at / ChipTensor::view directly. */ +#define view(t, off0, off1, n0, n1) esl_view_at(&(t), (off0), (off1), (n0), (n1)) +#endif + +/* --- tensormap.h -------------------------------------------------------- */ + +void tm_deps_init(void); + +void tm_in_ptr(uint32_t tid, const EslTensor *t); +void tm_out_ptr(uint32_t tid, const EslTensor *t); +void tm_inout_ptr(uint32_t tid, const EslTensor *t); +void tm_in_ro_ptr(uint32_t tid, const EslTensor *t); +void tm_out_ro_ptr(uint32_t tid, const EslTensor *t); +void tm_inout_ro_ptr(uint32_t tid, const EslTensor *t); +void tm_submit_ptr(uint32_t tid); + +#define tm_in(tid, t) tm_in_ptr((tid), &(t)) +#define tm_in_ro(tid, t) tm_in_ro_ptr((tid), &(t)) +#define tm_out(tid, t) tm_out_ptr((tid), &(t)) +#define tm_out_ro(tid, t) tm_out_ro_ptr((tid), &(t)) +#define tm_inout(tid, t) tm_inout_ptr((tid), &(t)) +#define tm_inout_ro(tid, t) tm_inout_ro_ptr((tid), &(t)) +#define tm_submit(tid) tm_submit_ptr((tid)) + +/* --- mem_pool.h --------------------------------------------------------- */ + +/* Signature is esl_proxy's verbatim, `int dim` / `int bytes` included: `bytes` + * is the dtype (see dtype_t above) and `dim` is the rank at every call site. */ +EslTensor alloc_tensors(uint32_t shape[], int dim, int bytes); + +/* --- driver-side accounting (not part of esl_proxy) --------------------- */ + +/* + * Read back after the entry returns so the report can state the case's own + * numbers — task count, SPMD subtask total (esl_proxy's g_subtask_cnt), and the + * sum of the DUR_* the case attaches — without the case being modified to + * export them. + */ +struct esl_stats { + uint64_t tasks; + uint64_t subtasks; + uint64_t duration_ns; + uint64_t allocs; + uint64_t tracked_args; /* tm_in / tm_out / tm_inout — build edges */ + uint64_t no_dep_args; /* tm_*_ro — build none */ + uint64_t scalars; +}; + +struct esl_stats esl_get_stats(void); +void esl_reset_state(void); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* ESL_SHIM_C_ABI_H */ diff --git a/l2-orchestrator-standalone/bench/esl_shim/esl_shim_impl.cpp b/l2-orchestrator-standalone/bench/esl_shim/esl_shim_impl.cpp new file mode 100644 index 0000000000..3baceaddb9 --- /dev/null +++ b/l2-orchestrator-standalone/bench/esl_shim/esl_shim_impl.cpp @@ -0,0 +1,317 @@ +/* + * esl_proxy C ABI implemented on the L2 orchestration API. + * + * THE TAG MAPPING — read this before trusting any edge count + * --------------------------------------------------------- + * Taken from esl_proxy's source, not inferred. In + * esl_proxy/include/algorithm/tensormap.h the `_ro` variants push NOTHING onto + * the pending list that tm_submit later looks up and inserts: + * + * tm_in_ptr: add_tensor_addr(); tm_pending_push(t, TM_PEND_IN) + * tm_out_ptr: add_tensor_addr(); tm_pending_push(t, TM_PEND_OUT) + * tm_inout_ptr: add_tensor_addr(); tm_pending_push(t, TM_PEND_INOUT) + * tm_in_ro_ptr: add_tensor_addr() <-- no push + * tm_out_ro_ptr: add_tensor_addr() <-- no push + * tm_inout_ro_ptr: add_tensor_addr() <-- no push + * + * So `_ro` means "hand this tensor to the kernel, create NO dependency". It is + * not a narrower access grant. Hence: + * + * | esl_proxy | dependency role | L2 tag | + * | tm_in | tensormap lookup (RaW) | INPUT | + * | tm_out | tensormap insert | OUTPUT_EXISTING | + * | tm_inout | lookup + insert | INOUT | + * | tm_*_ro | none | NO_DEP | + * + * L2's NO_DEP exists for exactly this ("No-dependency existing tensor: skips + * OverlapMap lookup, depends on creator only", pto_types.h): compute_task_fanin + * skips its Step-B lookup, register_task_outputs skips its insert, and the arg + * still reaches the kernel and the descriptor. + * + * tm_out maps to OUTPUT_EXISTING rather than OUTPUT because the case allocates + * its buffers up front with alloc_tensors and then writes into them. Only + * OUTPUT_EXISTING and INOUT get registered in the TensorMap; a runtime-created + * OUTPUT is explicitly skipped ("Runtime-created OUTPUT tensors are not looked + * up in the TensorMap since they have no dependencies"), so mapping tm_out to + * OUTPUT would allocate a second buffer and register no producer — silently + * erasing every RaW edge in the case. + * + * NOTE — this DISAGREES with the sibling L3 package's table in + * bench/qwen3_l3_replay.h, which maps `tm_in_ro -> INPUT` and + * `tm_out_ro -> OUTPUT_EXISTING`, i.e. gives the `_ro` args real edges. Against + * esl_proxy's source that is wrong and inflates the L3 replay's edge count. + * + * ONE RESIDUAL DIFFERENCE, GUARDED + * -------------------------------- + * NO_DEP still performs L2's Step A (creator retention): a NO_DEP tensor with a + * valid owner_task_id yields an edge to its creator, which esl_proxy would not + * produce. In this case every `_ro` arg is an entry external or a view of one, + * and externals have no creator — so Step A contributes nothing here. That is + * asserted at every `_ro` call rather than assumed. + */ + +#include +#include +#include +#include +#include +#include + +#include "esl_c_abi.h" +#include "pto_orchestration_api.h" + +// The blob-to-ChipTensor equivalence the whole C boundary rests on. +static_assert(sizeof(EslTensor) == sizeof(ChipTensor), "EslTensor must match ChipTensor size"); +static_assert(alignof(EslTensor) == alignof(ChipTensor), "EslTensor must match ChipTensor alignment"); +static_assert(std::is_standard_layout_v, "ChipTensor must be standard-layout to cross the C boundary"); +static_assert( + std::is_trivially_copyable_v, + "ChipTensor must be trivially copyable: the case copies Tensors by value" +); + +namespace { + +const ChipTensor &as_tensor(const EslTensor *t) { return *reinterpret_cast(t); } + +EslTensor from_tensor(const ChipTensor &t) { + EslTensor out; + std::memcpy(&out, &t, sizeof(out)); + return out; +} + +DataType to_l2_dtype(dtype_t d) { return d == BFLOAT16 ? DataType::BFLOAT16 : DataType::FLOAT32; } + +// The pending-task state the C header cannot hold: one CoreTaskArgs per ring +// slot, indexed exactly as esl_proxy indexes g_basic_buf. +struct Pending { + CoreTaskArgs args; + uint32_t count{1}; + task_type_t type{TASK_TYPE_VECTOR}; + uint32_t duration{0}; + bool open{false}; +}; + +Pending g_pending[RING_SIZE]; +esl_stats g_stats{}; + +// The case stores its tensors in its own locals and hands us pointers; L2's Arg +// also stores pointers, and both must stay valid until submit. The case's +// locals outlive their tm_submit in every instance, but a NO_DEP/INPUT arg +// pointing at a dead local would corrupt the DAG invisibly, so keep a per-slot +// copy and register that instead. Ownership is then ours and the lifetime +// question disappears. +struct ArgStore { + ChipTensor tensors[MAX_TENSOR_ARGS]; + int32_t n{0}; + + ChipTensor *add(const ChipTensor &t) { + if (n >= MAX_TENSOR_ARGS) return nullptr; + tensors[n] = t; + return &tensors[n++]; + } + void clear() { n = 0; } +}; + +ArgStore g_store[RING_SIZE]; + +[[noreturn]] void die(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + std::fprintf(stderr, "esl_shim: "); + std::vfprintf(stderr, fmt, ap); + std::fputc('\n', stderr); + va_end(ap); + std::abort(); +} + +ChipTensor *stash(uint32_t tid, const EslTensor *t) { + ChipTensor *p = g_store[tid & RING_MASK].add(as_tensor(t)); + if (p == nullptr) { + die("task %u exceeded MAX_TENSOR_ARGS (%d) tensor args", tid, MAX_TENSOR_ARGS); + } + return p; +} + +void assert_no_creator(const ChipTensor &t, const char *who) { + if (t.owner_task_id.is_valid()) { + die( + "%s received a tensor with a valid owner_task_id. An `_ro` arg creates no\n" + " dependency in esl_proxy, but L2's NO_DEP still retains its creator, so this\n" + " run's DAG would no longer match the case's semantics.", + who + ); + } +} + +} // namespace + +// --------------------------------------------------------------------------- + +extern "C" { + +struct esl_task_desc g_basic_buf[RING_SIZE]; +uint32_t g_task_id = 0; + +void esl_reset_state(void) { + std::memset(g_basic_buf, 0, sizeof(g_basic_buf)); + for (int i = 0; i < RING_SIZE; ++i) { + g_pending[i].args.reset(); + g_pending[i].count = 1; + g_pending[i].type = TASK_TYPE_VECTOR; + g_pending[i].duration = 0; + g_pending[i].open = false; + g_store[i].clear(); + } + g_task_id = 0; + g_stats = esl_stats{}; +} + +struct esl_stats esl_get_stats(void) { return g_stats; } + +void tm_deps_init(void) { + // esl_proxy builds its own TensorMap here. On L2 the TensorMap belongs to the + // runtime the harness already stood up, so there is nothing to construct — + // but per-run state must be cleared, because the bench rebuilds the runtime + // for every repetition and re-enters the case. + esl_reset_state(); +} + +int new_task(uint32_t task_id, uint32_t type, uint32_t count, uint32_t duration) { + const uint32_t slot = task_id & RING_MASK; + + struct esl_task_desc &d = g_basic_buf[slot]; + d.id = task_id; + d.type = static_cast(type); + d.count = count; + d.duration = duration; + // esl_proxy sets SPMD_SYNC whenever count > 1 (ring_buf.h:162). + d.mode = count > 1 ? ORG_MODE_SPMD_SYNC : ORG_MODE_SINGLE; + d.tensor_cnt = 0; + d.scalar_cnt = 0; + + Pending &p = g_pending[slot]; + p.args.reset(); + p.count = count; + p.type = static_cast(type); + p.duration = duration; + p.open = true; + p.args.launch_spec.set_block_num(static_cast(count)); + g_store[slot].clear(); + return 1; +} + +void add_scalar(uint32_t task_id, int64_t value) { + const uint32_t slot = task_id & RING_MASK; + g_pending[slot].args.add_scalar(value); + g_basic_buf[slot].scalar_cnt++; + g_stats.scalars++; +} + +EslTensor tensor_from_base_layout(uint64_t base, const uint32_t shapes[], uint32_t ndims, dtype_t dtype) { + return from_tensor(make_tensor_external( + reinterpret_cast(static_cast(base)), shapes, ndims, to_l2_dtype(dtype) + )); +} + +EslTensor esl_view_at(const EslTensor *t, uint32_t off0, uint32_t off1, uint32_t n0, uint32_t n1) { + // esl_proxy's view_at and ChipTensor::view compute the same thing: advance + // start_offset by sum(offset[i] * stride[i]) and keep the parent's strides. + // This is an argument reshuffle, not a reimplementation. + const uint32_t view_shapes[2] = {n0, n1}; + const uint32_t view_offsets[2] = {off0, off1}; + return from_tensor(as_tensor(t).view(view_shapes, view_offsets)); +} + +EslTensor alloc_tensors(uint32_t shape[], int dim, int bytes) { + // In esl_proxy this bumps a pool tail. On L2 it is a real hidden alloc task: + // it claims a ring slot, cuts the GM heap, registers the buffer in the + // TensorMap and sets owner_task_id so consumers retain their creator. That + // difference is the point — an alloc here costs what it costs in the engine + // under test, and shows up in the profile as `alloc_tensors`. + const uint32_t shapes[2] = {shape[0], shape[1]}; + TensorCreateInfo ci(shapes, static_cast(dim), to_l2_dtype(static_cast(bytes))); + TaskOutputTensors out = alloc_tensors(ci); + if (out.empty()) { + die("alloc_tensors([%u, %u]) returned nothing — the GM heap or ring is exhausted", shape[0], shape[1]); + } + g_stats.allocs++; + return from_tensor(out.get_ref(0)); +} + +void tm_in_ptr(uint32_t tid, const EslTensor *t) { + g_pending[tid & RING_MASK].args.add_input(*stash(tid, t)); + g_basic_buf[tid & RING_MASK].tensor_cnt++; + g_stats.tracked_args++; +} + +void tm_out_ptr(uint32_t tid, const EslTensor *t) { + g_pending[tid & RING_MASK].args.add_output(*stash(tid, t)); + g_basic_buf[tid & RING_MASK].tensor_cnt++; + g_stats.tracked_args++; +} + +void tm_inout_ptr(uint32_t tid, const EslTensor *t) { + g_pending[tid & RING_MASK].args.add_inout(*stash(tid, t)); + g_basic_buf[tid & RING_MASK].tensor_cnt++; + g_stats.tracked_args++; +} + +void tm_in_ro_ptr(uint32_t tid, const EslTensor *t) { + assert_no_creator(as_tensor(t), "tm_in_ro"); + g_pending[tid & RING_MASK].args.add_no_dep(*stash(tid, t)); + g_basic_buf[tid & RING_MASK].tensor_cnt++; + g_stats.no_dep_args++; +} + +void tm_out_ro_ptr(uint32_t tid, const EslTensor *t) { + assert_no_creator(as_tensor(t), "tm_out_ro"); + g_pending[tid & RING_MASK].args.add_no_dep(*stash(tid, t)); + g_basic_buf[tid & RING_MASK].tensor_cnt++; + g_stats.no_dep_args++; +} + +void tm_inout_ro_ptr(uint32_t tid, const EslTensor *t) { + assert_no_creator(as_tensor(t), "tm_inout_ro"); + g_pending[tid & RING_MASK].args.add_no_dep(*stash(tid, t)); + g_basic_buf[tid & RING_MASK].tensor_cnt++; + g_stats.no_dep_args++; +} + +void tm_submit_ptr(uint32_t tid) { + const uint32_t slot = tid & RING_MASK; + Pending &p = g_pending[slot]; + if (!p.open) die("tm_submit(%u) without a matching new_task", tid); + if (p.args.has_error) { + die("task %u built an invalid Arg: %s", tid, p.args.error_msg ? p.args.error_msg : "(unknown)"); + } + // The C-visible descriptor and the real pending state must agree; they can + // only diverge if the case wrote g_basic_buf directly (set_block_num), which + // would not reach the engine. Fail loudly instead of submitting a task whose + // SPMD width silently differs from what the case asked for. + if (g_basic_buf[slot].count != p.count) { + die( + "task %u: g_basic_buf.count=%u disagrees with the submitted block_num=%u.\n" + " The case mutated the descriptor directly; that path does not reach the engine.", + tid, g_basic_buf[slot].count, p.count + ); + } + + MixedKernels mk; + // CUBE -> AIC, VECTOR/MIX -> AIV0. Kernel ids are opaque to the + // orchestrator: it stores them in the descriptor and the AICore would + // resolve them, so any distinct valid value carries identical cost. + if (p.type == TASK_TYPE_CUBE) { + mk.aic_kernel_id = 1; + } else { + mk.aiv0_kernel_id = 2; + } + + (void)rt_submit_task(mk, p.args); + + g_stats.tasks++; + g_stats.subtasks += p.count; + g_stats.duration_ns += p.duration; + p.open = false; +} + +} // extern "C" diff --git a/l2-orchestrator-standalone/bench/esl_shim/mem_pool.h b/l2-orchestrator-standalone/bench/esl_shim/mem_pool.h new file mode 100644 index 0000000000..06e4fb86e7 --- /dev/null +++ b/l2-orchestrator-standalone/bench/esl_shim/mem_pool.h @@ -0,0 +1,8 @@ +/* + * esl_proxy mem_pool.h — the case includes this name; the API lives in the + * shared C ABI header alongside it. + */ +#ifndef ESL_SHIM_MEM_POOL_H +#define ESL_SHIM_MEM_POOL_H +#include "esl_c_abi.h" +#endif diff --git a/l2-orchestrator-standalone/bench/esl_shim/tensormap.h b/l2-orchestrator-standalone/bench/esl_shim/tensormap.h new file mode 100644 index 0000000000..d7ce2e1a09 --- /dev/null +++ b/l2-orchestrator-standalone/bench/esl_shim/tensormap.h @@ -0,0 +1,8 @@ +/* + * esl_proxy tensormap.h — the case includes this name; the API lives in the + * shared C ABI header alongside it. + */ +#ifndef ESL_SHIM_TENSORMAP_H +#define ESL_SHIM_TENSORMAP_H +#include "esl_c_abi.h" +#endif diff --git a/l2-orchestrator-standalone/bench/l2_bench.cpp b/l2-orchestrator-standalone/bench/l2_bench.cpp new file mode 100644 index 0000000000..d10354555d --- /dev/null +++ b/l2-orchestrator-standalone/bench/l2_bench.cpp @@ -0,0 +1,285 @@ +/* + * L2 orchestration bench driver. + * + * Measures the four-line sequence + * + * rt_scope_begin(rt); entry(orch_l2); rt_scope_end(rt); rt_orchestration_done(rt); + * + * against qwen3_dynamic_tensormap.h at QWEN3_SPMD_TIER=0, on a fresh runtime + * per repetition. + * + * Modes + * --mode=throughput N clean repetitions, wall clock only. No interception, + * so these are the numbers to quote. + * --mode=profile One clean run (for the honest total) followed by one + * instrumented run through the replacement ops table, so + * the instrument's own cost is reported instead of folded + * in silently. See bench/l2_profile.h. + * + * WHY A FRESH RUNTIME PER REPETITION + * ---------------------------------- + * Nothing ever completes here — there is no scheduler and no device — so + * last_task_alive never advances and the ring's watermark reclaim never fires. + * A second run on the same runtime would therefore start with the task window, + * GM heap and TensorMap entry pool already consumed by the first, and would + * measure back-pressure rather than steady-state submit. Tearing down and + * rebuilding is the only way each repetition sees the same initial conditions. + * + * That same fact bounds what the payload can ask for: task_window must exceed + * its total task count and heap_bytes must exceed its total allocation, because + * neither is ever recycled. --task-window and --heap-mb exist for that, and + * exhausting either latches a fatal that this driver reports rather than + * silently reporting a small, fast graph. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "l2_harness.h" +#include "l2_profile.h" +#include "payload.h" +#include "pto_runtime2.h" + +using namespace l2_bench; + +namespace { + +struct Options { + std::string mode = "throughput"; + int repeat = 5; + // Defaults hold the whole graph: 3875 engine tasks, nothing reclaimed. + uint64_t task_window = 8192; + uint64_t heap_mb = 2048; + bool prefault_sm = false; + bool prefault_arena = false; + std::string out; +}; + +[[noreturn]] void usage(int code) { + std::printf( + "l2_bench — bench and profile the L2 orchestration sequence\n" + " payload: qwen3_dynamic_tensormap.h, QWEN3_SPMD_TIER=0\n\n" + " --mode=throughput|profile default throughput\n" + " --repeat=N repetitions in throughput mode (default 5)\n" + " --task-window=N ring task slots (default 8192; never reclaimed here)\n" + " --heap-mb=N GM heap stand-in in MiB (default 2048; never reclaimed here)\n" + " --prefault-sm DIAGNOSTIC: pre-touch the SM so cold-slot cost is excluded\n" + " --prefault-arena DIAGNOSTIC: pre-touch the arena (TensorMap buckets + pool)\n" + " --prefault-all both of the above\n" + " --out=PATH write the JSON report here (profile mode)\n" + ); + std::exit(code); +} + +bool arg_u64(const std::string &a, const char *key, uint64_t &out) { + const std::string prefix = std::string("--") + key + "="; + if (a.rfind(prefix, 0) != 0) return false; + out = std::strtoull(a.c_str() + prefix.size(), nullptr, 10); + return true; +} + +bool arg_int(const std::string &a, const char *key, int &out) { + uint64_t v = 0; + if (!arg_u64(a, key, v)) return false; + out = static_cast(v); + return true; +} + +bool arg_str(const std::string &a, const char *key, std::string &out) { + const std::string prefix = std::string("--") + key + "="; + if (a.rfind(prefix, 0) != 0) return false; + out = a.substr(prefix.size()); + return true; +} + +Options parse(int argc, char **argv) { + Options o; + for (int i = 1; i < argc; ++i) { + const std::string a = argv[i]; + if (a == "--help" || a == "-h") usage(0); + else if (arg_str(a, "mode", o.mode)) {} + else if (arg_str(a, "out", o.out)) {} + else if (arg_int(a, "repeat", o.repeat)) {} + else if (arg_u64(a, "task-window", o.task_window)) {} + else if (arg_u64(a, "heap-mb", o.heap_mb)) {} + else if (a == "--prefault-sm") o.prefault_sm = true; + else if (a == "--prefault-arena") o.prefault_arena = true; + else if (a == "--prefault-all") { o.prefault_sm = true; o.prefault_arena = true; } + else { + std::fprintf(stderr, "unknown argument: %s\n", a.c_str()); + usage(2); + } + } + if (o.mode != "throughput" && o.mode != "profile") { + std::fprintf(stderr, "--mode must be throughput|profile\n"); + std::exit(2); + } + if (o.repeat < 1) o.repeat = 1; + return o; +} + +struct RunResult { + uint64_t scope_begin_ns{0}; + uint64_t entry_ns{0}; + uint64_t scope_end_ns{0}; + uint64_t done_ns{0}; + uint64_t total_ns{0}; + int32_t tasks{0}; + bool fatal{false}; + int32_t error_code{0}; +}; + +// One complete lifecycle: build a runtime, run the four lines, tear it down. +// `prof` non-null installs the replacement ops table for the duration of the +// entry only — the Level-1 numbers around it stay comparable either way. +RunResult run_once(const Options &o, Profiler *prof) { + HarnessConfig hc; + hc.task_window = o.task_window; + hc.heap_bytes = o.heap_mb << 20; + hc.prefault_sm = o.prefault_sm; + hc.prefault_arena = o.prefault_arena; + Harness h(hc); + + PayloadArgs args; + qwen3_dyn_build_args(args); + + PTO2Runtime *rt = h.rt(); + RunResult r; + const uint64_t t0 = now_ns(); + + { + // Scoped so the table is restored before the harness tears down. + std::unique_ptr interceptor; + if (prof != nullptr) interceptor.reset(new OpsInterceptor(rt, prof)); + + const uint64_t a = now_ns(); + rt_scope_begin(rt); + const uint64_t b = now_ns(); + qwen3_dyn_entry(args.args); + const uint64_t c = now_ns(); + rt_scope_end(rt); + const uint64_t d = now_ns(); + rt_orchestration_done(rt); + const uint64_t e = now_ns(); + + r.scope_begin_ns = b - a; + r.entry_ns = c - b; + r.scope_end_ns = d - c; + r.done_ns = e - d; + } + + r.total_ns = now_ns() - t0; + r.tasks = h.active_task_count(); + r.fatal = h.fatal(); + r.error_code = h.error_code(); + return r; +} + +void check(const RunResult &r) { + if (r.fatal) { + std::fprintf( + stderr, + "l2_bench: the orchestrator latched a fatal (error_code=%d) — every submit after\n" + "that point was a silent no-op, so the task count and timings below are NOT a\n" + "measurement of the workload. Raise --task-window / --heap-mb and re-run.\n", + r.error_code + ); + std::exit(1); + } + if (r.tasks <= 0) { + std::fprintf(stderr, "l2_bench: no tasks were claimed — the payload built an empty graph\n"); + std::exit(1); + } +} + +int mode_throughput(const Options &o) { + std::vector totals; + std::vector entries; + RunResult last; + for (int i = 0; i < o.repeat; ++i) { + last = run_once(o, nullptr); + check(last); + totals.push_back(last.total_ns); + entries.push_back(last.entry_ns); + } + std::sort(totals.begin(), totals.end()); + std::sort(entries.begin(), entries.end()); + const uint64_t med = totals[totals.size() / 2]; + const uint64_t med_entry = entries[entries.size() / 2]; + + std::printf("payload=qwen3-dyn tasks=%d repeat=%d\n", last.tasks, o.repeat); + std::printf( + "four-line block median %10.3f ms min %10.3f ms max %10.3f ms\n", med / 1e6, totals.front() / 1e6, + totals.back() / 1e6 + ); + std::printf(" of which entry() median %10.3f ms (%.1f%%)\n", med_entry / 1e6, 100.0 * med_entry / med); + std::printf( + "per task %8.2f us throughput %12.0f tasks/s\n", med / 1000.0 / last.tasks, + last.tasks / (med / 1e9) + ); + qwen3_dyn_report(stdout); + return 0; +} + +int mode_profile(const Options &o) { + // Clean pass FIRST, so the instrumented pass cannot be credited with a warm + // allocator or warm page cache the clean one did not have. + const RunResult clean = run_once(o, nullptr); + check(clean); + + Profiler prof; +#if SIMPLER_ORCH_PROFILING + // Drain the engine's global step counters NOW, while they hold only the + // clean run. They reset on read, so the instrumented run below starts from + // zero and never contaminates this snapshot. + prof.set_engine_steps(orchestrator_get_profiling()); +#endif +#if SIMPLER_TENSORMAP_PROFILING + prof.set_tensormap_stats(pto2_tensormap_get_profiling()); +#endif + prof.set_clock_cost(measure_clock_cost()); + const RunResult inst = run_once(o, &prof); + check(inst); + + // Level 1 rows come from the CLEAN run — the interception distorts entry() + // and nothing else, so mixing the two would misattribute its cost. + prof.phase("rt_scope_begin", clean.scope_begin_ns); + prof.phase("entry", clean.entry_ns); + prof.phase("rt_scope_end", clean.scope_end_ns); + prof.phase("rt_orchestration_done", clean.done_ns); + prof.set_totals(clean.tasks, clean.total_ns, inst.total_ns); + + prof.report(stdout); + qwen3_dyn_report(stdout); + if (!o.out.empty()) { + std::FILE *f = std::fopen(o.out.c_str(), "w"); + if (f == nullptr) { + std::fprintf(stderr, "cannot write %s\n", o.out.c_str()); + return 1; + } + prof.report_json(f); + std::fclose(f); + std::fprintf(stderr, "l2_bench: profile -> %s\n", o.out.c_str()); + } + return 0; +} + +} // namespace + +int main(int argc, char **argv) { + const Options o = parse(argc, argv); + try { + if (o.mode == "throughput") return mode_throughput(o); + return mode_profile(o); + } catch (const std::exception &e) { + std::fprintf(stderr, "l2_bench: failed: %s\n", e.what()); + return 1; + } +} diff --git a/l2-orchestrator-standalone/bench/l2_harness.h b/l2-orchestrator-standalone/bench/l2_harness.h new file mode 100644 index 0000000000..739921f7ad --- /dev/null +++ b/l2-orchestrator-standalone/bench/l2_harness.h @@ -0,0 +1,224 @@ +/* + * Host-only assembly of one L2 runtime, so the four lines + * + * rt_scope_begin(rt); + * entry_points->entry(orch_l2); + * rt_scope_end(rt); + * rt_orchestration_done(rt); + * + * can be driven without CANN, without a device, and without an orchestration + * .so. + * + * WHERE THIS SEQUENCE COMES FROM + * ------------------------------ + * It is the host half of simpler's own host_build_graph path, in call order: + * + * runtime_maker.cpp:831 sm_size = calculate_size_per_ring(task_window) + * runtime_maker.cpp:834 DeviceArena host_arena + * runtime_maker.cpp:835 runtime_reserve_layout(arena, task_window, heap) + * runtime_maker.cpp:836 arena.commit() + * runtime_maker.cpp:885 runtime_init_data_from_layout(..., PTO2_MODE_EXECUTE, ...) + * runtime_maker.cpp:891 runtime_wire_arena_pointers(arena, layout, rt) + * run_host_orchestration:487 host SM buffer + memset of the header segment + * run_host_orchestration:493 orchestrator.init_data_from_layout(host SM) + * run_host_orchestration:499 orchestrator.wire_arena_pointers(&rt->scheduler) + * run_host_orchestration:502 host_sm_handle.init_per_ring(...) + * run_host_orchestration:518 runtime_finalize_after_wire(rt, aic, aiv) + * run_host_orchestration:520 rt->mode = PTO2_MODE_EXECUTE + * run_host_orchestration:534 entry_points->bind(rt) + * + * THREE SUBSTITUTIONS, AND WHY EACH IS SOUND + * ------------------------------------------ + * 1. `gm_heap` is a host malloc rather than a device GM allocation. The + * orchestrator only does address arithmetic on it — it hands out slices as + * task output buffers and never dereferences one (the AICore would). A host + * address is therefore indistinguishable to the code under measurement. + * + * 2. The SM is a plain host buffer. That is not a substitution at all: the + * host-orch path already runs the orchestrator against a host SM mirror + * (run_host_orchestration:487-499) and only H2Ds the populated image + * afterwards. We stop before the H2D. + * + * 3. `entry_points->bind(rt)` becomes a direct framework_bind_runtime(rt) call. + * In simpler the orchestration code is a dlopen'd .so, so binding has to go + * through an exported symbol; here it is linked into the same binary and the + * function is the same one (orchestration/common.cpp:42). + * + * WHAT IS DELIBERATELY ABSENT + * --------------------------- + * Everything after mark_done: upload_graph_submissions, the host->device + * pointer relocation (runtime_maker.cpp:558+), the H2D of the SM and arena, and + * the device-side scheduler boot. This package measures graph CONSTRUCTION, not + * execution. No task ever runs. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "pto_orchestrator.h" +#include "pto_runtime2.h" +#include "pto_shared_memory.h" +#include "pto_types.h" +#include "utils/device_arena.h" + +namespace l2_bench { + +struct HarnessConfig { + uint64_t task_window = 8192; // ring task slots + uint64_t heap_bytes = 1ULL << 30; // GM heap stand-in, per ring + int32_t aic_count = 24; // MIX clusters this "run" advertises + int32_t aiv_count = 48; // AIV cores + uint64_t callable_hash = 0x1220BE0CULL; // rt->active_callable_hash + // DIAGNOSTIC ONLY. Touch every page of the SM before the run. + // + // Nothing is ever reclaimed here, so each submit writes into a slot that has + // never been touched: with task_window=8192 the payload region alone is + // ~38 MB, and every submit pays a cold miss plus a first-touch page fault on + // its own ~4.8 KB slot. Production wraps the ring and reuses warm slots, so + // that cost is an artefact of this harness. Setting this pre-faults the + // region so the difference can be measured instead of assumed. + // + // Semantically safe: every SM byte the engine reads is written first + // (init-on-write in prepare_task / payload.init), so pre-zeroing cannot + // change what it observes. + bool prefault_sm = false; + // Same diagnostic for the runtime arena, which holds the TensorMap buckets + // and entry pool that STEP 3/4 walk. Pre-faulting only the SM would leave + // the arena cold and mis-attribute its first-touch cost to the lookup. + bool prefault_arena = false; +}; + +// One complete runtime, torn down on destruction. Non-copyable: PTO2Runtime +// lives inside the arena and holds pointers back into it, so the whole thing is +// pinned in place for its lifetime. +class Harness { +public: + explicit Harness(const HarnessConfig &cfg) : cfg_(cfg) { build(); } + + Harness(const Harness &) = delete; + Harness &operator=(const Harness &) = delete; + + PTO2Runtime *rt() const { return rt_; } + PTO2OrchestratorState &orch() const { return rt_->orchestrator; } + + // Task slots consumed so far — the engine's own count, read straight off + // the ring allocator rather than tallied by the driver. + int32_t active_task_count() const { return rt_->orchestrator.ring.task_allocator.active_count(); } + + // Set by mark_done(); the scheduler's "no more work is coming" signal. + bool orchestration_done() const { + return rt_->orchestrator.sm_header->orchestrator_done.load(std::memory_order_acquire) != 0; + } + + // Non-zero once the orchestrator latches a fatal. Any submit after that + // point is a silent no-op, so a bench MUST check this before believing a + // task count or a throughput number. + int32_t error_code() const { + return rt_->orchestrator.sm_header->orch_error_code.load(std::memory_order_acquire); + } + bool fatal() const { return rt_->orchestrator.fatal || error_code() != 0; } + +private: + void build() { + task_window_[0] = cfg_.task_window; + heap_sizes_[0] = cfg_.heap_bytes; + + sm_size_ = PTO2SharedMemoryHandle::calculate_size_per_ring(task_window_); + + layout_ = runtime_reserve_layout(arena_, task_window_, heap_sizes_); + if (arena_.commit(DeviceArena::kDefaultBaseAlign) == nullptr) { + throw std::runtime_error("DeviceArena::commit failed (arena_size=" + std::to_string(layout_.arena_size) + ")"); + } + // Must run BEFORE init_data_from_layout writes into the arena, and it is + // safe for the same reason as the SM: every arena byte the engine reads + // is written by the init phases first. + if (cfg_.prefault_arena) std::memset(arena_.base(), 0, arena_.total_size()); + + // Stands in for the device GM heap. aligned_alloc keeps it on the same + // 1024-byte granularity the device allocator guarantees, so the + // orchestrator's alignment arithmetic sees what it would on-device. + gm_heap_ = std::aligned_alloc(DeviceArena::kDefaultBaseAlign, round_up(cfg_.heap_bytes)); + if (gm_heap_ == nullptr) throw std::bad_alloc(); + + sm_buf_.reset(new uint8_t[sm_size_]); + void *sm = sm_buf_.get(); + if (cfg_.prefault_sm) std::memset(sm, 0, sm_size_); + + rt_ = runtime_init_data_from_layout( + arena_, layout_, PTO2_MODE_EXECUTE, sm, sm_size_, gm_heap_, heap_sizes_ + ); + if (rt_ == nullptr) throw std::runtime_error("runtime_init_data_from_layout failed"); + runtime_wire_arena_pointers(arena_, layout_, rt_); + + // Init-on-write: only the fixed header segment is zeroed; per-slot + // descriptors/payloads/slot_states are written by prepare_task as each + // slot is claimed. Zeroing the whole SM here would both cost more and + // hide a missing per-slot init. + const pto2_sm_layout::PTO2RingSegmentOffsets segs = + pto2_sm_layout::ring_segment_offsets(task_window_[0]); + std::memset(sm, 0, segs.descriptors); + + if (!rt_->orchestrator.init_data_from_layout( + layout_.orch, arena_, sm, gm_heap_, heap_sizes_[0], task_window_[0] + )) { + throw std::runtime_error("orchestrator.init_data_from_layout failed"); + } + rt_->orchestrator.wire_arena_pointers(layout_.orch, arena_, &rt_->scheduler); + + if (!sm_handle_.init_per_ring(sm, sm_size_, task_window_, heap_sizes_)) { + throw std::runtime_error("PTO2SharedMemoryHandle::init_per_ring failed"); + } + + // Fills rt->ops from the runtime's own s_runtime_ops table and sets the + // orchestrator's core counts, which submit_task reads for its + // require_sync_start deadlock check. + runtime_finalize_after_wire(rt_, cfg_.aic_count, cfg_.aiv_count); + rt_->mode = PTO2_MODE_EXECUTE; + rt_->active_callable_hash = cfg_.callable_hash; + // No host tensor views are staged, so get_tensor_data/set_tensor_data + // fail closed rather than dereferencing a device address. Payloads that + // read tensor contents during orchestration are out of scope here. + rt_->tensor_access = nullptr; + + framework_bind_runtime(rt_); + } + + static size_t round_up(uint64_t n) { + const uint64_t a = DeviceArena::kDefaultBaseAlign; + return static_cast((n + a - 1) & ~(a - 1)); + } + + struct HeapDeleter { + void operator()(void *p) const { std::free(p); } + }; + + HarnessConfig cfg_; + uint64_t task_window_[PTO2_MAX_RING_DEPTH]{}; + uint64_t heap_sizes_[PTO2_MAX_RING_DEPTH]{}; + uint64_t sm_size_{0}; + + DeviceArena arena_; + PTO2RuntimeArenaLayout layout_{}; + std::unique_ptr sm_buf_; + void *gm_heap_{nullptr}; + PTO2SharedMemoryHandle sm_handle_{}; + PTO2Runtime *rt_{nullptr}; + +public: + ~Harness() { + framework_bind_runtime(nullptr); + if (rt_ != nullptr) runtime_destroy(rt_, arena_); + if (gm_heap_ != nullptr) std::free(gm_heap_); + } +}; + +} // namespace l2_bench diff --git a/l2-orchestrator-standalone/bench/l2_profile.cpp b/l2-orchestrator-standalone/bench/l2_profile.cpp new file mode 100644 index 0000000000..308e4afb4f --- /dev/null +++ b/l2-orchestrator-standalone/bench/l2_profile.cpp @@ -0,0 +1,256 @@ +/* + * Report rendering for l2_profile.h, plus the interceptor's statics. + */ + +#include "l2_profile.h" + +namespace l2_bench { + +const PTO2RuntimeOps *OpsInterceptor::g_original = nullptr; +Profiler *OpsInterceptor::g_prof = nullptr; + +void Profiler::table(std::FILE *f, const char *title, const std::map &rows, uint64_t denom) const { + std::fprintf(f, "\n%s\n", title); + std::fprintf( + f, " %-28s %8s %10s %10s %10s %10s %10s %7s\n", "step", "count", "total_ms", "mean_ns", "p50_ns", "p99_ns", + "max_ns", "%" + ); + std::fprintf(f, " %s\n", std::string(108, '-').c_str()); + for (const auto &kv : rows) { + const Step &s = kv.second; + const double pct = denom > 0 ? 100.0 * static_cast(s.total()) / static_cast(denom) : 0.0; + std::fprintf( + f, " %-28s %8zu %10.3f %10llu %10llu %10llu %10llu %6.2f%%\n", kv.first.c_str(), s.count(), + static_cast(s.total()) / 1e6, static_cast(s.mean()), + static_cast(s.pct(0.5)), static_cast(s.pct(0.99)), + static_cast(s.max()), pct + ); + } +} + +void Profiler::report(std::FILE *f) const { + // `tasks_` is the engine's own ring-slot count, which includes the framework + // alloc tasks; `subtasks_` sums block_num over kernel submits ONLY, because + // that is the only path the ops-table interceptor sees a launch_spec on. + // Printing the two side by side without saying so reads as "N tasks expand + // into M subtasks" and makes M < N look like a bug. Break the count out + // instead, so every number states its own population. + const auto op_count = [this](const char *name) -> uint64_t { + auto it = ops_.find(name); + return it == ops_.end() ? 0 : static_cast(it->second.count()); + }; + const uint64_t submits = op_count("submit_task") + op_count("submit_dummy_task"); + const uint64_t allocs = op_count("alloc_tensors"); + + std::fprintf(f, "%s\n", std::string(110, '=').c_str()); + std::fprintf(f, "L2 Orchestrator profile — qwen3_dynamic_tensormap.h (QWEN3_SPMD_TIER=0)\n"); + std::fprintf( + f, " %llu kernel submits (%llu SPMD subtasks) + %llu framework allocs = %d engine tasks\n", + static_cast(submits), static_cast(subtasks_), + static_cast(allocs), tasks_ + ); + // A mismatch means the clean and instrumented runs built different graphs, + // which would silently invalidate every per-call number below. + if (submits + allocs != static_cast(tasks_)) { + std::fprintf( + f, + " WARNING: %llu + %llu != %d — the instrumented run did not build the same\n" + " graph as the clean one, so the tables below are not comparable.\n", + static_cast(submits), static_cast(allocs), tasks_ + ); + } + std::fprintf(f, "engine: a2a3 / host_build_graph, PTO2OrchestratorState (L2)\n"); + std::fprintf(f, "%s\n", std::string(110, '=').c_str()); + + uint64_t phase_total = 0; + for (const auto &kv : phases_) phase_total += kv.second.total(); + + std::fprintf(f, "\nLEVEL 1 — the four lines (driver-bracketed, uninstrumented run)\n"); + std::fprintf(f, " %-28s %14s %14s\n", "step", "ns", "% of block"); + std::fprintf(f, " %s\n", std::string(58, '-').c_str()); + // Fixed order: this is a sequence, and sorting it alphabetically would + // destroy the one property a reader needs from it. + for (const char *name : {"rt_scope_begin", "entry", "rt_scope_end", "rt_orchestration_done"}) { + auto it = phases_.find(name); + if (it == phases_.end()) continue; + const uint64_t t = it->second.total(); + const double pct = phase_total > 0 ? 100.0 * static_cast(t) / static_cast(phase_total) : 0.0; + std::fprintf(f, " %-28s %14llu %13.2f%%\n", name, static_cast(t), pct); + } + std::fprintf(f, " %-28s %14llu\n", "TOTAL", static_cast(phase_total)); + if (tasks_ > 0) { + std::fprintf( + f, "\n per task: %.2f us throughput: %.0f tasks/s\n", + static_cast(phase_total) / 1000.0 / tasks_, + static_cast(tasks_) / (static_cast(phase_total) / 1e9) + ); + } + + if (clean_total_ns_ > 0 && inst_total_ns_ > 0) { + const double delta = + 100.0 * (static_cast(inst_total_ns_) - static_cast(clean_total_ns_)) / + static_cast(clean_total_ns_); + std::fprintf(f, "\nINSTRUMENT COST — quote throughput from the clean run, shape from the tables below\n"); + std::fprintf(f, " uninstrumented total %10.3f ms\n", static_cast(clean_total_ns_) / 1e6); + std::fprintf( + f, " instrumented total %10.3f ms (%+.1f%%)\n", static_cast(inst_total_ns_) / 1e6, delta + ); + std::fprintf(f, " one steady_clock read %10llu ns\n", static_cast(clock_cost_ns_)); + } + + std::fprintf( + f, + "\nREAD p50/p99, NOT mean. A single OS scheduling stall moves a mean by multiples\n" + "while leaving p50 and p99 untouched. The mean column reconciles total/count; it\n" + "is not the robust statistic.\n" + ); + + uint64_t ops_total = 0; + for (const auto &kv : ops_) ops_total += kv.second.total(); + table(f, "LEVEL 2 — every entry->engine call (intercepted ops table)", ops_, ops_total); + + if (!by_tensor_count_.empty()) { + std::fprintf(f, "\nSUBMIT LATENCY BY TENSOR-ARG COUNT — the per-tensor slope of STEP 3/4\n"); + std::fprintf(f, " %-12s %8s %10s %10s %12s\n", "tensors", "count", "p50_ns", "p99_ns", "p50/tensor"); + std::fprintf(f, " %s\n", std::string(56, '-').c_str()); + for (const auto &kv : by_tensor_count_) { + const uint64_t p50 = kv.second.pct(0.5); + std::fprintf( + f, " %-12d %8zu %10llu %10llu %12.1f\n", kv.first, kv.second.count(), + static_cast(p50), static_cast(kv.second.pct(0.99)), + kv.first > 0 ? static_cast(p50) / kv.first : 0.0 + ); + } + } + +#if SIMPLER_ORCH_PROFILING + if (!have_engine_steps_) { + std::fprintf(f, "\nLEVEL 3 — no snapshot was taken; call set_engine_steps() after the clean run.\n"); + } else { + const PTO2OrchProfilingData &d = engine_steps_; + // The clock the shim feeds get_sys_cnt_aicpu() is cntvct_el0, so the + // divisor must be cntfrq_el0 — NOT PLATFORM_PROF_SYS_CNT_FREQ, which is + // the device's 50 MHz counter and is 2x off on this host. + const uint64_t hz = device_time_frequency_hz(); + const auto to_ns = [hz](uint64_t cycles) -> double { + return hz > 0 ? static_cast(cycles) * 1e9 / static_cast(hz) : 0.0; + }; + uint64_t sum = d.alloc_cycle + d.sync_cycle + d.lookup_cycle + d.insert_cycle + d.args_cycle + d.fanin_cycle + + d.scope_end_cycle; + std::fprintf(f, "\nLEVEL 3 — the engine's own step counters inside submit_task_common\n"); + std::fprintf( + f, " (SIMPLER_ORCH_PROFILING=1, uninstrumented run; submit_count=%lld, clock=%llu Hz)\n", + static_cast(d.submit_count), static_cast(hz) + ); + std::fprintf(f, " %-46s %12s %12s %8s\n", "step", "cycles", "total_ms", "%"); + std::fprintf(f, " %s\n", std::string(82, '-').c_str()); + const struct { + const char *name; + uint64_t cycles; + } rows[] = { + {"STEP 1 prepare_task (slot + heap alloc)", d.alloc_cycle}, + {"STEP 2 sync_tensormap", d.sync_cycle}, + {"STEP 3 infer deps: TensorMap lookup", d.lookup_cycle}, + {"STEP 4 register outputs: TensorMap insert", d.insert_cycle}, + {"STEP 5 payload/descriptor GM write", d.args_cycle}, + {"STEP 6 publish fanin_count", d.fanin_cycle}, + {" end_scope (outside submit)", d.scope_end_cycle}, + }; + for (const auto &r : rows) { + std::fprintf( + f, " %-46s %12llu %12.3f %7.2f%%\n", r.name, static_cast(r.cycles), + to_ns(r.cycles) / 1e6, sum > 0 ? 100.0 * static_cast(r.cycles) / static_cast(sum) : 0.0 + ); + } + std::fprintf( + f, " waits: alloc %llu cycles, fanin %llu cycles\n", static_cast(d.alloc_wait_cycle), + static_cast(d.fanin_wait_cycle) + ); + } + +#if SIMPLER_TENSORMAP_PROFILING + if (have_tm_stats_) { + const PTO2TensorMapProfilingData &t = tm_stats_; + const double avg_chain = + t.lookup_count > 0 ? static_cast(t.lookup_chain_total) / static_cast(t.lookup_count) : 0.0; + std::fprintf(f, "\nLEVEL 4 — inside STEP 3: what the TensorMap lookup actually walks\n"); + std::fprintf(f, " lookups %10llu\n", (unsigned long long)t.lookup_count); + std::fprintf(f, " inserts %10llu\n", (unsigned long long)t.insert_count); + std::fprintf(f, " bucket entries walked (total) %10llu\n", (unsigned long long)t.lookup_chain_total); + std::fprintf(f, " avg chain length %13.2f\n", avg_chain); + std::fprintf(f, " MAX chain length %10d\n", t.lookup_chain_max); + std::fprintf(f, " overlap checks %10llu\n", (unsigned long long)t.overlap_checks); + std::fprintf( + f, " overlap hits %10llu (%.1f%% of checks)\n", (unsigned long long)t.overlap_hits, + t.overlap_checks > 0 ? 100.0 * (double)t.overlap_hits / (double)t.overlap_checks : 0.0 + ); + // The map hashes on buffer.addr ALONE (pto_tensormap.h:522), so every + // sub-view of one buffer shares a bucket. A chain far longer than the + // average therefore means one heavily-subdivided buffer is serialising + // every lookup that touches it — that is a data-structure problem, not a + // per-call constant, and it is what makes STEP 3 scale with SPMD width. + if (avg_chain > 0.0) { + std::fprintf( + f, " wasted walk (checks that found no overlap): %.1f%%\n", + t.overlap_checks > 0 ? 100.0 * (1.0 - (double)t.overlap_hits / (double)t.overlap_checks) : 0.0 + ); + } + } +#endif +#else + std::fprintf( + f, + "\nLEVEL 3 — not built. The engine's per-STEP counters are compiled out at\n" + "SIMPLER_ORCH_PROFILING=0 (the default). Rebuild with -DL2_ORCH_PROFILING=1\n" + "to get the six-step breakdown inside submit_task_common.\n" + ); +#endif + std::fprintf(f, "\n"); +} + +void Profiler::report_json(std::FILE *f) const { + std::fprintf(f, "{\n"); + std::fprintf(f, " \"payload\": \"qwen3-dyn\",\n"); + std::fprintf(f, " \"engine\": \"a2a3/host_build_graph L2 PTO2OrchestratorState\",\n"); + // Named for their populations, not "tasks"/"subtasks": engine_tasks counts + // ring slots (kernel submits + framework allocs), spmd_subtasks sums + // block_num over kernel submits only. + std::fprintf(f, " \"engine_tasks\": %d,\n", tasks_); + std::fprintf(f, " \"spmd_subtasks\": %llu,\n", static_cast(subtasks_)); + std::fprintf(f, " \"clean_total_ns\": %llu,\n", static_cast(clean_total_ns_)); + std::fprintf(f, " \"instrumented_total_ns\": %llu,\n", static_cast(inst_total_ns_)); + std::fprintf(f, " \"clock_read_ns\": %llu,\n", static_cast(clock_cost_ns_)); + std::fprintf(f, " \"orch_profiling_built\": %s,\n", SIMPLER_ORCH_PROFILING ? "true" : "false"); + + const auto emit_steps = [&](const char *key, const std::map &rows, bool last) { + std::fprintf(f, " \"%s\": {\n", key); + for (auto it = rows.begin(); it != rows.end(); ++it) { + std::fprintf( + f, " \"%s\": {\"count\": %zu, \"total_ns\": %llu, \"mean_ns\": %llu, \"p50_ns\": %llu, " + "\"p99_ns\": %llu, \"max_ns\": %llu}%s\n", + it->first.c_str(), it->second.count(), static_cast(it->second.total()), + static_cast(it->second.mean()), + static_cast(it->second.pct(0.5)), + static_cast(it->second.pct(0.99)), + static_cast(it->second.max()), std::next(it) == rows.end() ? "" : "," + ); + } + std::fprintf(f, " }%s\n", last ? "" : ","); + }; + + emit_steps("phases", phases_, false); + emit_steps("ops", ops_, false); + + std::fprintf(f, " \"submit_by_tensor_count\": {\n"); + for (auto it = by_tensor_count_.begin(); it != by_tensor_count_.end(); ++it) { + std::fprintf( + f, " \"%d\": {\"count\": %zu, \"p50_ns\": %llu, \"p99_ns\": %llu}%s\n", it->first, it->second.count(), + static_cast(it->second.pct(0.5)), + static_cast(it->second.pct(0.99)), + std::next(it) == by_tensor_count_.end() ? "" : "," + ); + } + std::fprintf(f, " }\n}\n"); +} + +} // namespace l2_bench diff --git a/l2-orchestrator-standalone/bench/l2_profile.h b/l2-orchestrator-standalone/bench/l2_profile.h new file mode 100644 index 0000000000..d6944ce001 --- /dev/null +++ b/l2-orchestrator-standalone/bench/l2_profile.h @@ -0,0 +1,268 @@ +/* + * Three-level profiler for the L2 orchestration sequence. + * + * WHERE THE TIMESTAMPS COME FROM — no engine source is modified. + * + * Level 1 The four calls, bracketed by the driver: + * rt_scope_begin / entry / rt_scope_end / rt_orchestration_done + * This is the only level whose numbers are free of instrument cost. + * + * Level 2 Every call the entry makes back into the runtime, via a REPLACEMENT + * OPS TABLE. `rt->ops` is a plain `const PTO2RuntimeOps *` that + * runtime_finalize_after_wire points at the runtime's own s_runtime_ops + * (pto_runtime2.cpp:366-380), and the orchestration API reaches the + * engine exclusively through it (pto_orchestration_api.h:175, 125, 245, + * 254, 262 — every one is `rt->ops->…`). Copying that table, wrapping + * each entry with a clock read, and pointing rt->ops at the copy + * therefore intercepts 100% of the entry->engine traffic without + * touching a line of engine code and without a build flag. + * + * This is a strictly better seam than L3's set_test_hook: it needs no + * cooperation from the engine at all, and it cannot miss a call site. + * + * Level 3 The engine's OWN per-step cycle counters, which exist in the source + * already: submit_task_common laps CYCLE_COUNT_LAP into g_orch_*_cycle + * at each of its six STEP boundaries, and orchestrator_get_profiling() + * returns and resets them (pto_orchestrator.cpp:2043). They are + * compiled out unless SIMPLER_ORCH_PROFILING=1, so this level is + * present only in the -DL2_ORCH_PROFILING=1 build and the report says + * so rather than printing zeros. + * + * READ p50/p99, NOT mean. One OS scheduling stall inflates a mean past + * recognition while leaving the percentiles untouched; the mean column exists + * because total/count must reconcile, not because it is robust. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +// Runtime side only. pto_orchestration_api.h carries its own partial +// PTO2Runtime / PTO2RuntimeOps definitions for the orchestration .so, and the +// two headers cannot coexist in one TU — including both is a hard redefinition +// error. The driver is the runtime side (same as runtime_maker.cpp), the +// payloads are the .so side, and PTO2RuntimeOps is field-for-field identical +// across the two declarations by contract. +#include "pto_runtime2.h" +#include "profiling_config.h" + +#if SIMPLER_ORCH_PROFILING +#include "pto_orchestrator.h" +#endif +#if SIMPLER_TENSORMAP_PROFILING +#include "pto_tensormap.h" +#endif + +namespace l2_bench { + +inline uint64_t now_ns() { + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count() + ); +} + +// Samples for one named step. Kept as raw sample vectors rather than running +// moments so percentiles are exact and the distribution can be re-derived. +struct Step { + std::vector ns; + + void add(uint64_t v) { ns.push_back(v); } + size_t count() const { return ns.size(); } + uint64_t total() const { + uint64_t t = 0; + for (uint64_t v : ns) t += v; + return t; + } + uint64_t pct(double q) const { + if (ns.empty()) return 0; + std::vector s = ns; + std::sort(s.begin(), s.end()); + size_t i = static_cast(q * static_cast(s.size() - 1) + 0.5); + return s[std::min(i, s.size() - 1)]; + } + uint64_t mean() const { return ns.empty() ? 0 : total() / ns.size(); } + uint64_t max() const { return ns.empty() ? 0 : *std::max_element(ns.begin(), ns.end()); } +}; + +class Profiler { +public: + // --- Level 1 ----------------------------------------------------------- + void phase(const char *name, uint64_t ns) { phases_[name].add(ns); } + + // --- Level 2 ----------------------------------------------------------- + void record_op(const char *op, uint64_t ns) { ops_[op].add(ns); } + + // Per-submit shape, so a submit's cost can be regressed against how much + // work it was actually given rather than averaged over a mixed population. + void record_submit_shape(int32_t tensors, int32_t scalars, int16_t blocks, uint64_t ns) { + by_tensor_count_[tensors].add(ns); + subtasks_ += static_cast(blocks); + scalars_total_ += static_cast(scalars); + } + + void set_totals(int32_t tasks, uint64_t clean_total_ns, uint64_t inst_total_ns) { + tasks_ = tasks; + clean_total_ns_ = clean_total_ns; + inst_total_ns_ = inst_total_ns; + } + + void set_clock_cost(uint64_t ns) { clock_cost_ns_ = ns; } + +#if SIMPLER_ORCH_PROFILING + // Level 3 must be snapshotted right after the CLEAN run: the engine's + // counters are process-global accumulators that orchestrator_get_profiling() + // resets on read, so reading them at report time would fold the clean and + // instrumented runs together and double every number. + void set_engine_steps(const PTO2OrchProfilingData &d) { + engine_steps_ = d; + have_engine_steps_ = true; + } +#endif + +#if SIMPLER_TENSORMAP_PROFILING + // Level 4. Same reset-on-read caveat as Level 3: snapshot after the clean run. + void set_tensormap_stats(const PTO2TensorMapProfilingData &d) { + tm_stats_ = d; + have_tm_stats_ = true; + } +#endif + + uint64_t subtasks() const { return subtasks_; } + + void report(std::FILE *f) const; + void report_json(std::FILE *f) const; + +private: + void table(std::FILE *f, const char *title, const std::map &rows, uint64_t denom) const; + + std::map phases_; + std::map ops_; + std::map by_tensor_count_; + uint64_t subtasks_{0}; + uint64_t scalars_total_{0}; + int32_t tasks_{0}; + uint64_t clean_total_ns_{0}; + uint64_t inst_total_ns_{0}; + uint64_t clock_cost_ns_{0}; +#if SIMPLER_ORCH_PROFILING + PTO2OrchProfilingData engine_steps_{}; + bool have_engine_steps_{false}; +#endif +#if SIMPLER_TENSORMAP_PROFILING + PTO2TensorMapProfilingData tm_stats_{}; + bool have_tm_stats_{false}; +#endif +}; + +// --------------------------------------------------------------------------- +// The replacement ops table. +// +// install() copies rt->ops, wraps the six entries the orchestration surface +// actually calls, and repoints rt->ops at the copy. remove() restores the +// original. The wrapped functions forward to the ORIGINAL table, so the engine +// path taken is byte-identical to production plus two clock reads. +// +// Single-threaded by construction: an orchestration entry runs on exactly one +// thread (the orchestrator is single-writer by design — see +// PTO2OrchestratorState's "single-thread access by orchestrator, no atomic +// needed"), so a plain global needs no synchronisation. +// --------------------------------------------------------------------------- +class OpsInterceptor { +public: + OpsInterceptor(PTO2Runtime *rt, Profiler *prof) : rt_(rt), saved_(rt->ops) { + g_original = rt->ops; + g_prof = prof; + + table_ = *saved_; // copy every field, then override the measured ones + table_.submit_task = &wrap_submit_task; + table_.alloc_tensors = &wrap_alloc_tensors; + table_.submit_dummy_task = &wrap_submit_dummy_task; + table_.scope_begin = &wrap_scope_begin; + table_.scope_end = &wrap_scope_end; + table_.orchestration_done = &wrap_orchestration_done; + // is_fatal is deliberately NOT wrapped: the inline API calls it before + // every single op, so wrapping it would add a clock pair per call that + // measures nothing and inflates everything else. + rt->ops = &table_; + } + + ~OpsInterceptor() { + rt_->ops = saved_; + g_original = nullptr; + g_prof = nullptr; + } + + OpsInterceptor(const OpsInterceptor &) = delete; + OpsInterceptor &operator=(const OpsInterceptor &) = delete; + +private: + static const PTO2RuntimeOps *g_original; + static Profiler *g_prof; + + static TaskOutputTensors wrap_submit_task( + PTO2Runtime *rt, const MixedKernels &mixed_kernels, const CoreTaskArgs &args + ) { + const uint64_t t0 = now_ns(); + TaskOutputTensors r = g_original->submit_task(rt, mixed_kernels, args); + const uint64_t dt = now_ns() - t0; + g_prof->record_op("submit_task", dt); + g_prof->record_submit_shape(args.tensor_count(), args.scalar_count(), args.launch_spec.block_num(), dt); + return r; + } + + static TaskOutputTensors wrap_alloc_tensors(PTO2Runtime *rt, const CoreTaskArgs &args) { + const uint64_t t0 = now_ns(); + TaskOutputTensors r = g_original->alloc_tensors(rt, args); + g_prof->record_op("alloc_tensors", now_ns() - t0); + return r; + } + + static TaskOutputTensors wrap_submit_dummy_task(PTO2Runtime *rt, const CoreTaskArgs &args) { + const uint64_t t0 = now_ns(); + TaskOutputTensors r = g_original->submit_dummy_task(rt, args); + g_prof->record_op("submit_dummy_task", now_ns() - t0); + return r; + } + + static void wrap_scope_begin(PTO2Runtime *rt) { + const uint64_t t0 = now_ns(); + g_original->scope_begin(rt); + g_prof->record_op("scope_begin", now_ns() - t0); + } + + static void wrap_scope_end(PTO2Runtime *rt) { + const uint64_t t0 = now_ns(); + g_original->scope_end(rt); + g_prof->record_op("scope_end", now_ns() - t0); + } + + static void wrap_orchestration_done(PTO2Runtime *rt) { + const uint64_t t0 = now_ns(); + g_original->orchestration_done(rt); + g_prof->record_op("orchestration_done", now_ns() - t0); + } + + PTO2Runtime *rt_; + const PTO2RuntimeOps *saved_; + PTO2RuntimeOps table_{}; +}; + +// Cost of one steady_clock read, so the instrument's own weight is a reported +// number rather than a silent addition to every row above. +inline uint64_t measure_clock_cost(int iters = 20000) { + volatile uint64_t sink = 0; + const uint64_t t0 = now_ns(); + for (int i = 0; i < iters; ++i) sink += now_ns(); + const uint64_t t1 = now_ns(); + (void)sink; + return (t1 - t0) / static_cast(iters); +} + +} // namespace l2_bench diff --git a/l2-orchestrator-standalone/bench/payload.h b/l2-orchestrator-standalone/bench/payload.h new file mode 100644 index 0000000000..989c9544e1 --- /dev/null +++ b/l2-orchestrator-standalone/bench/payload.h @@ -0,0 +1,32 @@ +/* + * The payload — the `entry` half of the four-line sequence. + * + * In simpler, `entry_points->entry` is a symbol dlsym'd out of a compiled + * orchestration .so and its type is `void (*)(const ChipTaskArgs &)` + * (runtime_maker.cpp:347). The payload here is exactly that function, wrapping + * qwen3_dynamic_tensormap.h compiled unmodified as C through the esl_shim. + */ + +#pragma once + +#include +#include + +#include "pto_types.h" + +namespace l2_bench { + +// Owns the ChipTensors the entry args point at: ChipTaskArgs stores TensorRefs +// into a ChipStorageTaskArgs, so that storage must outlive every call. +struct PayloadArgs { + ChipStorageTaskArgs storage; + ChipTaskArgs args; + + void finalize() { args.create_from_chip_args(storage); } +}; + +void qwen3_dyn_entry(const ChipTaskArgs &orch_args); +void qwen3_dyn_build_args(PayloadArgs &out); +void qwen3_dyn_report(std::FILE *f); + +} // namespace l2_bench diff --git a/l2-orchestrator-standalone/bench/payload_qwen3_dyn.cpp b/l2-orchestrator-standalone/bench/payload_qwen3_dyn.cpp new file mode 100644 index 0000000000..2728b7bc1e --- /dev/null +++ b/l2-orchestrator-standalone/bench/payload_qwen3_dyn.cpp @@ -0,0 +1,96 @@ +/* + * Adapter side of the qwen3_dynamic_tensormap payload. + * + * The case itself is compiled by payload_qwen3_dyn_case.c (as C); the esl_proxy + * API it calls is implemented by esl_shim/esl_shim_impl.cpp (as C++, on the L2 + * orchestration API). This file only adapts the entry signature and reports the + * case's own accounting back to the driver. + * + * WHAT THE CASE'S orch_args IS HERE + * --------------------------------- + * The case builds its own 20 entry tensors with + * `tensor_from_base_layout(orch_args + i, , ...)`, so orch_args + * is a base ADDRESS and the case owns all shape knowledge. It reads no TaskArgs + * container, which is why build_args below leaves the ChipTaskArgs empty rather + * than fabricating 20 tensors the case never looks at. + * + * Consequence worth stating: the 20 externals sit at consecutive BYTE addresses + * (orch_args+0 .. orch_args+19) and therefore overlap heavily. In esl_proxy that + * is harmless because every one of them is passed with an `_ro` tag and never + * enters the tensormap; the same holds here (NO_DEP), and esl_shim_impl.cpp + * asserts the invariant at every `_ro` call instead of trusting it. + */ + +#include + +#include "esl_shim/esl_c_abi.h" +#include "payload.h" + +// The case's entry, renamed at build time (see payload_qwen3_dyn_case.c). Its +// parameter is the case's own `const uint64_t orch_args`, not a ChipTaskArgs. +extern "C" void qwen3_dyn_orchestration_entry(const uint64_t orch_args); +// Defined in payload_qwen3_dyn_case.c from the macro the case itself saw. +extern "C" const int qwen3_dyn_spmd_tier; + +namespace l2_bench { + +// Host backing the case addresses its externals from. Never read or written — +// the orchestrator only does address arithmetic on external tensors — but the +// address must be real so ChipTensor's bounds assertions have something sane to +// work with. ChipTensor::view validates offsets against the recorded SHAPE, not +// against any allocation size, so no multi-GiB fixture is needed for a case +// whose largest external is k_cache at [2949120, 128]. +alignas(64) static uint8_t g_ext_backing[1 << 16]; + +void qwen3_dyn_entry(const ChipTaskArgs &orch_args) { + (void)orch_args; // see the header note: the case builds its own externals + qwen3_dyn_orchestration_entry(reinterpret_cast(g_ext_backing)); +} + +void qwen3_dyn_build_args(PayloadArgs &out) { + // Deliberately empty. Leaving it visibly empty documents that this case + // takes no entry tensors through the args container. + out.storage.clear(); + out.finalize(); +} + +/* + * The case's own accounting, which the engine's task count alone cannot show. + * + * `tasks` counts kernel submits only, so `tasks + allocs` is what the engine + * reports as its task count — on L2 an alloc_tensors IS a task, unlike in + * esl_proxy where it is a pool-tail bump. + * + * The tracked/no-dep split is the load-bearing number for anyone reading an edge + * count off this run: only tracked args can produce a TensorMap edge, because + * esl_proxy's `_ro` variants create no dependency (see esl_shim_impl.cpp). + */ +void qwen3_dyn_report(std::FILE *f) { + const esl_stats s = esl_get_stats(); + std::fprintf( + f, "\nqwen3_dynamic_tensormap.h — the case's own accounting (QWEN3_SPMD_TIER=%d)\n", qwen3_dyn_spmd_tier + ); + // The tier decides how many SPMD chunks fold into one task, so the task + // count below is meaningless without it. The build pins tier 0 — one chunk + // per task, so tasks == subtasks — and this line reads the value the case + // itself saw, which is what makes the pin verifiable rather than assumed. + std::fprintf( + f, " kernel submits %6llu framework allocs %6llu (engine tasks = %llu)\n", + static_cast(s.tasks), static_cast(s.allocs), + static_cast(s.tasks + s.allocs) + ); + std::fprintf( + f, " SPMD subtasks %6llu scalar args %6llu\n", + static_cast(s.subtasks), static_cast(s.scalars) + ); + std::fprintf( + f, " dep-tracked args %6llu _ro (NO_DEP) args %6llu\n", + static_cast(s.tracked_args), static_cast(s.no_dep_args) + ); + std::fprintf( + f, " sum of the case's DUR_* (virtual AICore time, NOT measured): %.3f us\n", + static_cast(s.duration_ns) / 1000.0 + ); +} + +} // namespace l2_bench diff --git a/l2-orchestrator-standalone/bench/payload_qwen3_dyn_case.c b/l2-orchestrator-standalone/bench/payload_qwen3_dyn_case.c new file mode 100644 index 0000000000..8e888976e6 --- /dev/null +++ b/l2-orchestrator-standalone/bench/payload_qwen3_dyn_case.c @@ -0,0 +1,34 @@ +/* + * Compiles qwen3_dynamic_tensormap.h — UNMODIFIED — as C. + * + * The case is a single-translation-unit orchestration source that happens to be + * named .h; including it is how it is meant to be consumed. It is compiled as C + * because it is C: its shapes are C99 compound literals, which g++ rejects + * outright (see the header of esl_shim/esl_c_abi.h for the full account). + * + * Its `#include "mem_pool.h"` and `#include "tensormap.h"` resolve to + * esl_shim/, which forwards to the C ABI that esl_shim_impl.cpp implements on + * top of the L2 orchestrator. + * + * The entry is renamed at build time via + * -Daicpu_orchestration_entry=qwen3_dyn_orchestration_entry, because the other + * qwen3 payload in this package already exports that symbol with a different + * signature and both have C linkage. The case file itself is untouched — the + * fidelity check asserts it is byte-identical to the repo-root original. + */ + +#include "qwen3_dynamic_tensormap.h" + +/* + * The SPMD tier this binary actually compiled the case at, read from the macro + * as the case itself saw it. Exported so the report cannot disagree with the + * binary: a tier printed from anywhere else could drift from the -D that shaped + * the DAG. The case's own guard is + * + * #ifndef QWEN3_SPMD_TIER + * #define QWEN3_SPMD_TIER 4 + * #endif + * + * so this picks up the build's -D when given and the case's default otherwise. + */ +const int qwen3_dyn_spmd_tier = QWEN3_SPMD_TIER; diff --git a/l2-orchestrator-standalone/qwen3_dynamic_tensormap.h b/l2-orchestrator-standalone/qwen3_dynamic_tensormap.h new file mode 100644 index 0000000000..8b9fb16ded --- /dev/null +++ b/l2-orchestrator-standalone/qwen3_dynamic_tensormap.h @@ -0,0 +1,349 @@ +// Orchestration Function: qwen3_decode (dynamic tensormap, configurable-SPMD +// variant). +// +// Mirrors +// V200-benchmark/qwen3/qwen3_dynamic_tensormap/orchestration/qwen3_decode.cpp. +// Dependencies are discovered automatically via tensormap +// (tm_in/tm_out/tm_submit). SPMD tier is selected at compile time via +// QWEN3_SPMD_TIER (0=non-spmd .. 4=all-spmd). +// +// Durations are V200-benchmark per-subtask means (README.md §1.2.1 AICore View) +// in ns. +#include +#include + + +#include "mem_pool.h" +#include "tensormap.h" + +#ifndef QWEN3_SPMD_TIER +#define QWEN3_SPMD_TIER 4 +#endif +#if QWEN3_SPMD_TIER < 0 || QWEN3_SPMD_TIER > 4 +#error "QWEN3_SPMD_TIER must be 0..4" +#endif + +#define DUR_RMSNORM 23950 +#define DUR_Q_PROJ 26060 +#define DUR_K_PROJ 18170 +#define DUR_V_PROJ 17890 +#define DUR_QK_NORM 13190 +#define DUR_ROPE_KV_CACHE 9480 +#define DUR_QK_MATMUL 29350 +#define DUR_SOFTMAX 19400 +#define DUR_SV_MATMUL 31650 +#define DUR_ONLINE_SOFTMAX 20820 +#define DUR_OUT_PROJ 40750 +#define DUR_POST_RMSNORM 24390 +#define DUR_GATE_PROJ 95700 +#define DUR_UP_PROJ 97140 +#define DUR_SILU 2820 +#define DUR_DOWN_PROJ 72220 +#define DUR_DOWN_PROJ_RES 2590 + +_Thread_local int g_subtask_cnt = 0; + +static inline void set_task_type(uint32_t task_id, task_type_t type) { + g_basic_buf[task_id & RING_MASK].type = type; +} + +static inline void set_block_num(uint32_t task_id, uint32_t count) { + g_basic_buf[task_id & RING_MASK].mode = ORG_MODE_SPMD_SYNC; + g_basic_buf[task_id & RING_MASK].count = count; + g_subtask_cnt += count; +} + +static inline int qwen3_min_i(int a, int b) { + return a < b ? a : b; +} + +static inline int qwen3_blocks_per_task(int total_chunks) { + static const int targets[5] = {1, 2, 4, 8, 1 << 30}; + int target = targets[QWEN3_SPMD_TIER]; + return qwen3_min_i(total_chunks, target); +} + +static inline int qwen3_cur_blocks(int total_chunks, int base) { + return qwen3_min_i(qwen3_blocks_per_task(total_chunks), total_chunks - base); +} + +void aicpu_orchestration_entry(const uint64_t orch_args) { + Tensor ext_hidden_states = tensor_from_base_layout(orch_args + 0, (uint32_t[]){90, 5120}, 2, BFLOAT16); // batch=90, hidden=5120 + Tensor ext_input_rms_weight = tensor_from_base_layout(orch_args + 1, (uint32_t[]){1, 5120}, 2, FLOAT32); // hidden=5120 + Tensor ext_wq = tensor_from_base_layout(orch_args + 2, (uint32_t[]){5120, 5120}, 2, BFLOAT16); // hidden=5120 + Tensor ext_wk = tensor_from_base_layout(orch_args + 3, (uint32_t[]){5120, 1024}, 2, BFLOAT16); // hidden=5120, kv_hidden=1024 + Tensor ext_wv = tensor_from_base_layout(orch_args + 4, (uint32_t[]){5120, 1024}, 2, BFLOAT16); // hidden=5120, kv_hidden=1024 + Tensor ext_q_norm_weight = tensor_from_base_layout(orch_args + 5, (uint32_t[]){1, 128}, 2, FLOAT32); // head_dim=128 + Tensor ext_k_norm_weight = tensor_from_base_layout(orch_args + 6, (uint32_t[]){1, 128}, 2, FLOAT32); // head_dim=128 + Tensor ext_seq_lens = tensor_from_base_layout(orch_args + 7, (uint32_t[]){90}, 1, INT32); // batch=90 + Tensor ext_block_table = tensor_from_base_layout(orch_args + 8, (uint32_t[]){2880}, 1, INT32); // num_blocks=2880 + Tensor ext_slot_mapping = tensor_from_base_layout(orch_args + 9, (uint32_t[]){90}, 1, INT32); // batch=90 + Tensor ext_rope_cos = tensor_from_base_layout(orch_args + 10, (uint32_t[]){4096, 128}, 2, FLOAT32); // max_seq=4096, head_dim=128 + Tensor ext_rope_sin = tensor_from_base_layout(orch_args + 11, (uint32_t[]){4096, 128}, 2, FLOAT32); // max_seq=4096, head_dim=128 + Tensor ext_k_cache = tensor_from_base_layout(orch_args + 12, (uint32_t[]){2949120, 128}, 2, BFLOAT16); // cache_rows=2880*8*128, head_dim=128 + Tensor ext_v_cache = tensor_from_base_layout(orch_args + 13, (uint32_t[]){2949120, 128}, 2, BFLOAT16); // cache_rows=2880*8*128, head_dim=128 + Tensor ext_wo = tensor_from_base_layout(orch_args + 14, (uint32_t[]){5120, 5120}, 2, BFLOAT16); // hidden=5120 + Tensor ext_post_rms_weight = tensor_from_base_layout(orch_args + 15, (uint32_t[]){1, 5120}, 2, FLOAT32); // hidden=5120 + Tensor ext_w_gate = tensor_from_base_layout(orch_args + 16, (uint32_t[]){5120, 17408}, 2, BFLOAT16); // hidden=5120, intermediate=17408 + Tensor ext_w_up = tensor_from_base_layout(orch_args + 17, (uint32_t[]){5120, 17408}, 2, BFLOAT16); // hidden=5120, intermediate=17408 + Tensor ext_w_down = tensor_from_base_layout(orch_args + 18, (uint32_t[]){17408, 5120}, 2, BFLOAT16); // intermediate=17408, hidden=5120 + Tensor ext_out = tensor_from_base_layout(orch_args + 19, (uint32_t[]){90, 5120}, 2, BFLOAT16); // batch=90, hidden=5120 + (void)ext_seq_lens; + (void)ext_slot_mapping; + tm_deps_init(); + const int64_t user_batch = 90; // batch=90 + const int64_t batch_padded = 96; // ((batch+15)/16)*16 + Tensor q_proj = alloc_tensors((uint32_t[2]){batch_padded, 5120}, 2, FLOAT32); + Tensor k_proj = alloc_tensors((uint32_t[2]){batch_padded, 1024}, 2, FLOAT32); + Tensor v_proj = alloc_tensors((uint32_t[2]){batch_padded, 1024}, 2, FLOAT32); + Tensor q_proj_norm = alloc_tensors((uint32_t[2]){batch_padded, 5120}, 2, FLOAT32); + Tensor k_proj_norm = alloc_tensors((uint32_t[2]){batch_padded, 1024}, 2, FLOAT32); + for (int64_t b0 = 0; b0 < batch_padded; b0 += 16) { + Tensor normed_tile = alloc_tensors((uint32_t[2]){16, 5120}, 2, BFLOAT16); + const int64_t cur_valid = (user_batch - b0 > 16) ? 16 : (user_batch - b0); + new_task(g_task_id, TASK_TYPE_VECTOR, 1, DUR_RMSNORM); + tm_in_ro(g_task_id, ext_hidden_states); + tm_out(g_task_id, normed_tile); + tm_in_ro(g_task_id, ext_input_rms_weight); + add_scalar(g_task_id, b0); + add_scalar(g_task_id, cur_valid); + tm_submit(g_task_id); + g_task_id++; + + for (int base = 0; base < 20; base += qwen3_blocks_per_task(20)) { + // 20: q_proj SPMD total chunks; cols/chunk = 5120/20 = 256 + int cur_blocks = qwen3_cur_blocks(20, base); + new_task(g_task_id, TASK_TYPE_CUBE, (uint32_t)cur_blocks, DUR_Q_PROJ); + Tensor q_piece = view(q_proj, (uint32_t)b0, base * 256u, 16u, cur_blocks * 256u); + tm_in(g_task_id, normed_tile); + tm_in_ro(g_task_id, ext_wq); + tm_out(g_task_id, q_piece); + add_scalar(g_task_id, b0); + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + } + + for (int base = 0; base < 8; base += qwen3_blocks_per_task(8)) { + int cur_blocks = qwen3_cur_blocks(8, base); + new_task(g_task_id, TASK_TYPE_CUBE, (uint32_t)cur_blocks, DUR_K_PROJ); + tm_in(g_task_id, normed_tile); + tm_in_ro(g_task_id, ext_wk); + Tensor k_piece = view(k_proj, (uint32_t)b0, base * 128u, 16u, cur_blocks * 128u); + tm_out(g_task_id, k_piece); + add_scalar(g_task_id, b0); + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + + new_task(g_task_id, TASK_TYPE_CUBE, (uint32_t)cur_blocks, DUR_V_PROJ); + tm_in(g_task_id, normed_tile); + tm_in_ro(g_task_id, ext_wv); + Tensor v_piece = view(v_proj, (uint32_t)b0, base * 128u, 16u, cur_blocks * 128u); + tm_out(g_task_id, v_piece); + add_scalar(g_task_id, b0); + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + } + + new_task(g_task_id, TASK_TYPE_VECTOR, 1, DUR_QK_NORM); + Tensor k0_norm = view(k_proj_norm, (uint32_t)b0, 0u, 16u, 1024u); + Tensor q0_norm = view(q_proj_norm, (uint32_t)b0, 0u, 16u, 5120u); + Tensor q0_in = view(q_proj, (uint32_t)b0, 0u, 16u, 5120u); + Tensor k0_in = view(k_proj, (uint32_t)b0, 0u, 16u, 1024u); + tm_out(g_task_id, k0_norm); + tm_out(g_task_id, q0_norm); + tm_in(g_task_id, q0_in); + tm_in_ro(g_task_id, ext_q_norm_weight); + tm_in_ro(g_task_id, ext_k_norm_weight); + tm_in(g_task_id, k0_in); + tm_submit(g_task_id); + g_task_id++; + } + + Tensor attn_out[6]; + for (int i = 0; i < 6; i++) { + attn_out[i] = alloc_tensors((uint32_t[2]){16, 5120}, 2, BFLOAT16); + } + + for (int64_t b = 0; b < user_batch; b += 1) { + Tensor all_raw_scores = alloc_tensors((uint32_t[2]){4096, 128}, 2, FLOAT32); + Tensor all_exp_padded = alloc_tensors((uint32_t[2]){4096, 128}, 2, BFLOAT16); + Tensor all_cur_mi = alloc_tensors((uint32_t[2]){4096, 1}, 2, FLOAT32); + Tensor all_cur_li = alloc_tensors((uint32_t[2]){4096, 1}, 2, FLOAT32); + Tensor all_oi_tmp = alloc_tensors((uint32_t[2]){4096, 128}, 2, FLOAT32); + Tensor q_padded_local = alloc_tensors((uint32_t[2]){128, 128}, 2, BFLOAT16); + Tensor k_cache_local = view(ext_k_cache, (uint32_t)b * 8u, 0u, 8u, 128u); // batch b: 8*head_dim=1024 kv_hidden + Tensor v_cache_local = view(ext_v_cache, (uint32_t)b * 8u, 0u, 8u, 128u); // batch b: 8*head_dim=1024 kv_hidden + Tensor k_cache_update = alloc_tensors((uint32_t[2]){8, 128}, 2, BFLOAT16); // ROPE KV write-back + Tensor v_cache_update = alloc_tensors((uint32_t[2]){8, 128}, 2, BFLOAT16); // ROPE KV write-back + const int64_t b_tile0 = (b / 16) * 16; + const int64_t slot = b; + const int64_t slot_block = slot / 128; + const int64_t slot_offset = slot - slot_block * 128; + new_task(g_task_id, TASK_TYPE_VECTOR, 1, DUR_ROPE_KV_CACHE); + Tensor k0_norm = view(k_proj_norm, (uint32_t)b_tile0, 0u, 16u, 1024u); + Tensor v0 = view(v_proj, (uint32_t)b_tile0, 0u, 16u, 1024u); + Tensor q0_norm = view(q_proj_norm, (uint32_t)b_tile0, 0u, 16u, 5120u); + tm_out(g_task_id, q_padded_local); + tm_in_ro(g_task_id, k_cache_local); + tm_in_ro(g_task_id, v_cache_local); + tm_out(g_task_id, k_cache_update); + tm_out(g_task_id, v_cache_update); + tm_in(g_task_id, k0_norm); + tm_in_ro(g_task_id, ext_rope_cos); + tm_in_ro(g_task_id, ext_rope_sin); + tm_in_ro(g_task_id, ext_rope_cos); + tm_in_ro(g_task_id, ext_rope_sin); + tm_in(g_task_id, v0); + tm_in(g_task_id, q0_norm); + add_scalar(g_task_id, slot_block); + add_scalar(g_task_id, slot_offset); + add_scalar(g_task_id, b); + tm_submit(g_task_id); + g_task_id++; + + for (int base = 0; base < 4; base += qwen3_blocks_per_task(4)) { + int cur_blocks = qwen3_cur_blocks(4, base); + Tensor row_piece = view(all_raw_scores, base * 1024u, 0u, (uint32_t)(cur_blocks * 1024), 128u); + new_task(g_task_id, TASK_TYPE_CUBE, (uint32_t)cur_blocks, DUR_QK_MATMUL); + tm_in(g_task_id, q_padded_local); + tm_out(g_task_id, row_piece); + tm_in_ro(g_task_id, ext_block_table); + tm_in(g_task_id, k_cache_update); + add_scalar(g_task_id, b); + add_scalar(g_task_id, 8); // (1024+127)/128: KV context blocks + add_scalar(g_task_id, b * 32); // block_table row offset for batch b + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + + new_task(g_task_id, TASK_TYPE_VECTOR, (uint32_t)cur_blocks, DUR_SOFTMAX); + Tensor cur_li_piece = view(all_cur_li, base * 1024u, 0u, (uint32_t)(cur_blocks * 1024), 1u); + Tensor cur_mi_piece = view(all_cur_mi, base * 1024u, 0u, (uint32_t)(cur_blocks * 1024), 1u); + Tensor exp_padded_piece = view(all_exp_padded, base * 1024u, 0u, (uint32_t)(cur_blocks * 1024), 128u); + tm_out(g_task_id, cur_li_piece); + tm_out(g_task_id, cur_mi_piece); + tm_out(g_task_id, exp_padded_piece); + tm_in(g_task_id, row_piece); + add_scalar(g_task_id, 8); // (1024+127)/128: KV context blocks + add_scalar(g_task_id, 1024); // context length (tokens) + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + + Tensor exp_piece = view(all_exp_padded, base * 1024u, 0u, (uint32_t)(cur_blocks * 1024), 128u); + new_task(g_task_id, TASK_TYPE_CUBE, (uint32_t)cur_blocks, DUR_SV_MATMUL); + Tensor oi_tmp_piece = view(all_oi_tmp, base * 1024u, 0u, (uint32_t)(cur_blocks * 1024), 128u); + tm_out(g_task_id, oi_tmp_piece); + tm_in_ro(g_task_id, ext_block_table); + tm_in(g_task_id, exp_piece); + tm_in(g_task_id, v_cache_update); + add_scalar(g_task_id, 8); // (1024+127)/128: KV context blocks + add_scalar(g_task_id, b * 32); // block_table row offset for batch b + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + + new_task(g_task_id, TASK_TYPE_VECTOR, (uint32_t)cur_blocks, DUR_ONLINE_SOFTMAX); + tm_in(g_task_id, oi_tmp_piece); + tm_in(g_task_id, cur_mi_piece); + tm_in(g_task_id, cur_li_piece); + Tensor attn_out_piece = view(attn_out[b / 16], (uint32_t)(b % 16), + base * 1280u, 1u, cur_blocks * 1280u); + tm_inout(g_task_id, attn_out_piece); + add_scalar(g_task_id, 8); // (1024+127)/128: KV context blocks + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + } + } + + for (int64_t b0 = 0; b0 < batch_padded; b0 += 16) { + Tensor resid1_tile = alloc_tensors((uint32_t[2]){16, 5120}, 2, FLOAT32); + Tensor gm_pipe_buffer_0 = alloc_tensors((uint32_t[2]){16384, 40}, 2, FLOAT32); + Tensor post_norm_tile = alloc_tensors((uint32_t[2]){16, 5120}, 2, BFLOAT16); + Tensor mlp_tile = alloc_tensors((uint32_t[2]){16, 17408}, 2, BFLOAT16); + Tensor gate_tile = alloc_tensors((uint32_t[2]){16, 17408}, 2, FLOAT32); + Tensor up_tile = alloc_tensors((uint32_t[2]){16, 17408}, 2, FLOAT32); + Tensor down_tile = alloc_tensors((uint32_t[2]){16, 5120}, 2, FLOAT32); + const int64_t cur_valid = (user_batch - b0 > 16) ? 16 : (user_batch - b0); + for (int base = 0; base < 40; base += qwen3_blocks_per_task(40)) { + // 40: out_proj SPMD total chunks; cols/chunk = 5120/40 = 128 + int cur_blocks = qwen3_cur_blocks(40, base); + new_task(g_task_id, TASK_TYPE_MIX, (uint32_t)cur_blocks, DUR_OUT_PROJ); + Tensor attn_out_tile = view(attn_out[b0 / 16], 0u, 0u, (uint32_t)cur_valid, 5120u); + Tensor resid1_piece = view(resid1_tile, 0u, base * 128u, 16u, (uint32_t)(cur_blocks * 128)); + tm_in_ro(g_task_id, ext_hidden_states); + tm_in(g_task_id, attn_out_tile); + tm_in_ro(g_task_id, ext_wo); + tm_inout(g_task_id, resid1_piece); + tm_out(g_task_id, gm_pipe_buffer_0); + add_scalar(g_task_id, b0); + add_scalar(g_task_id, cur_valid); + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + } + + new_task(g_task_id, TASK_TYPE_VECTOR, 1, DUR_POST_RMSNORM); + tm_in(g_task_id, resid1_tile); + tm_out(g_task_id, post_norm_tile); + tm_in_ro(g_task_id, ext_post_rms_weight); + tm_submit(g_task_id); + g_task_id++; + + for (int base = 0; base < 34; base += qwen3_blocks_per_task(34)) { + int cur_blocks = qwen3_cur_blocks(34, base); + Tensor gate_piece = view(gate_tile, 0u, base * 512u, 16u, (uint32_t)(cur_blocks * 512)); + Tensor up_piece = view(up_tile, 0u, base * 512u, 16u, (uint32_t)(cur_blocks * 512)); + new_task(g_task_id, TASK_TYPE_CUBE, (uint32_t)cur_blocks, DUR_GATE_PROJ); + tm_in(g_task_id, post_norm_tile); + tm_in_ro(g_task_id, ext_w_gate); + tm_inout(g_task_id, gate_piece); + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + + new_task(g_task_id, TASK_TYPE_CUBE, (uint32_t)cur_blocks, DUR_UP_PROJ); + tm_in(g_task_id, post_norm_tile); + tm_in_ro(g_task_id, ext_w_up); + tm_inout(g_task_id, up_piece); + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + + new_task(g_task_id, TASK_TYPE_VECTOR, (uint32_t)cur_blocks, DUR_SILU); + tm_in(g_task_id, gate_piece); + tm_in(g_task_id, up_piece); + Tensor mlp_piece = view(mlp_tile, 0u, base * 512u, 16u, (uint32_t)(cur_blocks * 512)); + tm_inout(g_task_id, mlp_piece); + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + } + for (int base = 0; base < 40; base += qwen3_blocks_per_task(40)) { + int cur_blocks = qwen3_cur_blocks(40, base); + Tensor down_piece = view(down_tile, 0u, base * 128u, 16u, (uint32_t)(cur_blocks * 128)); + Tensor resid1_piece = view(resid1_tile, 0u, base * 128u, 16u, (uint32_t)(cur_blocks * 128)); + new_task(g_task_id, TASK_TYPE_CUBE, (uint32_t)cur_blocks, DUR_DOWN_PROJ); + tm_in(g_task_id, mlp_tile); + tm_in_ro(g_task_id, ext_w_down); + tm_inout(g_task_id, down_piece); + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + + new_task(g_task_id, TASK_TYPE_VECTOR, (uint32_t)cur_blocks, DUR_DOWN_PROJ_RES); + tm_in(g_task_id, down_piece); + tm_in(g_task_id, resid1_piece); + tm_out_ro(g_task_id, ext_out); + add_scalar(g_task_id, cur_valid); + add_scalar(g_task_id, b0); + add_scalar(g_task_id, base); + tm_submit(g_task_id); + g_task_id++; + } + } +} diff --git a/l2-orchestrator-standalone/reports/perf-report-perf-hbg-orch-qwen3-dyn.md b/l2-orchestrator-standalone/reports/perf-report-perf-hbg-orch-qwen3-dyn.md new file mode 100644 index 0000000000..deab307d16 --- /dev/null +++ b/l2-orchestrator-standalone/reports/perf-report-perf-hbg-orch-qwen3-dyn.md @@ -0,0 +1,267 @@ +# L2 orchestration performance — branch `perf/hbg-orch`, payload `qwen3-dyn` + +Measured 2026-08-19 with the pristine l2-orchestrator-standalone harness +(commit f06646d of the extraction, before any of its later optimization work), +against this branch's engine sources compiled **in place from `../src`** at +`perf/hbg-orch` @ 72c3163d. The four lines under test +are `../src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp:538-541` +(`host/` is the one directory this package does not compile, because that is +where runtime_maker's CANN dependencies live): + +```c +rt_scope_begin(rt); +entry_points->entry(orch_l2); +rt_scope_end(rt); +rt_orchestration_done(rt); +``` + +driven by `qwen3_dynamic_tensormap.h` (QWEN3_SPMD_TIER=0), compiled unmodified +as C via the esl_shim. + +Engine `file:line` citations below are into `../src` at that revision; bench +paths are relative to this package. Line numbers move when the runtime is +edited — re-run the grep next to each claim rather than trusting the number +after a change. + +## Environment + +| | | +|---|---| +| CPU | Kunpeng-920, 192 cores, aarch64 | +| Kernel | 5.10.0-60.18.0.50.r865_35.hce2 | +| Compiler | g++ 11.4.0, `-O3` (CMake Release) | +| Pinning | `taskset -c 0-3` (harness is single-threaded) | +| Bench args | `--task-window=8192 --heap-mb=2048 --repeat=20`, median quoted | + +Graph size cross-check (must match the L3 package's independent translation, +and does): 3096 kernel submits + 779 framework allocs = 3875 engine tasks, +3096 SPMD subtasks, 9162 dep-tracked args, 7326 scalars. ctest 4/4, including +the prefault graph-size assertion. + +## Where the code is + +Every number in this report comes from one of these places. + +### The measured path + +| what | file | symbol / line | +|---|---|---| +| the four lines | `../src/…/host_build_graph/host/runtime_maker.cpp` | 538-541 | +| the API the entry calls | `../src/…/host_build_graph/orchestration/pto_orchestration_api.h` | `rt_submit_task` 177-180, `rt_alloc_tensors` 127-130 | +| **the submit body — where STEPs 1-6 live** | `../src/…/runtime/orchestrator_core/pto_orchestrator.cpp` | `submit_task_common` 1112 | +| the alloc body | same | `PTO2OrchestratorState::alloc_tensors` 1896 | +| slot + heap reservation (STEP 1) | same | `prepare_task` 808 | +| TensorMap lookup / insert (STEP 3/4) | `../src/…/runtime/pto_tensormap.h` | `lookup` 521, `link_entry` 647 | +| the hash (STEP 3's bucket choice) | same | `hash` 639 | +| descriptor + payload GM write (STEP 5) | `pto_orchestrator.cpp` | 1250-1284 | + +### Where the counters come from + +Nothing in the engine was modified to produce this report; each level reads an +existing seam. + +| level | seam | file | +|---|---|---| +| 1 — the four lines | driver brackets each call with `steady_clock` | `bench/l2_bench.cpp:162-176` | +| 2 — every entry→engine call | copies `rt->ops` and repoints it; the orchestration API reaches the engine *only* through that table | `bench/l2_profile.h:177` (`OpsInterceptor`) | +| 3 — per-STEP cycles | the engine's own `CYCLE_COUNT_LAP` accumulators, compiled in by `SIMPLER_ORCH_PROFILING=1` | macros at `pto_orchestrator.cpp:160-170`; laps at 1150 / 1165 / 1235 / 1248 / 1284 / 1298 | +| 4 — TensorMap chain stats | the engine's own `g_lookup_*` counters, compiled in by `SIMPLER_TENSORMAP_PROFILING=1` | `pto_tensormap.h:87-91`, incremented inside `lookup` | + +The six Level-3 lap counters map to the STEP names one-for-one, in source +order: `g_orch_alloc_cycle` (STEP 1, line 1150), `g_orch_sync_cycle` (STEP 2, +1165), `g_orch_lookup_cycle` (STEP 3, 1235), `g_orch_insert_cycle` (STEP 4, +1248), `g_orch_args_cycle` (STEP 5, 1284), `g_orch_fanin_cycle` (STEP 6, 1298). +Levels 3 and 4 both **reset on read**, so `bench/l2_bench.cpp:242-245` snapshots +them immediately after the clean run — before the instrumented run can +contaminate them. + +### What the harness substitutes + +Three seams, all in `bench/l2_harness.h`; none of them is on the measured path +in a way that changes the engine's work: + +| simpler | here | line | +|---|---|---| +| device GM heap | host `aligned_alloc` at the device's 1024 B granularity | 149 | +| dlopen + dlsym'd entry | direct call through the same `framework_bind_runtime` | 192 | +| SM mirror / arena first touch | optional `memset` before init, for the cold/warm split | 144, 154 | + +The payload's own side: `bench/payload_qwen3_dyn.cpp` adapts the entry +signature, `bench/payload_qwen3_dyn_case.c` compiles the case, and +`bench/esl_shim/esl_shim_impl.cpp:22-24` holds the esl_proxy→L2 tag mapping +(`tm_in`→`INPUT`, `tm_out`→`OUTPUT_EXISTING`, `tm_inout`→`INOUT`, +`tm_*_ro`→`NO_DEP`) that decides which args create edges at all. + +## Headline throughput + +| condition | four-line block (median) | per task | throughput | +|---|---|---|---| +| cold (fresh runtime, pages untouched) | 11.41 ms | 2.94 µs | ~340 k tasks/s | +| `--prefault-all` (first-touch excluded) | 4.71–4.90 ms | 1.22–1.26 µs | ~790–820 k tasks/s | + +Roughly 58% of the cold block is first-touch paging of the SM mirror and the +TensorMap arena, not engine work — see the cold/warm attribution below. + +Measured before the package was cut down to this one payload, the `qwen3` +example ran 1.94 µs/task and the synthetic DAG 2.07 µs/task on the same build. +This case is the slowest per task of the three, and Level 4 explains why. + +## Level 1 — the four lines themselves + +| step | ns | % of block | +|---|---:|---:| +| rt_scope_begin | 810 | 0.01% | +| entry | 11 788 550 | 99.96% | +| rt_scope_end | 160 | 0.00% | +| rt_orchestration_done | 3 620 | 0.03% | + +Everything is `entry()`; the scope bracket and the done-flag write are noise. + +## Level 2 — every entry→engine call (intercepted ops table, cold run) + +| call | count | total ms | p50 ns | p99 ns | % of entry | +|---|---:|---:|---:|---:|---:| +| alloc_tensors | 779 | 1.41 | 1 560 | 4 480 | 13.8% | +| submit_task | 3096 | 8.77 | 2 550 | 8 150 | 86.2% | + +Submit latency by tensor-arg count (p50), from the same run: + +| tensor args | submits | p50 ns | p99 ns | ns per tensor | +|---:|---:|---:|---:|---:| +| 3 | 1320 | 2 780 | 8 310 | 927 | +| 4 | 1440 | 1 730 | 7 820 | 433 | +| 5 | 240 | 4 290 | 11 810 | 858 | +| 6 | 6 | 1 970 | 3 800 | 328 | +| 12 | 90 | 4 850 | 7 290 | 404 | + +**Arg count does not predict submit cost here** — 4 args is faster than 3, and +6 args is faster than 5. Whatever dominates is a property of *which* buffers +those args view, not how many there are, which is what Level 4 pins down. (The +per-arg slope is real and was isolated on the synthetic payload before it was +removed; in this workload it is confounded with chain length and cannot be read +off this table.) + +## Level 3 — inside `submit_task_common` (engine's own cycle counters) + +Cold vs `--prefault-all`, per-STEP totals over the whole graph: + +| step | cold ms | warm ms | warm % | first-touch share of cold | +|---|---:|---:|---:|---:| +| 1 prepare_task (slot + heap alloc) | 0.506 | 0.314 | 10.6% | 38% | +| 2 sync_tensormap | 0.030 | 0.012 | 0.4% | 60% | +| 3 infer deps: TensorMap lookup | 2.554 | 2.213 | **74.7%** | 13% | +| 4 register outputs: TensorMap insert | 0.299 | 0.120 | 4.0% | 60% | +| 5 payload/descriptor GM write | 6.523 | 0.294 | 9.9% | **95%** | +| 6 publish fanin_count | 0.014 | 0.008 | 0.3% | 43% | +| TOTAL | 9.926 | 2.961 | | | + +Read cold and warm as two different questions. Cold, STEP 5 looks like the +bottleneck (66%), but 95% of it is the kernel faulting in SM pages — a cost +that in production is paid once per runtime, not per orchestration. Once +prefaulted, the real engine bottleneck is **STEP 3, the TensorMap dependency +lookup, at 75% of submit cost**. No alloc/fanin waits were ever entered +(nothing is reclaimed in this harness). + +Why STEP 5's cold number is that large is arithmetic, not mystery. +`sizeof(PTO2TaskPayload)` is **4864 B** (`pto_runtime2_types.h:265`), of which +`tensors[32] × sizeof(ChipTensor) = 32 × 128 = 4096 B`. At `--task-window=8192` +the payload ring alone is **39.8 MB** of fresh mapping, and STEP 5 is the first +thing to touch each slot (`pto_orchestrator.cpp:1250`, a deliberate batched +write — the comment there says it is deferred out of the allocation phase to +avoid scattered GM writes). So STEP 5 is charged for the page faults of every +slot it is merely the first writer of. At the production default +`PTO2_TASK_WINDOW_SIZE = 16384` (`pto_runtime2_types.h:74`) that doubles. + +## Level 4 — why STEP 3 dominates + +| counter | value | +|---|---:| +| lookups | 6 612 | +| inserts | 4 002 | +| bucket entries walked | 88 590 | +| avg chain length | 13.4 | +| max chain length | **120** | +| overlap checks | 86 970 | +| overlap hits | 28 260 (32.5%) | +| wasted walk (no overlap found) | **67.5%** | + +**The long chains are the design working as specified, not a hash that got +unlucky.** `pto_tensormap.h:30-34` states the constraint outright: + +> CRITICAL: Hash only by base_ptr. For overlap detection to work, ALL +> sub-regions of the same base tensor MUST be in the SAME hash bucket. + +`hash()` (`pto_tensormap.h:639`) is a golden-ratio multiplicative hash over +`tensor.buffer.addr`, and a sub-view keeps its parent's `buffer.addr` while +differing in `start_offset` — so every sub-view of one buffer lands in one +bucket *by construction*, which is exactly what lets `lookup` +(`pto_tensormap.h:521`) find byte-range overlaps at all. The walk then does +`if (tensor.buffer.addr == cur_entry->buffer_addr)` before calling +`check_overlap`, so the chain cost is the price of that guarantee. + +Two counters pin down which of the two possible causes it is: + +- **It is not table pressure.** `PTO2_TENSORMAP_NUM_BUCKETS` is **4096** + (`pto_runtime2_types.h:87`) and this case makes only **4002 inserts** — about + one entry per bucket if addresses were distinct. Observed average chain: + **13.4**. More buckets would change nothing. +- **It is same-address sub-views.** 86 970 of the 88 590 entries walked + (**98.2%**) passed the `buffer_addr` equality test, i.e. nearly every entry on + a walked chain is another view of the very buffer being probed. Only 1.8% of + the walk is unrelated buffers colliding. + +So the 67.5% wasted walk is `check_overlap` returning `NO_OVERLAP` for +sub-views of the same buffer that do not intersect this probe's byte range. +Attacking it means giving the bucket a secondary order on `start_offset` — an +interval structure, or simply keeping the chain sorted so the scan can stop +early — not resizing the table. This cost is specific to workloads shaped like +this one: a payload that allocates whole buffers keeps its max chain at 1 and +pays in the ring/heap allocator (STEP 1) instead. + +## Branch vs simpler-main baseline + +The original f06646d package, whose engine TUs were byte-copies of +simpler-main, was rebuilt and run interleaved with this one on the same +machine, same flags: + +| engine | prefaulted median (5 interleaved rounds) | +|---|---| +| simpler-main copies (f06646d) | 4.64 – 4.81 ms | +| `perf/hbg-orch` (this repo) | 4.64 – 4.82 ms (one 5.64 ms outlier) | + +**No measurable difference.** The branch's engine diffs (the +`CoreTaskArgs` → `GraphTaskArgs` graph-recording rework in +`pto_orchestrator.cpp` / `pto_orchestration_api.h`) are on the graph +record/replay path, which this payload never enters, and the per-STEP and +Level-4 counters are identical to the baseline within noise +(STEP 3 warm: 2.197 ms baseline vs 2.187 ms branch; chain stats identical). + +## Comparing an optimization against this baseline + +To measure an edit to the runtime under `../src`: + +```bash +scripts/check_extraction.sh # assert the harness still matches the runtime +scripts/profile_l2.sh --repeat 20 # rebuilds all three variants, rewrites reports/ +``` + +Since the engine is compiled in place, an edit to `../src` is measured by the +next build — but it is also a change to the repo's runtime, so run the repo's +own tests before believing a number, not just this package's four. + +Compare against this file, and quote the **prefaulted, pinned** number — the +cold one is dominated by first-touch paging and moves with the machine's page +state rather than with your change. An engine edit that changes the graph is a +correctness failure first: `ctest` asserts 3096 submits / 779 allocs, and a +latched `orch_error_code` turns further submits into silent no-ops, which would +otherwise read as a small, very fast graph. + +## Caveats (unchanged from the harness's own README) + +- Nothing is ever reclaimed: no scheduler, no completions, so ring watermark + reclaim, tensormap eviction and back-pressure never fire. Numbers are the + pure submit path. +- Quote throughput from the clean build only; the Level 3/4 build adds ~4–5% + instrument cost and is used only for the shape of the breakdown. +- Full machine-readable data: `profile-qwen3-dyn.{txt,json}` in this directory, + regenerated by `scripts/profile_l2.sh --repeat 20`. diff --git a/l2-orchestrator-standalone/reports/profile-qwen3-dyn.json b/l2-orchestrator-standalone/reports/profile-qwen3-dyn.json new file mode 100644 index 0000000000..5646890fe8 --- /dev/null +++ b/l2-orchestrator-standalone/reports/profile-qwen3-dyn.json @@ -0,0 +1,27 @@ +{ + "payload": "qwen3-dyn", + "engine": "a2a3/host_build_graph L2 PTO2OrchestratorState", + "engine_tasks": 3875, + "spmd_subtasks": 3096, + "clean_total_ns": 11937341, + "instrumented_total_ns": 12304183, + "clock_read_ns": 30, + "orch_profiling_built": true, + "phases": { + "entry": {"count": 1, "total_ns": 11931741, "mean_ns": 11931741, "p50_ns": 11931741, "p99_ns": 11931741, "max_ns": 11931741}, + "rt_orchestration_done": {"count": 1, "total_ns": 3900, "mean_ns": 3900, "p50_ns": 3900, "p99_ns": 3900, "max_ns": 3900}, + "rt_scope_begin": {"count": 1, "total_ns": 950, "mean_ns": 950, "p50_ns": 950, "p99_ns": 950, "max_ns": 950}, + "rt_scope_end": {"count": 1, "total_ns": 100, "mean_ns": 100, "p50_ns": 100, "p99_ns": 100, "max_ns": 100} + }, + "ops": { + "alloc_tensors": {"count": 779, "total_ns": 1480396, "mean_ns": 1900, "p50_ns": 1580, "p99_ns": 6510, "max_ns": 18260}, + "submit_task": {"count": 3096, "total_ns": 8759927, "mean_ns": 2829, "p50_ns": 2500, "p99_ns": 8620, "max_ns": 17280} + }, + "submit_by_tensor_count": { + "3": {"count": 1320, "p50_ns": 2800, "p99_ns": 8100}, + "4": {"count": 1440, "p50_ns": 1730, "p99_ns": 7610}, + "5": {"count": 240, "p50_ns": 4310, "p99_ns": 11340}, + "6": {"count": 6, "p50_ns": 1950, "p99_ns": 3450}, + "12": {"count": 90, "p50_ns": 4970, "p99_ns": 10581} + } +} diff --git a/l2-orchestrator-standalone/reports/profile-qwen3-dyn.txt b/l2-orchestrator-standalone/reports/profile-qwen3-dyn.txt new file mode 100644 index 0000000000..de86e1734b --- /dev/null +++ b/l2-orchestrator-standalone/reports/profile-qwen3-dyn.txt @@ -0,0 +1,60 @@ +============================================================================================================== +L2 Orchestrator profile — qwen3_dynamic_tensormap.h (QWEN3_SPMD_TIER=0) + 3096 kernel submits (3096 SPMD subtasks) + 779 framework allocs = 3875 engine tasks +engine: a2a3 / host_build_graph, PTO2OrchestratorState (L2) +============================================================================================================== + +LEVEL 1 — the four lines (driver-bracketed, uninstrumented run) + step ns % of block + ---------------------------------------------------------- + rt_scope_begin 950 0.01% + entry 11931741 99.96% + rt_scope_end 100 0.00% + rt_orchestration_done 3900 0.03% + TOTAL 11936691 + + per task: 3.08 us throughput: 324629 tasks/s + +INSTRUMENT COST — quote throughput from the clean run, shape from the tables below + uninstrumented total 11.937 ms + instrumented total 12.304 ms (+3.1%) + one steady_clock read 30 ns + +READ p50/p99, NOT mean. A single OS scheduling stall moves a mean by multiples +while leaving p50 and p99 untouched. The mean column reconciles total/count; it +is not the robust statistic. + +LEVEL 2 — every entry->engine call (intercepted ops table) + step count total_ms mean_ns p50_ns p99_ns max_ns % + ------------------------------------------------------------------------------------------------------------ + alloc_tensors 779 1.480 1900 1580 6510 18260 14.46% + submit_task 3096 8.760 2829 2500 8620 17280 85.54% + +SUBMIT LATENCY BY TENSOR-ARG COUNT — the per-tensor slope of STEP 3/4 + tensors count p50_ns p99_ns p50/tensor + -------------------------------------------------------- + 3 1320 2800 8100 933.3 + 4 1440 1730 7610 432.5 + 5 240 4310 11340 862.0 + 6 6 1950 3450 325.0 + 12 90 4970 10581 414.2 + +LEVEL 3 — the engine's own step counters inside submit_task_common + (SIMPLER_ORCH_PROFILING=1, uninstrumented run; submit_count=3875, clock=100000000 Hz) + step cycles total_ms % + ---------------------------------------------------------------------------------- + STEP 1 prepare_task (slot + heap alloc) 46006 0.460 4.63% + STEP 2 sync_tensormap 2013 0.020 0.20% + STEP 3 infer deps: TensorMap lookup 250500 2.505 25.19% + STEP 4 register outputs: TensorMap insert 30664 0.307 3.08% + STEP 5 payload/descriptor GM write 663528 6.635 66.73% + STEP 6 publish fanin_count 1662 0.017 0.17% + end_scope (outside submit) 1 0.000 0.00% + waits: alloc 0 cycles, fanin 0 cycles + + +qwen3_dynamic_tensormap.h — the case's own accounting (QWEN3_SPMD_TIER=0) + kernel submits 3096 framework allocs 779 (engine tasks = 3875) + SPMD subtasks 3096 scalar args 7326 + dep-tracked args 9162 _ro (NO_DEP) args 2874 + sum of the case's DUR_* (virtual AICore time, NOT measured): 110168.700 us diff --git a/l2-orchestrator-standalone/scripts/check_extraction.sh b/l2-orchestrator-standalone/scripts/check_extraction.sh new file mode 100755 index 0000000000..00bd08755f --- /dev/null +++ b/l2-orchestrator-standalone/scripts/check_extraction.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Assert this package still corresponds to the runtime it measures. +# +# The engine is compiled straight out of ../src, so there is no copy to drift +# and nothing to diff. What can still go wrong is the package's own description +# of what it builds: +# 1. the TU list in CMakeLists.txt matches build_config.py's `host` target +# (minus host/, which is runtime_maker and its CANN dependencies); +# 2. every include directory and engine TU CMake names actually exists; +# 3. the qwen3_dynamic_tensormap case is present, in exactly one place, and +# the build still pins it to QWEN3_SPMD_TIER=0. +# +# Usage: scripts/check_extraction.sh [path-to-simpler-checkout] + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SIMPLER="${1:-$ROOT/..}" + +GREEN='\033[32m'; RED='\033[31m'; YELLOW='\033[33m'; RESET='\033[0m' +ok() { printf "${GREEN}%s${RESET}\n" "$*"; } +warn() { printf "${YELLOW}WARN: %s${RESET}\n" "$*"; } +FAIL=0 +fail() { printf "${RED}FAIL: %s${RESET}\n" "$*"; FAIL=$((FAIL + 1)); } + +if [[ ! -d "$SIMPLER/src/a2a3/runtime/host_build_graph" ]]; then + warn "simpler tree not found at $SIMPLER — pass its path as \$1. Skipping." + exit 0 +fi + +printf '\n\033[1m### 1/3 TU list matches build_config.py host target\033[0m\n' +# The engine TUs CMake compiles, relative to the runtime dir. +CMAKE_TUS=$(grep -oE '\$\{R\}/[a-z0-9_/]+\.cpp' "$ROOT/CMakeLists.txt" | sed 's|${R}/||' | sort) +# What build_config.py's host target says, minus host/ (runtime_maker + CANN). +EXPECTED=$(cd "$SIMPLER/src/a2a3/runtime/host_build_graph" \ + && find runtime/orchestrator_core runtime/shared orchestration -name '*.cpp' | sort) +if [[ "$CMAKE_TUS" == "$EXPECTED" ]]; then + ok "$(echo "$CMAKE_TUS" | wc -l) TUs match" +else + fail "TU list disagrees with the source repo" + diff <(echo "$CMAKE_TUS") <(echo "$EXPECTED") | sed 's/^/ /' +fi + +printf '\n\033[1m### 2/3 every path CMake names exists\033[0m\n' +# A moved or renamed engine header fails the build anyway, but it fails as a +# wall of missing-include errors. Naming the dead path is faster to act on. +MISSING=0 +R="$SIMPLER/src/a2a3/runtime/host_build_graph" +while IFS= read -r tu; do + [[ -f "$R/$tu" ]] || { fail "engine TU not found: $tu"; MISSING=$((MISSING + 1)); } +done <<< "$CMAKE_TUS" +# Include dirs are written against ${R} or ${S}. Read them only from inside the +# target_include_directories block — matching ${R}/... anywhere in the file +# would also pick up the TU list with its .cpp suffix stripped. +while IFS= read -r inc; do + resolved="${inc/\$\{R\}/$R}" + resolved="${resolved/\$\{S\}/$SIMPLER/src}" + [[ -d "$resolved" ]] || { fail "include dir not found: $inc"; MISSING=$((MISSING + 1)); } +done < <(awk '/^target_include_directories\(l2_engine/,/^\)/' "$ROOT/CMakeLists.txt" \ + | grep -oE '\$\{[RS]\}(/[a-zA-Z0-9_/]+)?' | sort -u) +[[ $MISSING -eq 0 ]] && ok "all engine TUs and include dirs resolve" + +printf '\n\033[1m### 3/3 qwen3_dynamic_tensormap case: present once, pinned to tier 0\033[0m\n' +CASES=$(find "$ROOT" -name 'qwen3_dynamic_tensormap.h' -not -path "*/build*" 2>/dev/null) +CASE_N=$(echo "$CASES" | grep -c . ) +if [[ $CASE_N -eq 0 ]]; then + fail "qwen3_dynamic_tensormap.h not found in the package" +elif [[ $CASE_N -gt 1 ]]; then + fail "several copies of the case — the build may pick up the wrong one:" + echo "$CASES" | sed 's/^/ /' +else + ok "case present at ${CASES#"$ROOT"/} ($(wc -l < "$CASES") lines)" +fi +# The tier changes the DAG, so a build that lost the pin measures a different +# graph while still printing plausible numbers. +if grep -q 'QWEN3_SPMD_TIER=0' "$ROOT/CMakeLists.txt"; then + ok "build pins QWEN3_SPMD_TIER=0" +else + fail "CMakeLists.txt no longer pins QWEN3_SPMD_TIER=0" +fi + +echo +if [[ $FAIL -eq 0 ]]; then + ok "harness corresponds to the repo" + exit 0 +fi +printf "${RED}%d failure(s)${RESET}\n" "$FAIL" +exit 1 diff --git a/l2-orchestrator-standalone/scripts/cold_warm_table.py b/l2-orchestrator-standalone/scripts/cold_warm_table.py new file mode 100644 index 0000000000..511a085cab --- /dev/null +++ b/l2-orchestrator-standalone/scripts/cold_warm_table.py @@ -0,0 +1,39 @@ +"""Per-STEP cold vs prefaulted table, from two --mode=profile reports. + +The first-touch share is the whole point: a STEP that is large cold and small +warm was paging, not orchestrating, and attributing its cost to engine work +sends optimization at the wrong target. +""" + +import re +import sys + + +def steps(path): + out = [] + for line in open(path): + m = re.match(r"\s{2}(STEP \d.*?)\s+(\d+)\s+([\d.]+)\s+([\d.]+)%\s*$", line) + if m: + out.append((m.group(1).strip(), float(m.group(3)), float(m.group(4)))) + return out + + +def main(cold_path, warm_path): + cold, warm = steps(cold_path), steps(warm_path) + if not cold or not warm: + print("no STEP rows found — was this a Level-3 build?", file=sys.stderr) + return 1 + print("qwen3-dyn — per-STEP totals, cold vs prefaulted (ms)") + print(f" {'step':<44}{'cold':>10}{'warm':>10}{'warm %':>9} {'first-touch share':>18}") + print(" " + "-" * 93) + for (name, cms, _), (_, wms, wpct) in zip(cold, warm): + ft = 100.0 * (cms - wms) / cms if cms > 0 else 0.0 + print(f" {name:<44}{cms:>10.3f}{wms:>10.3f}{wpct:>8.1f}% {ft:>17.0f}%") + print(" " + "-" * 93) + print(f" {'TOTAL':<44}{sum(x[1] for x in cold):>10.3f}{sum(x[1] for x in warm):>10.3f}") + print(" first-touch share = how much of the cold number was paging, not engine work.") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1], sys.argv[2])) diff --git a/l2-orchestrator-standalone/scripts/profile_l2.sh b/l2-orchestrator-standalone/scripts/profile_l2.sh new file mode 100755 index 0000000000..d26d3059d1 --- /dev/null +++ b/l2-orchestrator-standalone/scripts/profile_l2.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# +# Full bench + profile of the L2 orchestration sequence, driven by +# qwen3_dynamic_tensormap.h at QWEN3_SPMD_TIER=0. +# +# 1. Throughput, uninstrumented — the numbers to quote. +# 2. Three-level profile (four lines / intercepted ops table / engine's own +# per-STEP counters). +# 3. Cold/warm attribution: how much of each STEP is first-touch page cost +# rather than engine work. Without this, STEP 5 reads as 66% of submit cost +# and the real bottleneck (STEP 3) is invisible. Needs the Level-4 build +# for the TensorMap chain stats. +# +# No engine source is modified; see the header of bench/l2_profile.h for which +# seams the timestamps come from. +# +# Usage: scripts/profile_l2.sh [--repeat N] [--jobs N] + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +REPEAT=5 +JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)" +while [[ $# -gt 0 ]]; do + case "$1" in + --repeat) REPEAT="$2"; shift 2 ;; + --jobs) JOBS="$2"; shift 2 ;; + -h|--help) sed -n '2,17p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +GREEN='\033[32m'; RED='\033[31m'; YELLOW='\033[33m'; RESET='\033[0m' +step() { printf '\n\033[1m### %s\033[0m\n' "$*"; } +green() { printf "${GREEN}%s${RESET}\n" "$*"; } +warn() { printf "${YELLOW}WARN: %s${RESET}\n" "$*"; } +FAILURES=0 +fail() { printf "${RED}FAIL: %s${RESET}\n" "$*"; FAILURES=$((FAILURES + 1)); } + +LIBSTDCXX_DIR="$(dirname "$(readlink -f "$(g++ -print-file-name=libstdc++.so)")")" +export LD_LIBRARY_PATH="${LIBSTDCXX_DIR}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + +REPORTS="$ROOT/reports" +LOGS="$ROOT/build/logs" +mkdir -p "$REPORTS" "$LOGS" + +# Pin to a small CPU set, not one CPU. Everything here is single-threaded (the +# orchestrator is single-writer by design), so a 4-CPU set costs nothing and +# keeps the run off whichever core the OS is busy with. +PIN=() +if command -v taskset > /dev/null 2>&1 && taskset -c 0-3 true 2>/dev/null; then + PIN=(taskset -c 0-3) +else + warn "taskset unavailable — measurements will be noisier" +fi + +# NOTHING is ever reclaimed here (no scheduler, no completions), so both the +# task window and the heap must hold the entire graph at once. These are the +# binary's own defaults, restated so a change to either is visible in one place. +ARGS=(--task-window=8192 --heap-mb=2048) + +# --------------------------------------------------------------------------- +step "Build: default, SIMPLER_ORCH_PROFILING=1, and +TENSORMAP" +# --------------------------------------------------------------------------- +cmake -B build -S . > "$LOGS/build-default.log" 2>&1 \ + && cmake --build build --parallel "$JOBS" >> "$LOGS/build-default.log" 2>&1 \ + || fail "default build (see $LOGS/build-default.log)" + +cmake -B build-prof -S . -DL2_ORCH_PROFILING=1 > "$LOGS/build-prof.log" 2>&1 \ + && cmake --build build-prof --parallel "$JOBS" >> "$LOGS/build-prof.log" 2>&1 \ + || fail "profiling build (see $LOGS/build-prof.log)" + +# Level 4 adds the TensorMap chain counters, which is what makes STEP 3's cost +# attributable to a root cause rather than just large. +cmake -B build-tm -S . -DL2_TENSORMAP_PROFILING=1 > "$LOGS/build-tm.log" 2>&1 \ + && cmake --build build-tm --parallel "$JOBS" >> "$LOGS/build-tm.log" 2>&1 \ + || fail "tensormap-profiling build (see $LOGS/build-tm.log)" +green "built build/ (quote from this), build-prof/ (L3), build-tm/ (L3+L4)" + +# --------------------------------------------------------------------------- +step "1/3 Throughput (uninstrumented)" +# --------------------------------------------------------------------------- +"${PIN[@]}" ./build/l2_bench --mode=throughput --repeat="$REPEAT" "${ARGS[@]}" || fail "throughput" + +# --------------------------------------------------------------------------- +step "2/3 Per-step profile" +# --------------------------------------------------------------------------- +"${PIN[@]}" ./build-prof/l2_bench --mode=profile "${ARGS[@]}" \ + --out="$REPORTS/profile-qwen3-dyn.json" | tee "$REPORTS/profile-qwen3-dyn.txt" \ + || fail "profile" + +# --------------------------------------------------------------------------- +step "3/3 Cold/warm attribution (which STEPs are really first-touch cost)" +# --------------------------------------------------------------------------- +# Prefaulting is a DIAGNOSTIC, not an optimization: it moves first-touch cost out +# of the measured window, it does not avoid it. The production fix for the SM +# half is to pool the host mirror across runs (runtime_maker.cpp:487 allocates it +# fresh every time). +echo +"${PIN[@]}" ./build-tm/l2_bench --mode=profile "${ARGS[@]}" \ + > "$LOGS/attr-cold.txt" 2>/dev/null || fail "attribution (cold)" +"${PIN[@]}" ./build-tm/l2_bench --mode=profile "${ARGS[@]}" --prefault-all \ + > "$LOGS/attr-warm.txt" 2>/dev/null || fail "attribution (warm)" +python3 "$ROOT/scripts/cold_warm_table.py" "$LOGS/attr-cold.txt" "$LOGS/attr-warm.txt" \ + || fail "attribution table" +sed -n '/LEVEL 4/,/wasted/p' "$LOGS/attr-warm.txt" | sed 's/^/ /' + +echo +echo "Reports: $REPORTS/profile-qwen3-dyn.txt (human), profile-qwen3-dyn.json" +if [[ $FAILURES -eq 0 ]]; then + green "profile complete" + exit 0 +fi +printf "${RED}%d failure(s)${RESET}\n" "$FAILURES" +exit 1 diff --git a/l2-orchestrator-standalone/src/host_shim/host_shim.cpp b/l2-orchestrator-standalone/src/host_shim/host_shim.cpp new file mode 100644 index 0000000000..4e312b0b3b --- /dev/null +++ b/l2-orchestrator-standalone/src/host_shim/host_shim.cpp @@ -0,0 +1,85 @@ +/* + * Host-side definitions for the handful of symbols the extracted L2 TUs + * reference but do not define. + * + * In simpler these live in the AICPU / platform binaries. None of them is part + * of the orchestrator's logic: four are the scope-stats collector (a DFX + * sampler the orchestrator calls only behind is_scope_stats_enabled(), which is + * weak-false in host builds), five are the unified-log sinks, and + * get_sys_cnt_aicpu is the cycle source the SIMPLER_ORCH_PROFILING build reads. + * + * The log sinks write to stderr and are silent unless L2_ORCH_VERBOSE is set, + * so a bench run's output stays parseable. The scope-stats entries are inert: + * this package never enables the collector, so they exist only to close the + * link. + */ + +#include +#include +#include +#include + +#include "aicpu/device_time.h" + +namespace { + +bool verbose() { + static const bool on = std::getenv("L2_ORCH_VERBOSE") != nullptr; + return on; +} + +void emit(const char *level, const char *func, const char *fmt, va_list args) { + if (!verbose()) return; + std::fprintf(stderr, "[%s] %s: ", level, func); + std::vfprintf(stderr, fmt, args); + std::fputc('\n', stderr); +} + +} // namespace + +extern "C" { + +void unified_log_error(const char *func, const char *fmt, ...) { + // Errors print regardless of L2_ORCH_VERBOSE: a fatal latched inside the + // orchestrator silently turns every later submit into a no-op, which would + // otherwise show up only as an implausibly good throughput number. + va_list args; + va_start(args, fmt); + std::fprintf(stderr, "[ERROR] %s: ", func); + std::vfprintf(stderr, fmt, args); + std::fputc('\n', stderr); + va_end(args); +} + +#define L2_SHIM_LOG(name, level) \ + void name(const char *func, const char *fmt, ...) { \ + va_list args; \ + va_start(args, fmt); \ + emit(level, func, fmt, args); \ + va_end(args); \ + } + +L2_SHIM_LOG(unified_log_warn, "WARN") +L2_SHIM_LOG(unified_log_timing, "TIMING") +L2_SHIM_LOG(unified_log_info, "INFO") +L2_SHIM_LOG(unified_log_debug, "DEBUG") + +#undef L2_SHIM_LOG + +void scope_stats_begin( + int, int32_t, int32_t, uint64_t, uint64_t, int32_t, int32_t, int32_t +) {} +void scope_stats_end( + int, int32_t, int32_t, uint64_t, uint64_t, int32_t, int32_t, int32_t +) {} +void scope_stats_on_fatal() {} +void scope_stats_set_pending_site(const char *, int) {} + +} // extern "C" + +// The engine's own CYCLE_COUNT_* macros (SIMPLER_ORCH_PROFILING=1) read this. +// device_time_now_ticks() is cntvct_el0 on aarch64 and steady_clock elsewhere, +// so the per-step cycle accounting is sourced identically to the device build. +// Declared with C++ linkage in aicpu/device_time.h, so it must be defined that +// way too — an extern "C" definition is a different symbol. +uint64_t get_sys_cnt_aicpu() { return device_time_now_ticks(); }