diff --git a/docs/dfx/scope-stats.md b/docs/dfx/scope-stats.md index 57c2aa5733..2c44a12945 100644 --- a/docs/dfx/scope-stats.md +++ b/docs/dfx/scope-stats.md @@ -7,9 +7,9 @@ high-water mark. When a model runs out of task windows, heap, or tensormap / dependency-list entries, the failure tells you *which* resource is exhausted but not *where*; scope stats gives you the where. -It is a diagnostic-only, opt-in feature for the -**tensormap_and_ringbuffer (T&R)** runtime. When disabled (the default) -it costs a single bool load per probe. +It is a diagnostic-only, opt-in feature for both +**tensormap_and_ringbuffer (T&R)** and **host_build_graph (HBG)**. When +disabled (the default) it costs a single bool load per probe. This guide also covers the background behind the feature, the T&R resource/ring_depth/scope model, and the data flow behind the HTML report. @@ -21,7 +21,7 @@ resulting `scope_stats/scope_stats.jsonl` into an HTML report. ### Step 1 — Run with `--enable-scope-stats` -Pass the flag to any T&R example or scene test: +Pass the flag to a T&R or HBG example or scene test: ```bash CASE=... @@ -29,9 +29,9 @@ NAME=... python "tests/st/${CASE}/test_${NAME}.py" -p a2a3 -d 0 --enable-scope-stats ``` -The flag is bit 4 of `enable_profiling_flag`; on a T&R run it turns on -per-scope peak tracking. On other runtimes the flag is accepted but -produces no records. +The flag is bit 4 of `enable_profiling_flag` on the device-orchestrated +T&R path. HBG consumes the same flag before graph construction and records +the host orchestrator's scope boundaries directly. ### Step 2 — Locate the output @@ -68,6 +68,8 @@ PY The three arrays are indexed by `ring` (`0..3`) and should match the effective runtime configuration. Per-sample `ring` values show which scope-depth rings were actually touched by the run; they are scope records, not task counts. +HBG is whole-graph-resident with one polling ring and no dependency-list pool, +so only index 0 is populated and `dep_pool_max[0]` is zero. ### Step 3 — Visualize with `scope_stats_plot.py` @@ -310,15 +312,16 @@ render `used/cap` without a second device→host query. | chip swimlane | platform only | all runtimes | reads AICore ring buffers | | dep_gen | platform only | all runtimes | traces `submit_task` | | args dump | platform only | all runtimes | dumps argument data | -| **scope stats** | **platform API + runtime call sites** | **T&R only** | runtime extracts values, platform tracks peaks | +| **scope stats** | **platform API + runtime call sites** | **all runtimes** | T&R streams from AICPU; HBG records during host graph construction | ### 4.4 Symbol resolution -`kernel.cpp` (platform, shared by all runtimes) always calls -`set_scope_stats_enabled` / `set_platform_scope_stats_base`, so the -collector symbols resolve into every AICPU `.so`. Only the T&R runtime -adds the `begin`/`end`/capacity call sites, so only it produces records; -host_build_graph links the collector but never invokes it. +`kernel.cpp` (platform, shared by all runtimes) calls +`set_scope_stats_enabled` / `set_platform_scope_stats_base` for the +device-backed T&R collector. HBG resolves the same begin/end probes to a +host capture implementation. Its runner turns the device flag off, avoids +allocating an unused device buffer pool, and emits the captured host records +after execution completes. ### 4.5 Data flow @@ -347,9 +350,12 @@ A worked example is in — it runs the `vector_example` orchestration with `--enable-scope-stats` and asserts the resulting NDJSON. -### 4.6 Future: cross-runtime support +### 4.6 Host-build-graph capture -If host_build_graph adds scope-like concepts, extending scope_stats only -requires adding the same platform call sites in HBG — no platform -changes. The collector is already runtime-agnostic: it accepts plain -values and has no knowledge of T&R types. +HBG runs its complete orchestration during `bind_callable_to_runtime`, before +`prepare_execution` creates any device collectors. Its begin/end probes +therefore write to a thread-local host recorder. The recorder captures the +same `ScopeStatsRecord` values and calls the shared JSONL serializer, so the +version 6 schema and plotting tool are identical across runtime variants. +The capture is reset once per host orchestration and emitted by the same +progress thread, matching HBG's host-direct `dep_gen` lifecycle. diff --git a/src/a2a3/platform/onboard/host/device_runner.cpp b/src/a2a3/platform/onboard/host/device_runner.cpp index c3e52b591a..7ede49fd76 100644 --- a/src/a2a3/platform/onboard/host/device_runner.cpp +++ b/src/a2a3/platform/onboard/host/device_runner.cpp @@ -306,7 +306,9 @@ int DeviceRunner::prepare_execution( if (enable_dep_gen_ && !dep_gen_host_graph_active()) { SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_DEP_GEN); } - if (enable_scope_stats_) SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_SCOPE_STATS); + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { + SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_SCOPE_STATS); + } execution->kernel_args.args.enable_profiling_flag = enable_profiling_flag; resolve_task_binary_addrs(runtime); @@ -403,7 +405,7 @@ int DeviceRunner::prepare_execution( } } - if (enable_scope_stats_) { + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { rc = init_scope_stats(launch_aicpu_num, device_id_, execution->kernel_args); if (rc != 0) { LOG_ERROR("init_scope_stats failed: %d", rc); diff --git a/src/a2a3/platform/sim/host/device_runner.cpp b/src/a2a3/platform/sim/host/device_runner.cpp index 493955e223..0a6a4c9544 100644 --- a/src/a2a3/platform/sim/host/device_runner.cpp +++ b/src/a2a3/platform/sim/host/device_runner.cpp @@ -341,7 +341,7 @@ int DeviceRunner::prepare_execution( if (enable_dep_gen_ && !dep_gen_host_graph_active()) { SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_DEP_GEN); } - if (enable_scope_stats_) { + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_SCOPE_STATS); } kernel_args_.enable_profiling_flag = enable_profiling_flag; @@ -407,7 +407,7 @@ int DeviceRunner::prepare_execution( } } - if (enable_scope_stats_) { + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { rc = init_scope_stats(launch_aicpu_num); if (rc != 0) { LOG_ERROR("init_scope_stats failed: %d", rc); @@ -519,7 +519,7 @@ DeviceRunner::launch_execution(std::unique_ptr prepared, Laun set_pmu_enabled_func_(enable_pmu_); set_platform_dep_gen_base_func_(kernel_args_.dep_gen_data_base); set_dep_gen_enabled_func_(enable_dep_gen_ && !dep_gen_host_graph_active()); - set_scope_stats_enabled_func_(enable_scope_stats_); + set_scope_stats_enabled_func_(enable_scope_stats_ && !scope_stats_uses_host_capture()); set_platform_scope_stats_base_func_(kernel_args_.scope_stats_data_base); auto thread_factory = [this](std::function fn) { @@ -529,7 +529,9 @@ DeviceRunner::launch_execution(std::unique_ptr prepared, Laun if (enable_dump_args_) dump_collector_.start(thread_factory); if (enable_pmu_) pmu_collector_.start(thread_factory); if (enable_dep_gen_ && !dep_gen_host_graph_active()) dep_gen_collector_.start(thread_factory); - if (enable_scope_stats_) scope_stats_collector_.start(thread_factory); + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { + scope_stats_collector_.start(thread_factory); + } if (kernel_args_.device_wall_data_base != 0) { *reinterpret_cast(kernel_args_.device_wall_data_base) = 0; @@ -693,9 +695,16 @@ int DeviceRunner::drain_execution(ActiveExecution &) { } if (enable_scope_stats_) { - scope_stats_collector_.stop(); - scope_stats_collector_.reconcile_counters(); - scope_stats_collector_.write_jsonl(output_prefix_); + if (scope_stats_uses_host_capture()) { + int rc = write_host_scope_stats(); + if (rc != 0) { + LOG_ERROR("scope_stats host graph emit failed (%d) — scope_stats.jsonl not produced", rc); + } + } else { + scope_stats_collector_.stop(); + scope_stats_collector_.reconcile_counters(); + scope_stats_collector_.write_jsonl(output_prefix_); + } } print_handshake_results(); diff --git a/src/a2a3/runtime/host_build_graph/host/host_orch_compat_stubs.cpp b/src/a2a3/runtime/host_build_graph/host/host_orch_compat_stubs.cpp index 81c2c0dd7c..592a82911c 100644 --- a/src/a2a3/runtime/host_build_graph/host/host_orch_compat_stubs.cpp +++ b/src/a2a3/runtime/host_build_graph/host/host_orch_compat_stubs.cpp @@ -8,25 +8,9 @@ * See LICENSE in the root of the software repository for the full text of the License. * ----------------------------------------------------------------------------------------------------------- */ -/** - * Host-side weak stubs for AICPU-only scope-stats symbols. - * - * host_build_graph runs the orchestrator on the host (host-orch-first), so the - * orchestrator core (pto_orchestrator.cpp / pto_runtime2.cpp) is compiled into - * libhost_runtime.so, which is dlopen'd RTLD_LOCAL and must therefore resolve - * all of its symbols. The scope-stats collector is AICPU-only (defined in - * common/platform/.../aicpu) and is NOT linked into the host library. It records - * on-device diagnostics; the host orchestrator only builds the task graph, so - * no-op definitions are correct here. - * - * Marked weak + hidden so they never leak into the global dynamic symbol table - * (RTLD_LOCAL keeps them library-local anyway) and never shadow the AICPU - * library's strong definitions, mirroring the weak-stub pattern in - * pto_orchestrator.cpp. - */ - #include "aicpu/scope_stats_collector_aicpu.h" +// Minimal host-orchestrator link targets omit the host capture implementation. __attribute__((weak, visibility("hidden"))) void scope_stats_begin(int, int32_t, int32_t, uint64_t, uint64_t, int32_t, int32_t, int32_t) {} diff --git a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp index 572d3828a1..aac697419d 100644 --- a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp @@ -62,6 +62,7 @@ #include "../runtime/pto_shared_memory.h" #include "../runtime/pto_types.h" #include "../runtime/runtime.h" +#include "../runtime/scope_stats_host_graph.h" #include "../../../../common/runtime_status/error_log.h" #include "../../../../common/task_interface/call_config.h" #include "../../../../common/worker/pto_runtime_c_api.h" @@ -498,6 +499,11 @@ int32_t run_host_orchestration( } rt->orchestrator.wire_arena_pointers(layout.orch, host_arena, &rt->scheduler); + auto &scope_alloc = rt->orchestrator.ring.task_allocator; + scope_stats_host_graph_begin_capture( + scope_alloc.window_size(), scope_alloc.heap_capacity(), rt->orchestrator.tensor_map.pool_capacity() + ); + PTO2SharedMemoryHandle host_sm_handle; if (!host_sm_handle.init_per_ring(host_sm, sm_size, eff_task_window_sizes, eff_heap_sizes)) { LOG_ERROR("host-orch: host SM init_per_ring failed"); diff --git a/src/a2a3/runtime/host_build_graph/host/scope_stats_host_graph.cpp b/src/a2a3/runtime/host_build_graph/host/scope_stats_host_graph.cpp new file mode 100644 index 0000000000..9a0341e4a7 --- /dev/null +++ b/src/a2a3/runtime/host_build_graph/host/scope_stats_host_graph.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "scope_stats_host_graph.h" + +#include "host/scope_stats_host_capture.h" + +namespace { +thread_local ScopeStatsHostCapture g_scope_stats_capture; +} // namespace + +void scope_stats_host_graph_begin_capture(int32_t task_window_cap, uint64_t heap_cap, int32_t tensormap_cap) { + g_scope_stats_capture.begin_capture(task_window_cap, heap_cap, tensormap_cap); +} + +extern "C" bool scope_stats_host_graph_active() { return true; } + +extern "C" void scope_stats_host_graph_set_enabled(bool enabled) { g_scope_stats_capture.set_enabled(enabled); } + +extern "C" int scope_stats_host_graph_write_jsonl(const char *output_dir) { + return g_scope_stats_capture.write_jsonl(output_dir); +} + +extern "C" bool is_scope_stats_enabled() { return g_scope_stats_capture.enabled(); } + +extern "C" void scope_stats_set_pending_site(const char *file, int line) { + g_scope_stats_capture.set_pending_site(file, line); +} + +extern "C" void scope_stats_begin( + int ring_id, int32_t task_start, int32_t task_end, uint64_t heap_start, uint64_t heap_end, int32_t dep_pool_start, + int32_t dep_pool_end, int32_t tensormap_used +) { + g_scope_stats_capture.begin( + ring_id, task_start, task_end, heap_start, heap_end, dep_pool_start, dep_pool_end, tensormap_used + ); +} + +extern "C" void scope_stats_end( + int ring_id, int32_t task_start, int32_t task_end, uint64_t heap_start, uint64_t heap_end, int32_t dep_pool_start, + int32_t dep_pool_end, int32_t tensormap_used +) { + g_scope_stats_capture.end( + ring_id, task_start, task_end, heap_start, heap_end, dep_pool_start, dep_pool_end, tensormap_used + ); +} + +extern "C" void scope_stats_note_heap_wrap(int side) { g_scope_stats_capture.note_heap_wrap(side); } + +extern "C" void scope_stats_on_fatal() { g_scope_stats_capture.on_fatal(); } diff --git a/src/a2a3/runtime/host_build_graph/runtime/scope_stats_host_graph.h b/src/a2a3/runtime/host_build_graph/runtime/scope_stats_host_graph.h new file mode 100644 index 0000000000..9551cdc963 --- /dev/null +++ b/src/a2a3/runtime/host_build_graph/runtime/scope_stats_host_graph.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include + +// Reset host-side capture after the HBG orchestrator has initialized its +// resource pools and before the outer executor scope begins. +void scope_stats_host_graph_begin_capture(int32_t task_window_cap, uint64_t heap_cap, int32_t tensormap_cap); + +extern "C" { +bool scope_stats_host_graph_active(); +void scope_stats_host_graph_set_enabled(bool enabled); +int scope_stats_host_graph_write_jsonl(const char *output_dir); +} diff --git a/src/a5/platform/onboard/host/device_runner.cpp b/src/a5/platform/onboard/host/device_runner.cpp index 5feb045751..11f047563c 100644 --- a/src/a5/platform/onboard/host/device_runner.cpp +++ b/src/a5/platform/onboard/host/device_runner.cpp @@ -288,7 +288,9 @@ int DeviceRunner::prepare_execution( if (enable_chip_swimlane_) SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_CHIP_SWIMLANE); if (enable_pmu_) SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_PMU); if (enable_dep_gen_) SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_DEP_GEN); - if (enable_scope_stats_) SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_SCOPE_STATS); + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { + SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_SCOPE_STATS); + } execution->kernel_args.args.enable_profiling_flag = enable_profiling_flag; resolve_task_binary_addrs(runtime); @@ -407,7 +409,7 @@ int DeviceRunner::prepare_execution( } } - if (enable_scope_stats_) { + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { rc = init_scope_stats(active_aicpu_num, device_id_, execution->kernel_args); if (rc != 0) { LOG_ERROR("init_scope_stats failed: %d", rc); diff --git a/src/a5/platform/sim/host/device_runner.cpp b/src/a5/platform/sim/host/device_runner.cpp index 56a9520cb1..40c270c1fe 100644 --- a/src/a5/platform/sim/host/device_runner.cpp +++ b/src/a5/platform/sim/host/device_runner.cpp @@ -311,7 +311,7 @@ int DeviceRunner::prepare_execution( if (enable_dep_gen_) { SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_DEP_GEN); } - if (enable_scope_stats_) { + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { SIMPLER_SET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_SCOPE_STATS); } @@ -375,7 +375,7 @@ int DeviceRunner::prepare_execution( } } - if (enable_scope_stats_) { + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { rc = init_scope_stats(launch_aicpu_num); if (rc != 0) { LOG_ERROR("init_scope_stats failed: %d", rc); @@ -464,7 +464,7 @@ DeviceRunner::launch_execution(std::unique_ptr prepared, Laun set_pmu_enabled_func_(enable_pmu_); set_platform_dep_gen_base_func_(kernel_args_.dep_gen_data_base); set_dep_gen_enabled_func_(enable_dep_gen_); - set_scope_stats_enabled_func_(enable_scope_stats_); + set_scope_stats_enabled_func_(enable_scope_stats_ && !scope_stats_uses_host_capture()); set_platform_scope_stats_base_func_(kernel_args_.scope_stats_data_base); auto thread_factory = [this](std::function fn) { @@ -474,7 +474,9 @@ DeviceRunner::launch_execution(std::unique_ptr prepared, Laun if (enable_dump_args_) dump_collector_.start(thread_factory); if (enable_pmu_) pmu_collector_.start(thread_factory); if (enable_dep_gen_) dep_gen_collector_.start(thread_factory); - if (enable_scope_stats_) scope_stats_collector_.start(thread_factory); + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { + scope_stats_collector_.start(thread_factory); + } if (kernel_args_.device_wall_data_base != 0) { *reinterpret_cast(kernel_args_.device_wall_data_base) = 0; @@ -630,9 +632,16 @@ int DeviceRunner::drain_execution(ActiveExecution &) { } if (enable_scope_stats_) { - scope_stats_collector_.stop(); - scope_stats_collector_.reconcile_counters(); - scope_stats_collector_.write_jsonl(output_prefix_); + if (scope_stats_uses_host_capture()) { + int rc = write_host_scope_stats(); + if (rc != 0) { + LOG_ERROR("scope_stats host graph emit failed (%d) — scope_stats.jsonl not produced", rc); + } + } else { + scope_stats_collector_.stop(); + scope_stats_collector_.reconcile_counters(); + scope_stats_collector_.write_jsonl(output_prefix_); + } } print_handshake_results(); diff --git a/src/a5/runtime/host_build_graph/host/host_orch_compat_stubs.cpp b/src/a5/runtime/host_build_graph/host/host_orch_compat_stubs.cpp index 81c2c0dd7c..592a82911c 100644 --- a/src/a5/runtime/host_build_graph/host/host_orch_compat_stubs.cpp +++ b/src/a5/runtime/host_build_graph/host/host_orch_compat_stubs.cpp @@ -8,25 +8,9 @@ * See LICENSE in the root of the software repository for the full text of the License. * ----------------------------------------------------------------------------------------------------------- */ -/** - * Host-side weak stubs for AICPU-only scope-stats symbols. - * - * host_build_graph runs the orchestrator on the host (host-orch-first), so the - * orchestrator core (pto_orchestrator.cpp / pto_runtime2.cpp) is compiled into - * libhost_runtime.so, which is dlopen'd RTLD_LOCAL and must therefore resolve - * all of its symbols. The scope-stats collector is AICPU-only (defined in - * common/platform/.../aicpu) and is NOT linked into the host library. It records - * on-device diagnostics; the host orchestrator only builds the task graph, so - * no-op definitions are correct here. - * - * Marked weak + hidden so they never leak into the global dynamic symbol table - * (RTLD_LOCAL keeps them library-local anyway) and never shadow the AICPU - * library's strong definitions, mirroring the weak-stub pattern in - * pto_orchestrator.cpp. - */ - #include "aicpu/scope_stats_collector_aicpu.h" +// Minimal host-orchestrator link targets omit the host capture implementation. __attribute__((weak, visibility("hidden"))) void scope_stats_begin(int, int32_t, int32_t, uint64_t, uint64_t, int32_t, int32_t, int32_t) {} diff --git a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp index fc52dd4b9c..8ee6373267 100644 --- a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp @@ -62,6 +62,7 @@ #include "../runtime/pto_shared_memory.h" #include "../runtime/pto_types.h" #include "../runtime/runtime.h" +#include "../runtime/scope_stats_host_graph.h" #include "../../../../common/runtime_status/error_log.h" #include "../../../../common/task_interface/call_config.h" #include "../../../../common/worker/pto_runtime_c_api.h" @@ -544,6 +545,11 @@ int32_t run_host_orchestration( } rt->orchestrator.wire_arena_pointers(layout.orch, host_arena, &rt->scheduler); + auto &scope_alloc = rt->orchestrator.ring.task_allocator; + scope_stats_host_graph_begin_capture( + scope_alloc.window_size(), scope_alloc.heap_capacity(), rt->orchestrator.tensor_map.pool_capacity() + ); + // Initialize the host SM header (ring flow control) so submit_task can run. PTO2SharedMemoryHandle host_sm_handle; if (!host_sm_handle.init_per_ring(host_sm, sm_size, eff_task_window_sizes, eff_heap_sizes)) { diff --git a/src/a5/runtime/host_build_graph/host/scope_stats_host_graph.cpp b/src/a5/runtime/host_build_graph/host/scope_stats_host_graph.cpp new file mode 100644 index 0000000000..9a0341e4a7 --- /dev/null +++ b/src/a5/runtime/host_build_graph/host/scope_stats_host_graph.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "scope_stats_host_graph.h" + +#include "host/scope_stats_host_capture.h" + +namespace { +thread_local ScopeStatsHostCapture g_scope_stats_capture; +} // namespace + +void scope_stats_host_graph_begin_capture(int32_t task_window_cap, uint64_t heap_cap, int32_t tensormap_cap) { + g_scope_stats_capture.begin_capture(task_window_cap, heap_cap, tensormap_cap); +} + +extern "C" bool scope_stats_host_graph_active() { return true; } + +extern "C" void scope_stats_host_graph_set_enabled(bool enabled) { g_scope_stats_capture.set_enabled(enabled); } + +extern "C" int scope_stats_host_graph_write_jsonl(const char *output_dir) { + return g_scope_stats_capture.write_jsonl(output_dir); +} + +extern "C" bool is_scope_stats_enabled() { return g_scope_stats_capture.enabled(); } + +extern "C" void scope_stats_set_pending_site(const char *file, int line) { + g_scope_stats_capture.set_pending_site(file, line); +} + +extern "C" void scope_stats_begin( + int ring_id, int32_t task_start, int32_t task_end, uint64_t heap_start, uint64_t heap_end, int32_t dep_pool_start, + int32_t dep_pool_end, int32_t tensormap_used +) { + g_scope_stats_capture.begin( + ring_id, task_start, task_end, heap_start, heap_end, dep_pool_start, dep_pool_end, tensormap_used + ); +} + +extern "C" void scope_stats_end( + int ring_id, int32_t task_start, int32_t task_end, uint64_t heap_start, uint64_t heap_end, int32_t dep_pool_start, + int32_t dep_pool_end, int32_t tensormap_used +) { + g_scope_stats_capture.end( + ring_id, task_start, task_end, heap_start, heap_end, dep_pool_start, dep_pool_end, tensormap_used + ); +} + +extern "C" void scope_stats_note_heap_wrap(int side) { g_scope_stats_capture.note_heap_wrap(side); } + +extern "C" void scope_stats_on_fatal() { g_scope_stats_capture.on_fatal(); } diff --git a/src/a5/runtime/host_build_graph/runtime/scope_stats_host_graph.h b/src/a5/runtime/host_build_graph/runtime/scope_stats_host_graph.h new file mode 100644 index 0000000000..9551cdc963 --- /dev/null +++ b/src/a5/runtime/host_build_graph/runtime/scope_stats_host_graph.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include + +// Reset host-side capture after the HBG orchestrator has initialized its +// resource pools and before the outer executor scope begins. +void scope_stats_host_graph_begin_capture(int32_t task_window_cap, uint64_t heap_cap, int32_t tensormap_cap); + +extern "C" { +bool scope_stats_host_graph_active(); +void scope_stats_host_graph_set_enabled(bool enabled); +int scope_stats_host_graph_write_jsonl(const char *output_dir); +} diff --git a/src/common/platform/include/host/scope_stats_collector.h b/src/common/platform/include/host/scope_stats_collector.h index 379933ec65..a961dad79b 100644 --- a/src/common/platform/include/host/scope_stats_collector.h +++ b/src/common/platform/include/host/scope_stats_collector.h @@ -53,8 +53,7 @@ * "tensormap":int} */ -#ifndef SRC_COMMON_PLATFORM_INCLUDE_HOST_SCOPE_STATS_COLLECTOR_H_ -#define SRC_COMMON_PLATFORM_INCLUDE_HOST_SCOPE_STATS_COLLECTOR_H_ +#pragma once #include #include @@ -145,6 +144,11 @@ using ScopeStatsRegisterCallback = profiling_common::ProfRegisterCallback; using ScopeStatsUnregisterCallback = profiling_common::ProfUnregisterCallback; using ScopeStatsFreeCallback = profiling_common::ProfFreeCallback; +int write_scope_stats_jsonl( + const std::string &output_dir, const ScopeStatsDataHeader &header, uint32_t dropped_record_count, + uint32_t total_record_count, const std::vector &records +); + // --------------------------------------------------------------------------- // ScopeStatsCollector // --------------------------------------------------------------------------- @@ -207,5 +211,3 @@ class ScopeStatsCollector : public profiling_common::ProfilerBase +#include +#include +#include +#include +#include +#include +#include + +#include "aicpu/scope_stats_collector_aicpu.h" +#include "host/scope_stats_collector.h" + +// HBG scope boundaries execute before device collectors start. +class ScopeStatsHostCapture { +public: + void set_enabled(bool enabled) { enabled_ = enabled; } + bool enabled() const { return enabled_; } + + void begin_capture(int32_t task_window_cap, uint64_t heap_cap, int32_t tensormap_cap) { + captured_ = enabled_; + records_.clear(); + scopes_.clear(); + pending_site_file_.clear(); + pending_site_line_ = 0; + heap_wraps_.assign(1, {}); + header_ = {}; + header_.num_instances = 1; + header_.task_window_cap[0] = task_window_cap; + header_.heap_cap[0] = heap_cap; + // HBG polling readiness has no dependency-list pool. + header_.dep_pool_cap[0] = 0; + header_.tensormap_cap = tensormap_cap; + } + + void set_pending_site(const char *file, int line) { + if (!enabled_) return; + pending_site_file_ = file == nullptr ? "" : file; + pending_site_line_ = line; + } + + void begin( + int ring_id, int32_t task_start, int32_t task_end, uint64_t heap_start, uint64_t heap_end, + int32_t dep_pool_start, int32_t dep_pool_end, int32_t tensormap_used + ) { + if (!enabled_ || scopes_.size() >= static_cast(std::numeric_limits::max())) return; + scopes_.push_back({std::move(pending_site_file_), pending_site_line_, ring_id}); + pending_site_file_.clear(); + pending_site_line_ = 0; + append_record( + ring_id, SCOPE_STATS_PHASE_BEGIN, task_start, task_end, heap_start, heap_end, dep_pool_start, dep_pool_end, + tensormap_used + ); + } + + void + end(int ring_id, int32_t task_start, int32_t task_end, uint64_t heap_start, uint64_t heap_end, + int32_t dep_pool_start, int32_t dep_pool_end, int32_t tensormap_used) { + if (!enabled_ || scopes_.empty()) return; + append_record( + ring_id, SCOPE_STATS_PHASE_END, task_start, task_end, heap_start, heap_end, dep_pool_start, dep_pool_end, + tensormap_used + ); + scopes_.pop_back(); + } + + void note_heap_wrap(int side) { + if (!enabled_ || scopes_.empty()) return; + if (side != SCOPE_STATS_HEAP_SIDE_ALLOC && side != SCOPE_STATS_HEAP_SIDE_RECLAIM) return; + const int ring_id = scopes_.back().ring_id; + if (ring_id < 0 || static_cast(ring_id) >= heap_wraps_.size()) return; + ++heap_wraps_[ring_id][side]; + } + + void on_fatal() { + if (enabled_) header_.fatal_latched = 1; + } + + int write_jsonl(const char *output_dir) const { + if (output_dir == nullptr) return -1; + if (!captured_) return -3; + return write_scope_stats_jsonl( + output_dir, header_, /*dropped_record_count=*/0, static_cast(records_.size()), records_ + ); + } + + const std::vector &records() const { return records_; } + const ScopeStatsDataHeader &header() const { return header_; } + +private: + static std::string_view basename_of(std::string_view path) { + if (path.empty()) return "(unknown)"; + const size_t separator = path.find_last_of("/\\"); + return separator == std::string_view::npos ? path : path.substr(separator + 1); + } + + static void copy_basename(char (&dst)[32], std::string_view path) { + const std::string_view basename = basename_of(path); + const size_t length = std::min(basename.size(), sizeof(dst) - 1); + std::copy_n(basename.begin(), length, dst); + dst[length] = '\0'; + } + + uint64_t unroll_heap_offset(uint64_t offset, int ring_id, int side) const { + if (ring_id < 0 || static_cast(ring_id) >= heap_wraps_.size()) return offset; + return offset + heap_wraps_[ring_id][side] * header_.heap_cap[ring_id]; + } + + void append_record( + int ring_id, int16_t phase, int32_t task_start, int32_t task_end, uint64_t heap_start, uint64_t heap_end, + int32_t dep_pool_start, int32_t dep_pool_end, int32_t tensormap_used + ) { + ScopeStatsRecord record{}; + const ScopeFrame &scope = scopes_.back(); + copy_basename(record.site_file_basename, scope.site_file); + record.site_line = scope.site_line; + record.depth = static_cast(scopes_.size() - 1); + record.ring_id = static_cast(ring_id); + record.phase = phase; + record.task_start = task_start; + record.task_end = task_end; + record.dep_pool_start = dep_pool_start; + record.dep_pool_end = dep_pool_end; + record.tensormap_used = tensormap_used; + record.heap_start = unroll_heap_offset(heap_start, ring_id, SCOPE_STATS_HEAP_SIDE_RECLAIM); + record.heap_end = unroll_heap_offset(heap_end, ring_id, SCOPE_STATS_HEAP_SIDE_ALLOC); + records_.push_back(record); + } + + struct ScopeFrame { + std::string site_file; + int32_t site_line; + int32_t ring_id; + }; + + bool enabled_{false}; + bool captured_{false}; + std::string pending_site_file_; + int32_t pending_site_line_{0}; + std::vector scopes_; + std::vector> heap_wraps_; + ScopeStatsDataHeader header_{}; + std::vector records_; +}; diff --git a/src/common/platform/onboard/host/device_runner_base.cpp b/src/common/platform/onboard/host/device_runner_base.cpp index f4da5de12e..8afc5bf638 100644 --- a/src/common/platform/onboard/host/device_runner_base.cpp +++ b/src/common/platform/onboard/host/device_runner_base.cpp @@ -1037,6 +1037,23 @@ extern "C" __attribute__((weak)) int prewarm_config_impl( return 0; } +extern "C" __attribute__((weak, visibility("hidden"))) bool scope_stats_host_graph_active() { return false; } +extern "C" __attribute__((weak, visibility("hidden"))) void scope_stats_host_graph_set_enabled(bool) {} +extern "C" __attribute__((weak, visibility("hidden"))) int scope_stats_host_graph_write_jsonl(const char *) { + return -1; +} + +void DeviceRunnerBase::set_scope_stats_enabled(bool enable) { + enable_scope_stats_ = enable; + scope_stats_host_graph_set_enabled(enable); +} + +bool DeviceRunnerBase::scope_stats_uses_host_capture() const { return scope_stats_host_graph_active(); } + +int DeviceRunnerBase::write_host_scope_stats() const { + return scope_stats_host_graph_write_jsonl(output_prefix_.c_str()); +} + void DeviceRunnerBase::apply_call_config(const CallConfig &config) { set_chip_swimlane_enabled(config.enable_chip_swimlane); set_dump_args_enabled(config.enable_dump_args); @@ -1571,7 +1588,7 @@ void DeviceRunnerBase::start_shared_collectors_for_run() { if (enable_pmu_) { pmu_collector_.start(thread_factory); } - if (enable_scope_stats_) { + if (enable_scope_stats_ && !scope_stats_uses_host_capture()) { scope_stats_collector_.start(thread_factory); } } @@ -1600,9 +1617,16 @@ void DeviceRunnerBase::teardown_shared_collectors_after_run() { } if (enable_scope_stats_) { - scope_stats_collector_.stop(); - scope_stats_collector_.reconcile_counters(); - scope_stats_collector_.write_jsonl(output_prefix_); + if (scope_stats_uses_host_capture()) { + int rc = write_host_scope_stats(); + if (rc != 0) { + LOG_ERROR("scope_stats host graph emit failed (%d) — scope_stats.jsonl not produced", rc); + } + } else { + scope_stats_collector_.stop(); + scope_stats_collector_.reconcile_counters(); + scope_stats_collector_.write_jsonl(output_prefix_); + } } } diff --git a/src/common/platform/onboard/host/device_runner_base.h b/src/common/platform/onboard/host/device_runner_base.h index 47c4f3daf6..1e5e9ff1b7 100644 --- a/src/common/platform/onboard/host/device_runner_base.h +++ b/src/common/platform/onboard/host/device_runner_base.h @@ -679,7 +679,9 @@ class DeviceRunnerBase { enable_pmu_ = (enable_pmu > 0); pmu_event_type_ = resolve_pmu_event_type(enable_pmu); } - void set_scope_stats_enabled(bool enable) { enable_scope_stats_ = enable; } + void set_scope_stats_enabled(bool enable); + bool scope_stats_uses_host_capture() const; + int write_host_scope_stats() const; /** * Latch this run's per-run diagnostic config onto the runner's `enable_*_` diff --git a/src/common/platform/shared/host/scope_stats_collector.cpp b/src/common/platform/shared/host/scope_stats_collector.cpp index fccfdf01c7..7caec46fcf 100644 --- a/src/common/platform/shared/host/scope_stats_collector.cpp +++ b/src/common/platform/shared/host/scope_stats_collector.cpp @@ -251,9 +251,10 @@ bool ScopeStatsCollector::reconcile_counters() { // NDJSON export // --------------------------------------------------------------------------- -int ScopeStatsCollector::write_jsonl(const std::string &output_dir) { - if (!initialized_ || shm_host_ == nullptr) return 0; - +int write_scope_stats_jsonl( + const std::string &output_dir, const ScopeStatsDataHeader &header, uint32_t dropped_record_count, + uint32_t total_record_count, const std::vector &records +) { std::filesystem::path dir = std::filesystem::path(output_dir) / "scope_stats"; std::error_code ec; std::filesystem::create_directories(dir, ec); @@ -268,9 +269,6 @@ int ScopeStatsCollector::write_jsonl(const std::string &output_dir) { return -1; } - const ScopeStatsDataHeader *hdr = scope_stats_header(); - const ScopeStatsBufferState *state = scope_stats_state(0); - // Line 1: run metadata. Per-ring capacities and the tensormap capacity are // run-constants, so they live here once rather than on every record. std::string task_window_max; @@ -278,11 +276,11 @@ int ScopeStatsCollector::write_jsonl(const std::string &output_dir) { std::string dep_pool_max; for (int r = 0; r < PTO2_SCOPE_STATS_MAX_RING_DEPTH; r++) { char buf[32]; - std::snprintf(buf, sizeof(buf), "%s%d", r == 0 ? "" : ", ", hdr->task_window_cap[r]); + std::snprintf(buf, sizeof(buf), "%s%d", r == 0 ? "" : ", ", header.task_window_cap[r]); task_window_max += buf; - std::snprintf(buf, sizeof(buf), "%s%" PRIu64, r == 0 ? "" : ", ", hdr->heap_cap[r]); + std::snprintf(buf, sizeof(buf), "%s%" PRIu64, r == 0 ? "" : ", ", header.heap_cap[r]); heap_max += buf; - std::snprintf(buf, sizeof(buf), "%s%d", r == 0 ? "" : ", ", hdr->dep_pool_cap[r]); + std::snprintf(buf, sizeof(buf), "%s%d", r == 0 ? "" : ", ", header.dep_pool_cap[r]); dep_pool_max += buf; } std::fprintf( @@ -291,18 +289,17 @@ int ScopeStatsCollector::write_jsonl(const std::string &output_dir) { // wrapping ring offsets in v5) — see docs/dfx/scope-stats.md. "{\"version\": 6, \"fatal\": %s, \"dropped\": %u, \"total\": %u, " "\"task_window_max\": [%s], \"heap_max\": [%s], \"dep_pool_max\": [%s], \"tensormap_max\": %d}\n", - hdr->fatal_latched ? "true" : "false", state->dropped_record_count, state->total_record_count, - task_window_max.c_str(), heap_max.c_str(), dep_pool_max.c_str(), hdr->tensormap_cap + header.fatal_latched ? "true" : "false", dropped_record_count, total_record_count, task_window_max.c_str(), + heap_max.c_str(), dep_pool_max.c_str(), header.tensormap_cap ); // Serialize every record into one in-memory buffer, then a single fwrite. // The hot loop is one snprintf per record (not 6 fprintf): stdio format // parsing + per-call FILE locking on ~6×N calls was the dominant host cost. - std::scoped_lock lock(records_mutex_); std::string out; - out.reserve(records_.size() * 384); + out.reserve(records.size() * 384); char line[512]; - for (const ScopeStatsRecord &rec : records_) { + for (const ScopeStatsRecord &rec : records) { const int site_len = static_cast(strnlen(rec.site_file_basename, sizeof(rec.site_file_basename))); const char *phase = (rec.phase == SCOPE_STATS_PHASE_BEGIN) ? "begin" : "end"; int n = std::snprintf( @@ -321,12 +318,23 @@ int ScopeStatsCollector::write_jsonl(const std::string &output_dir) { std::fclose(fp); LOG_INFO( - "scope_stats: wrote %lu records (dropped=%u) to %s", static_cast(records_.size()), - state->dropped_record_count, path.c_str() + "scope_stats: wrote %lu records (dropped=%u) to %s", static_cast(records.size()), + dropped_record_count, path.c_str() ); return 0; } +int ScopeStatsCollector::write_jsonl(const std::string &output_dir) { + if (!initialized_ || shm_host_ == nullptr) return 0; + + const ScopeStatsDataHeader *header = scope_stats_header(); + const ScopeStatsBufferState *state = scope_stats_state(0); + std::scoped_lock lock(records_mutex_); + return write_scope_stats_jsonl( + output_dir, *header, state->dropped_record_count, state->total_record_count, records_ + ); +} + // --------------------------------------------------------------------------- // finalize // --------------------------------------------------------------------------- diff --git a/src/common/platform/sim/host/device_runner_base.cpp b/src/common/platform/sim/host/device_runner_base.cpp index 6be79bdf26..b743dc0463 100644 --- a/src/common/platform/sim/host/device_runner_base.cpp +++ b/src/common/platform/sim/host/device_runner_base.cpp @@ -665,6 +665,23 @@ extern "C" __attribute__((weak)) int prewarm_config_impl( return 0; } +extern "C" __attribute__((weak, visibility("hidden"))) bool scope_stats_host_graph_active() { return false; } +extern "C" __attribute__((weak, visibility("hidden"))) void scope_stats_host_graph_set_enabled(bool) {} +extern "C" __attribute__((weak, visibility("hidden"))) int scope_stats_host_graph_write_jsonl(const char *) { + return -1; +} + +void SimDeviceRunnerBase::set_scope_stats_enabled(bool enable) { + enable_scope_stats_ = enable; + scope_stats_host_graph_set_enabled(enable); +} + +bool SimDeviceRunnerBase::scope_stats_uses_host_capture() const { return scope_stats_host_graph_active(); } + +int SimDeviceRunnerBase::write_host_scope_stats() const { + return scope_stats_host_graph_write_jsonl(output_prefix_.c_str()); +} + void SimDeviceRunnerBase::apply_call_config(const CallConfig &config) { set_chip_swimlane_enabled(config.enable_chip_swimlane); set_dump_args_enabled(config.enable_dump_args); diff --git a/src/common/platform/sim/host/device_runner_base.h b/src/common/platform/sim/host/device_runner_base.h index 394e719521..31e59c53dd 100644 --- a/src/common/platform/sim/host/device_runner_base.h +++ b/src/common/platform/sim/host/device_runner_base.h @@ -276,7 +276,9 @@ class SimDeviceRunnerBase { enable_pmu_ = (enable_pmu > 0); pmu_event_type_ = resolve_pmu_event_type(enable_pmu); } - void set_scope_stats_enabled(bool enable) { enable_scope_stats_ = enable; } + void set_scope_stats_enabled(bool enable); + bool scope_stats_uses_host_capture() const; + int write_host_scope_stats() const; // Diagnostic artifact root directory (CallConfig::validate() enforces non-empty // upstream when any diagnostic is enabled). void set_output_prefix(const char *prefix) { output_prefix_ = (prefix != nullptr) ? prefix : ""; } diff --git a/tests/st/a2a3/host_build_graph/dfx/scope_stats/test_scope_stats.py b/tests/st/a2a3/host_build_graph/dfx/scope_stats/test_scope_stats.py new file mode 100644 index 0000000000..bce72b928d --- /dev/null +++ b/tests/st/a2a3/host_build_graph/dfx/scope_stats/test_scope_stats.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Host-orchestration scope-stats capture and JSONL export.""" + +import json +import time + +import torch +from simpler.task_interface import ArgDirection as D + +from simpler_setup import SceneTestCase, TaskArgsBuilder, TensorArg, scene_test +from simpler_setup.scene_test import _outputs_dir, _sanitize_for_filename + +KERNELS_BASE = "../../../../../../examples/a2a3/tensormap_and_ringbuffer/vector_example/kernels" + + +@scene_test(level=2, runtime="host_build_graph") +class TestA2A3ScopeStatsHostBuildGraph(SceneTestCase): + """Capture the executor scope plus one nested orchestration scope.""" + + CALLABLE = { + "orchestration": { + "source": f"{KERNELS_BASE}/orchestration/example_orchestration.cpp", + "function_name": "aicpu_orchestration_entry", + "signature": [D.IN, D.IN, D.OUT], + }, + "incores": [ + { + "func_id": 0, + "source": f"{KERNELS_BASE}/aiv/kernel_add.cpp", + "core_type": "aiv", + "signature": [D.IN, D.IN, D.OUT], + }, + { + "func_id": 1, + "source": f"{KERNELS_BASE}/aiv/kernel_add_scalar.cpp", + "core_type": "aiv", + "signature": [D.IN, D.OUT], + }, + { + "func_id": 2, + "source": f"{KERNELS_BASE}/aiv/kernel_mul.cpp", + "core_type": "aiv", + "signature": [D.IN, D.IN, D.OUT], + }, + ], + } + + CASES = [{"name": "nested", "platforms": ["a2a3sim", "a2a3"], "params": {}}] + + def generate_args(self, params): + size = 128 * 128 + return TaskArgsBuilder( + TensorArg("a", torch.full((size,), 2.0, dtype=torch.float32)), + TensorArg("b", torch.full((size,), 3.0, dtype=torch.float32)), + TensorArg("f", torch.zeros(size, dtype=torch.float32)), + ) + + def compute_golden(self, args, params): + args.f[:] = (args.a + args.b + 1) * (args.a + args.b + 2) + (args.a + args.b) + + def test_run(self, st_platform, st_worker, request): + run_marker = int(time.time()) + super().test_run(st_platform, st_worker, request) + if not request.config.getoption("--enable-scope-stats", default=False): + return + self._validate_artifact(run_marker) + + def _validate_artifact(self, run_marker): + safe_label = _sanitize_for_filename("TestA2A3ScopeStatsHostBuildGraph_nested") + matches = [p for p in _outputs_dir().glob(f"{safe_label}_*") if p.stat().st_mtime >= run_marker] + assert matches, "scope-stats run produced no output directory" + out_dir = max(matches, key=lambda p: p.stat().st_mtime) + path = out_dir / "scope_stats" / "scope_stats.jsonl" + assert path.exists(), f"host scope-stats did not produce {path}" + + lines = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + assert len(lines) == 5, f"expected metadata plus four boundaries, got {lines!r}" + meta, *records = lines + assert meta["version"] == 6 + assert meta["fatal"] is False + assert meta["dropped"] == 0 + assert meta["total"] == 4 + assert meta["task_window_max"][0] > 0 + assert meta["heap_max"][0] > 0 + assert meta["dep_pool_max"][0] == 0 + assert meta["tensormap_max"] > 0 + + assert [(record["depth"], record["phase"]) for record in records] == [ + (0, "begin"), + (1, "begin"), + (1, "end"), + (0, "end"), + ] + inner_begin, inner_end = records[1], records[2] + assert inner_begin["site"].startswith("example_orchestration.cpp:") + assert inner_end["site"] == inner_begin["site"] + assert inner_end["task_window_end"] - inner_begin["task_window_end"] == 4 + assert inner_end["heap_end"] > inner_begin["heap_end"] + + +if __name__ == "__main__": + SceneTestCase.run_module(__name__) diff --git a/tests/st/a5/host_build_graph/dfx/scope_stats/test_scope_stats.py b/tests/st/a5/host_build_graph/dfx/scope_stats/test_scope_stats.py new file mode 100644 index 0000000000..6a3dc46c41 --- /dev/null +++ b/tests/st/a5/host_build_graph/dfx/scope_stats/test_scope_stats.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Host-orchestration scope-stats capture and JSONL export.""" + +import json +import time + +import torch +from simpler.task_interface import ArgDirection as D + +from simpler_setup import SceneTestCase, TaskArgsBuilder, TensorArg, scene_test +from simpler_setup.scene_test import _outputs_dir, _sanitize_for_filename + +KERNELS_BASE = "../../../../../../examples/a5/tensormap_and_ringbuffer/vector_example/kernels" + + +@scene_test(level=2, runtime="host_build_graph") +class TestA5ScopeStatsHostBuildGraph(SceneTestCase): + """Capture the executor scope plus one nested orchestration scope.""" + + CALLABLE = { + "orchestration": { + "source": f"{KERNELS_BASE}/orchestration/example_orchestration.cpp", + "function_name": "aicpu_orchestration_entry", + "signature": [D.IN, D.IN, D.OUT], + }, + "incores": [ + { + "func_id": 0, + "source": f"{KERNELS_BASE}/aiv/kernel_add.cpp", + "core_type": "aiv", + "signature": [D.IN, D.IN, D.OUT], + }, + { + "func_id": 1, + "source": f"{KERNELS_BASE}/aiv/kernel_add_scalar.cpp", + "core_type": "aiv", + "signature": [D.IN, D.OUT], + }, + { + "func_id": 2, + "source": f"{KERNELS_BASE}/aiv/kernel_mul.cpp", + "core_type": "aiv", + "signature": [D.IN, D.IN, D.OUT], + }, + ], + } + + CASES = [{"name": "nested", "platforms": ["a5sim", "a5"], "params": {}}] + + def generate_args(self, params): + size = 128 * 128 + return TaskArgsBuilder( + TensorArg("a", torch.full((size,), 2.0, dtype=torch.float32)), + TensorArg("b", torch.full((size,), 3.0, dtype=torch.float32)), + TensorArg("f", torch.zeros(size, dtype=torch.float32)), + ) + + def compute_golden(self, args, params): + args.f[:] = (args.a + args.b + 1) * (args.a + args.b + 2) + (args.a + args.b) + + def test_run(self, st_platform, st_worker, request): + run_marker = int(time.time()) + super().test_run(st_platform, st_worker, request) + if not request.config.getoption("--enable-scope-stats", default=False): + return + self._validate_artifact(run_marker) + + def _validate_artifact(self, run_marker): + safe_label = _sanitize_for_filename("TestA5ScopeStatsHostBuildGraph_nested") + matches = [p for p in _outputs_dir().glob(f"{safe_label}_*") if p.stat().st_mtime >= run_marker] + assert matches, "scope-stats run produced no output directory" + out_dir = max(matches, key=lambda p: p.stat().st_mtime) + path = out_dir / "scope_stats" / "scope_stats.jsonl" + assert path.exists(), f"host scope-stats did not produce {path}" + + lines = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + assert len(lines) == 5, f"expected metadata plus four boundaries, got {lines!r}" + meta, *records = lines + assert meta["version"] == 6 + assert meta["fatal"] is False + assert meta["dropped"] == 0 + assert meta["total"] == 4 + assert meta["task_window_max"][0] > 0 + assert meta["heap_max"][0] > 0 + assert meta["dep_pool_max"][0] == 0 + assert meta["tensormap_max"] > 0 + + assert [(record["depth"], record["phase"]) for record in records] == [ + (0, "begin"), + (1, "begin"), + (1, "end"), + (0, "end"), + ] + inner_begin, inner_end = records[1], records[2] + assert inner_begin["site"].startswith("example_orchestration.cpp:") + assert inner_end["site"] == inner_begin["site"] + assert inner_end["task_window_end"] - inner_begin["task_window_end"] == 4 + assert inner_end["heap_end"] > inner_begin["heap_end"] + + +if __name__ == "__main__": + SceneTestCase.run_module(__name__) diff --git a/tests/ut/cpp/common/test_scope_stats_collector.cpp b/tests/ut/cpp/common/test_scope_stats_collector.cpp index ed47b845ea..8eb19ff0a2 100644 --- a/tests/ut/cpp/common/test_scope_stats_collector.cpp +++ b/tests/ut/cpp/common/test_scope_stats_collector.cpp @@ -10,6 +10,7 @@ */ #include "host/scope_stats_collector.h" +#include "host/scope_stats_host_capture.h" #include @@ -111,3 +112,71 @@ TEST(ScopeStatsCollectorTest, ReconcileRecoversUnflushedCurrentBuffer) { std::filesystem::remove_all(out_dir); collector.finalize(nullptr, test_free); } + +TEST(ScopeStatsHostCaptureTest, DisabledCaptureProducesNoRecords) { + ScopeStatsHostCapture capture; + capture.begin_capture(/*task_window_cap=*/128, /*heap_cap=*/4096, /*tensormap_cap=*/256); + capture.set_pending_site("disabled.cpp", 17); + capture.begin(0, 0, 0, 0, 0, 0, 0, 0); + capture.end(0, 0, 1, 0, 64, 0, 0, 1); + + EXPECT_TRUE(capture.records().empty()); + EXPECT_EQ(capture.write_jsonl("/tmp"), -3); +} + +TEST(ScopeStatsHostCaptureTest, NestedCapturePreservesSitesCapacitiesWrapsAndFatal) { + ScopeStatsHostCapture capture; + capture.set_enabled(true); + capture.begin_capture(/*task_window_cap=*/128, /*heap_cap=*/4096, /*tensormap_cap=*/256); + + capture.begin(0, 0, 0, 0, 0, 0, 0, 0); + capture.set_pending_site("/tmp/source/nested_scope.cpp", 42); + capture.begin(0, 0, 1, 0, 1024, 0, 0, 1); + capture.note_heap_wrap(SCOPE_STATS_HEAP_SIDE_ALLOC); + capture.note_heap_wrap(SCOPE_STATS_HEAP_SIDE_ALLOC); + capture.end(0, 0, 5, 0, 512, 0, 0, 3); + capture.on_fatal(); + capture.end(0, 0, 5, 0, 512, 0, 0, 3); + + const auto &records = capture.records(); + ASSERT_EQ(records.size(), 4u); + EXPECT_EQ(records[0].depth, 0); + EXPECT_EQ(records[1].depth, 1); + EXPECT_EQ(records[1].phase, SCOPE_STATS_PHASE_BEGIN); + EXPECT_EQ(records[2].phase, SCOPE_STATS_PHASE_END); + EXPECT_STREQ(records[1].site_file_basename, "nested_scope.cpp"); + EXPECT_EQ(records[1].site_line, 42); + EXPECT_EQ(records[2].heap_end, 2 * 4096 + 512); + + const auto &header = capture.header(); + EXPECT_EQ(header.task_window_cap[0], 128); + EXPECT_EQ(header.heap_cap[0], 4096u); + EXPECT_EQ(header.dep_pool_cap[0], 0); + EXPECT_EQ(header.tensormap_cap, 256); + EXPECT_EQ(header.fatal_latched, 1u); + + std::filesystem::path out_dir = + std::filesystem::temp_directory_path() / ("scope_stats_host_capture_test_" + std::to_string(::getpid())); + std::filesystem::remove_all(out_dir); + ASSERT_EQ(capture.write_jsonl(out_dir.c_str()), 0); + std::string jsonl = read_file(out_dir / "scope_stats" / "scope_stats.jsonl"); + EXPECT_NE(jsonl.find("\"fatal\": true"), std::string::npos); + EXPECT_NE(jsonl.find("\"total\": 4"), std::string::npos); + EXPECT_NE(jsonl.find("\"site\": \"nested_scope.cpp:42\""), std::string::npos); + std::filesystem::remove_all(out_dir); +} + +TEST(ScopeStatsHostCaptureTest, BeginCaptureClearsPriorRunState) { + ScopeStatsHostCapture capture; + capture.set_enabled(true); + capture.begin_capture(64, 2048, 32); + capture.begin(0, 0, 0, 0, 0, 0, 0, 0); + capture.end(0, 0, 1, 0, 128, 0, 0, 1); + ASSERT_EQ(capture.records().size(), 2u); + + capture.begin_capture(32, 1024, 16); + EXPECT_TRUE(capture.records().empty()); + EXPECT_EQ(capture.header().task_window_cap[0], 32); + EXPECT_EQ(capture.header().heap_cap[0], 1024u); + EXPECT_EQ(capture.header().fatal_latched, 0u); +}