From f367dfdf31e0376e3ab9d2e0b3751a919cc7fe13 Mon Sep 17 00:00:00 2001 From: zhusy54 Date: Wed, 2 Sep 2026 13:57:30 +0800 Subject: [PATCH] Update: switch A5 HBG single-lane scheduling to AICore - Move ordinary DAG dependency resolution and dispatch to resident AICore workers - Fall back whole valid MIX, SPMD, and sync-start runs to the legacy scheduler - Preserve scheduler timeout reporting across resident waits and refresh context after READY publication - Keep Graph replay on its explicit legacy compatibility executor - Keep profiling requests on the resident path with best-effort diagnostics - Add automatic lifecycle, DAG, root, and legacy-fallback coverage --- .../aicore/aicore_executor.cpp | 996 +++++++++++++----- .../aicore/aicore_legacy_executor.cpp | 303 ++++++ .../aicpu/aicore_lifecycle.cpp | 442 ++++++++ .../host_build_graph/aicpu/aicore_lifecycle.h | 52 + .../aicpu/aicore_scheduler_error.h | 54 + .../aicpu/aicore_scheduler_state.h | 48 + .../host_build_graph/aicpu/aicpu_executor.cpp | 638 ++++++----- .../aicpu/aicpu_legacy_executor.cpp | 505 +++++++++ .../runtime/host_build_graph/build_config.py | 11 +- .../host_build_graph/docs/profiling_levels.md | 7 + .../host_build_graph/host/runtime_maker.cpp | 355 +++++++ .../runtime/scheduler/scheduler_graph.h | 15 +- .../runtime/scheduler/scheduler_layout.h | 51 + .../runtime/scheduler/scheduler_ready.h | 7 +- .../runtime/scheduler/scheduler_types.h | 60 +- .../runtime/scheduler/scheduler_watchdog.h | 22 + .../chip_swimlane/kernels/aiv/kernel_noop.cpp | 11 +- .../orchestration/scheduler_phases_orch.cpp | 2 +- .../chip_swimlane/test_scheduler_phases.py | 48 +- .../kernels/orchestration/empty_orch.cpp | 23 + .../empty_lifecycle/test_empty_lifecycle.py | 44 + .../multi_core_dag/kernels/check_stress.cpp | 55 + .../orchestration/multi_core_dag_orch.cpp | 125 +++ .../multi_core_dag/test_multi_core_dag.py | 79 ++ .../paged_attention/test_paged_attention.py | 12 +- .../single_core_dag/kernels/check_dag.cpp | 47 + .../orchestration/single_core_dag_orch.cpp | 137 +++ .../single_core_dag/test_single_core_dag.py | 91 ++ .../orchestration/single_aic_root_orch.cpp | 27 + .../single_aic_spmd_fallback_orch.cpp | 28 + .../orchestration/single_aiv_root_orch.cpp | 29 + .../single_root/test_single_root.py | 115 ++ tests/ut/cpp/CMakeLists.txt | 3 + .../cpp/a5/test_hbg_scheduler_contracts.cpp | 55 + 34 files changed, 3879 insertions(+), 618 deletions(-) create mode 100644 src/a5/runtime/host_build_graph/aicore/aicore_legacy_executor.cpp create mode 100644 src/a5/runtime/host_build_graph/aicpu/aicore_lifecycle.cpp create mode 100644 src/a5/runtime/host_build_graph/aicpu/aicore_lifecycle.h create mode 100644 src/a5/runtime/host_build_graph/aicpu/aicore_scheduler_error.h create mode 100644 src/a5/runtime/host_build_graph/aicpu/aicore_scheduler_state.h create mode 100644 src/a5/runtime/host_build_graph/aicpu/aicpu_legacy_executor.cpp create mode 100644 src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_layout.h create mode 100644 src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_watchdog.h create mode 100644 tests/st/a5/host_build_graph/empty_lifecycle/kernels/orchestration/empty_orch.cpp create mode 100644 tests/st/a5/host_build_graph/empty_lifecycle/test_empty_lifecycle.py create mode 100644 tests/st/a5/host_build_graph/multi_core_dag/kernels/check_stress.cpp create mode 100644 tests/st/a5/host_build_graph/multi_core_dag/kernels/orchestration/multi_core_dag_orch.cpp create mode 100644 tests/st/a5/host_build_graph/multi_core_dag/test_multi_core_dag.py create mode 100644 tests/st/a5/host_build_graph/single_core_dag/kernels/check_dag.cpp create mode 100644 tests/st/a5/host_build_graph/single_core_dag/kernels/orchestration/single_core_dag_orch.cpp create mode 100644 tests/st/a5/host_build_graph/single_core_dag/test_single_core_dag.py create mode 100644 tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aic_root_orch.cpp create mode 100644 tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aic_spmd_fallback_orch.cpp create mode 100644 tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aiv_root_orch.cpp create mode 100644 tests/st/a5/host_build_graph/single_root/test_single_root.py diff --git a/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp b/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp index f04e55d228..2315e7df62 100644 --- a/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp +++ b/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp @@ -11,294 +11,786 @@ #include "aicore/aicore.h" #include "aicore/aicore_profiling_state.h" -#include "aicore/chip_swimlane_collector_aicore.h" -#include "aicore/pmu_collector_aicore.h" -#include "common/chip_swimlane_profiling.h" -#include "common/platform_config.h" // Register-based communication +// Cluster-local dependency scheduling uses one device-side protocol. +#include "scheduler/scheduler_dispatch.h" +#include "common/platform_config.h" #include "dispatch_payload.h" #include "runtime.h" #include "scheduler/scheduler_memory.h" #include "scheduler/scheduler_ready.h" +#include "scheduler/scheduler_watchdog.h" -/** - * Unified function pointer type for kernel dispatch - * - * All kernels follow the same signature: void kernel(__gm__ int64_t* args) - * This enables simple, switch-free dispatch. - */ typedef void (*UnifiedKernelFunc)(__gm__ int64_t *); -/** - * Execute task from DispatchPayload. - * - * Reads function_bin_addr and args from the dispatch payload. - * - * @param payload Pointer to DispatchPayload in global memory - */ -__aicore__ __attribute__((always_inline)) static void execute_task(__gm__ DispatchPayload *payload) { - if (payload == nullptr || payload->function_bin_addr == 0) { - return; +__aicore__ void legacy_aicore_execute(__gm__ Runtime *runtime, int block_idx, CoreType core_type); + +namespace { + +constexpr uint32_t kInitialBackoffIterations = 8; +constexpr uint32_t kMaximumBackoffIterations = 128; +constexpr uint32_t kSchedulerErrorPollInterval = 64; + +static_assert(SCHEDULER_CALLABLE_CAPACITY == RUNTIME_MAX_FUNC_ID, "AICore ready scheduler layout mismatch"); + +struct SchedulerWorkerStats { + uint64_t bootstrap_task_count{0}; + SchedulerReadyStats ready{}; + uint64_t executed_task_count{0}; + uint64_t task_state_poll_count{0}; + SchedulerWakeStats wake{}; + SchedulerCompletionStats completion{}; + uint64_t idle_iteration_count{0}; + uint64_t backoff_cycles{0}; + uint64_t payload_cycles{0}; + uint64_t kernel_cycles{0}; + uint64_t completion_enqueue_cycles{0}; + uint64_t bootstrap_start_cycles{0}; + uint64_t bootstrap_scan_end_cycles{0}; + uint64_t bootstrap_end_cycles{0}; + uint64_t target_bootstrap_start_cycles{0}; + uint64_t target_bootstrap_end_cycles{0}; + uint64_t bootstrap_target_cycles[SCHEDULER_CORE_TYPE_COUNT]{}; + uint64_t bootstrap_ready_claim_cycles[SCHEDULER_CORE_TYPE_COUNT]{}; + uint64_t bootstrap_slot_fill_cycles[SCHEDULER_CORE_TYPE_COUNT]{}; + uint64_t drain_start_cycles{0}; + uint64_t drain_end_cycles{0}; + uint64_t exit_wait_start_cycles{0}; + uint64_t exit_observed_cycles{0}; + uint64_t final_stats_publish_start_cycles{0}; + uint64_t final_stats_publish_end_cycles{0}; + uint64_t exit_ack_publish_cycles{0}; +}; + +struct SchedulerInterTaskTiming { + uint64_t completion_service_cycles{0}; + uint64_t dispatch_cycles[SCHEDULER_CORE_TYPE_COUNT]{}; + SchedulerCompletionServiceTiming completion{}; + uint64_t gang_service_cycles{0}; + SchedulerNormalDispatchTiming dispatch{}; + uint64_t ready_poll_cycles{0}; + uint64_t backoff_cycles{0}; + + __aicore__ void reset() { *this = {}; } +}; + +__aicore__ __attribute__((always_inline)) void publish_scheduler_tail_trace( + __gm__ SchedulerWorkerContext *context, uint64_t start_cycles, uint64_t end_cycles, + const SchedulerInterTaskTiming &timing +) { + __gm__ SchedulerTailTrace *trace = &context->scheduler_tail_trace; + trace->start_cycles = start_cycles; + trace->end_cycles = end_cycles; + trace->completion_scan_cycles = timing.completion.scan_cycles; + trace->completion_consume_cycles = timing.completion.consume_cycles; + trace->completion_resolve_cycles = timing.completion.resolve_cycles; + trace->completion_ready_publish_cycles = timing.completion.ready_publish_cycles; + trace->completion_refill_cycles = timing.completion.refill_cycles; + trace->completion_finalize_cycles = timing.completion.finalize_cycles; + trace->gang_service_cycles = timing.gang_service_cycles; + for (uint32_t type = 0; type < SCHEDULER_CORE_TYPE_COUNT; ++type) { + trace->dispatch_probe_cycles[type] = timing.dispatch.probe_cycles[type]; + trace->dispatch_claim_cycles[type] = timing.dispatch.claim_cycles[type]; + trace->dispatch_prepare_cycles[type] = timing.dispatch.prepare_cycles[type]; + trace->dispatch_materialize_cycles[type] = timing.dispatch.materialize_cycles[type]; + trace->dispatch_publish_cycles[type] = timing.dispatch.publish_cycles[type]; } + trace->ready_poll_cycles = timing.ready_poll_cycles; + trace->backoff_cycles = timing.backoff_cycles; + scheduler_publish_cache_line(&trace->start_cycles); + scheduler_publish_cache_line(&trace->dispatch_materialize_cycles[0]); + scheduler_gm_publish(trace->valid, UINT64_C(1)); +} + +struct SchedulerExecutionRecord { + int64_t task_id{SCHEDULER_TASK_ID_INVALID}; + uint64_t claim_worker_id{0}; + uint64_t claim_start_cycles{0}; + uint64_t claim_end_cycles{0}; + SchedulerReadySource ready_source{SchedulerReadySource::LOCAL}; +}; +__aicore__ __attribute__((always_inline)) void execute_task(__gm__ DispatchPayload *payload) { + if (payload == nullptr || payload->function_bin_addr == 0) return; UnifiedKernelFunc kernel = (UnifiedKernelFunc)payload->function_bin_addr; kernel(reinterpret_cast<__gm__ int64_t *>(payload->args)); OUT_OF_ORDER_STORE_BARRIER(); } -/** - * AICore main execution loop - * - * Implements the AICPU-AICore register-based dispatch protocol: - * 1. Report physical core ID and core type, signal aicore_done (no AICPU wait) - * 2. Wait for the AICPU to open our register window (DATA_MAIN_BASE != 0) - * 3. Cache per-core DispatchPayload pointer from my_hank->task - * 4. Poll DATA_MAIN_BASE register for task dispatch until exit signal - * - * AICore reports on launch; the AICPU writes &s_payload_per_core[i] to - * my_hank->task and then opens the register window (DATA_MAIN_BASE = IDLE), which - * is itself the acknowledgement. AICore caches this pointer and reads - * function_bin_addr + args pointer from it on each dispatch. reg_val is a - * monotonically increasing task ID used only for dispatch signaling and - * ACK/FIN protocol. - * - * Profiling state (enable flag, chip swimlane rotation channel) is published into the platform - * via set_aicore_profiling_flag / set_chip_swimlane_aicore_head_slot at kernel entry — - * this routine reads it through the matching getters, so neither Handshake - * nor this signature carry profiling fields. - * - * @param runtime Pointer to Runtime in global memory - * @param block_idx Block index (core ID) - * @param core_type Core type (AIC or AIV) - */ -__aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, int block_idx, CoreType core_type) { - __gm__ Handshake *my_hank = (__gm__ Handshake *)(&runtime->workers[block_idx]); - - // Phase 1: report physical core ID + core type and signal done in one write, - // with no wait for the AICPU — both fields are self-known. The AICPU opens - // this core's register window only after it observes aicore_done, so a single - // report suffices. The host clears aicore_done before this kernel launches, - // so the value the AICPU reads is this run's report, never a stale prior one. - my_hank->physical_core_id = get_physical_core_id(); - my_hank->core_type = core_type; - OUT_OF_ORDER_STORE_BARRIER(); - my_hank->aicore_done = block_idx + 1; // Signal ready (use block_idx + 1 to avoid 0) - dcci(my_hank, SINGLE_CACHE_LINE, CACHELINE_OUT); - - // Phase 2: Wait for the AICPU to open our register window. A kernel launch - // resets DATA_MAIN_BASE to 0 (verified on a2a3 silicon); the AICPU writes - // DATA_MAIN_BASE = AICPU_IDLE_TASK_ID (non-zero) as it opens FAST_PATH, so a - // non-zero read means the window is open and reads/writes are valid. The - // AICPU runs assign_cores_to_threads (µs) between opening the window and the - // first dispatch, so this IDLE is observed long before any task_id lands — - // the poll cannot miss it and mistake a later task for the reset value. - // Window-open is the sync point for everything the AICPU publishes (task - // pointer, swimlane head): the AICPU writes those before opening the window. - while (read_reg(RegId::DATA_MAIN_BASE) == 0) { - SPIN_WAIT_HINT(); +__aicore__ __attribute__((always_inline)) bool +should_commit_scheduler_trace(__gm__ void *, __gm__ SchedulerWorkerContext *, __gm__ SchedulerDispatchSlot *slot) { + return slot->gang == 0; +} + +__aicore__ __attribute__((always_inline)) void local_backoff(uint32_t iterations) { + for (volatile uint32_t spin = 0; spin < iterations; ++spin) {} +} + +__aicore__ __attribute__((always_inline)) void +publish_worker_stats(__gm__ SchedulerWorkerContext *context, const SchedulerWorkerStats &stats) { + context->bootstrap_task_count = stats.bootstrap_task_count; + context->ready_enqueue_count = stats.ready.enqueue_count; + context->ready_batch_count = stats.ready.batch_count; + context->ready_pop_count = stats.ready.pop_count; + context->ready_steal_count = stats.ready.steal_count; + context->ready_cas_retry_count = stats.ready.cas_retry_count; + context->ready_contention_giveup_count = stats.ready.contention_giveup_count; + context->executed_task_count = stats.executed_task_count; + context->task_state_poll_count = stats.task_state_poll_count; + context->fanin_state_load_count = stats.wake.fanin_state_load_count; + context->wake_register_count = stats.wake.wake_register_count; + context->idle_iteration_count = stats.idle_iteration_count; + context->backoff_cycles = stats.backoff_cycles; + scheduler_publish_cache_line(&context->bootstrap_task_count); + + context->wake_cas_retry_count = stats.wake.wake_cas_retry_count; + context->wake_closed_retry_count = stats.wake.wake_closed_retry_count; + context->wake_migrate_count = stats.wake.wake_migrate_count; + context->wake_close_count = stats.wake.wake_close_count; + context->completion_enqueue_count = stats.completion.enqueue_count; + context->completion_resolve_count = stats.completion.resolve_count; + context->ready_to_kernel_cycles = stats.completion.ready_to_kernel_cycles; + context->ready_to_kernel_max_cycles = stats.completion.ready_to_kernel_max_cycles; + context->payload_cycles = stats.payload_cycles; + context->kernel_cycles = stats.kernel_cycles; + scheduler_publish_cache_line(&context->wake_cas_retry_count); + + context->completion_enqueue_cycles = stats.completion_enqueue_cycles; + context->bootstrap_start_cycles = stats.bootstrap_start_cycles; + context->bootstrap_end_cycles = stats.bootstrap_end_cycles; + context->drain_start_cycles = stats.drain_start_cycles; + context->drain_end_cycles = stats.drain_end_cycles; + context->exit_wait_start_cycles = stats.exit_wait_start_cycles; + context->exit_observed_cycles = stats.exit_observed_cycles; + context->final_stats_publish_start_cycles = stats.final_stats_publish_start_cycles; + context->final_stats_publish_end_cycles = stats.final_stats_publish_end_cycles; + context->exit_ack_publish_cycles = stats.exit_ack_publish_cycles; + context->bootstrap_slot_fill_aic_cycles = stats.bootstrap_slot_fill_cycles[0]; + context->bootstrap_slot_fill_aiv_cycles = stats.bootstrap_slot_fill_cycles[1]; + scheduler_publish_cache_line(&context->completion_enqueue_cycles); + scheduler_publish_cache_line(&context->bootstrap_slot_fill_aic_cycles); + + scheduler_gm_publish(context->bootstrap_scan_end_cycles, stats.bootstrap_scan_end_cycles); + scheduler_gm_publish(context->target_bootstrap_start_cycles, stats.target_bootstrap_start_cycles); + scheduler_gm_publish(context->target_bootstrap_end_cycles, stats.target_bootstrap_end_cycles); + scheduler_gm_publish(context->bootstrap_target_aic_cycles, stats.bootstrap_target_cycles[0]); + scheduler_gm_publish(context->bootstrap_target_aiv_cycles, stats.bootstrap_target_cycles[1]); + scheduler_gm_publish(context->bootstrap_ready_claim_aic_cycles, stats.bootstrap_ready_claim_cycles[0]); + scheduler_gm_publish(context->bootstrap_ready_claim_aiv_cycles, stats.bootstrap_ready_claim_cycles[1]); +} + +__aicore__ __attribute__((always_inline)) void commit_task_trace( + __gm__ void *scheduler_state_base, __gm__ SchedulerWorkerContext *context, const SchedulerExecutionRecord &record, + uint64_t ready_scan_start, uint64_t ready_observe, uint64_t kernel_start, uint64_t kernel_end, + uint64_t completion_end, uint64_t bookkeeping_end, uint64_t previous_trace_commit_end, uint64_t aicore_entry_cycles, + uint64_t handshake_publish_cycles, uint64_t register_release_cycles, uint64_t descriptor_cache_observed_cycles, + uint64_t completion_id, uint64_t completion_inbox_index, const SchedulerInterTaskTiming &inter_task_timing +) { + __gm__ SchedulerTaskTrace *cells = + scheduler_state_at(scheduler_state_base, context->trace_cells_offset); + __gm__ SchedulerTaskTrace *trace = &cells[record.task_id]; + trace->ready_source = static_cast(record.ready_source); + trace->worker_id = context->worker_index; + trace->task_id = static_cast(record.task_id); + trace->claim_worker_id = record.claim_worker_id; + trace->claim_start_cycles = record.claim_start_cycles; + trace->claim_end_cycles = record.claim_end_cycles; + trace->previous_trace_commit_end_cycles = previous_trace_commit_end; + trace->kernel_start_cycles = kernel_start; + trace->kernel_end_cycles = kernel_end; + trace->completion_end_cycles = completion_end; + trace->ready_scan_start_cycles = ready_scan_start; + trace->ready_observe_cycles = ready_observe; + trace->completion_bookkeeping_end_cycles = bookkeeping_end; + trace->completion_id = completion_id; + trace->completion_inbox_index = completion_inbox_index; + scheduler_observe_cache_line(&trace->ready_transition_cycles); + trace->inter_task_completion_service_cycles = inter_task_timing.completion_service_cycles; + trace->inter_task_dispatch_aic_cycles = inter_task_timing.dispatch_cycles[0]; + trace->inter_task_dispatch_aiv_cycles = inter_task_timing.dispatch_cycles[1]; + trace->inter_task_ready_poll_cycles = inter_task_timing.ready_poll_cycles; + trace->inter_task_backoff_cycles = inter_task_timing.backoff_cycles; + trace->inter_task_completion_scan_cycles = inter_task_timing.completion.scan_cycles; + trace->inter_task_completion_consume_cycles = inter_task_timing.completion.consume_cycles; + trace->inter_task_completion_resolve_cycles = inter_task_timing.completion.resolve_cycles; + trace->inter_task_completion_ready_publish_cycles = inter_task_timing.completion.ready_publish_cycles; + trace->inter_task_completion_refill_cycles = inter_task_timing.completion.refill_cycles; + trace->inter_task_completion_finalize_cycles = inter_task_timing.completion.finalize_cycles; + trace->inter_task_gang_service_cycles = inter_task_timing.gang_service_cycles; + for (uint32_t type = 0; type < SCHEDULER_CORE_TYPE_COUNT; ++type) { + trace->inter_task_dispatch_probe_cycles[type] = inter_task_timing.dispatch.probe_cycles[type]; + trace->inter_task_dispatch_claim_cycles[type] = inter_task_timing.dispatch.claim_cycles[type]; + trace->inter_task_dispatch_prepare_cycles[type] = inter_task_timing.dispatch.prepare_cycles[type]; + trace->inter_task_dispatch_materialize_cycles[type] = inter_task_timing.dispatch.materialize_cycles[type]; + trace->inter_task_dispatch_publish_cycles[type] = inter_task_timing.dispatch.publish_cycles[type]; + } + if (previous_trace_commit_end == 0) { + trace->aicore_entry_cycles = aicore_entry_cycles; + trace->handshake_publish_cycles = handshake_publish_cycles; + trace->register_release_cycles = register_release_cycles; + trace->descriptor_cache_observed_cycles = descriptor_cache_observed_cycles; + scheduler_publish_cache_line(&trace->register_release_cycles); } - // Report initial idle status via register (FAST_PATH is now open). - write_reg(RegId::COND, AICORE_IDLE_VALUE); - - // The AICPU writes task after observing our report (so our CACHELINE_OUT flush - // above cannot clobber it) and before opening the window; dcci to read its - // fresh value here. - dcci(my_hank, SINGLE_CACHE_LINE); - __gm__ DispatchPayload *payload = reinterpret_cast<__gm__ DispatchPayload *>(my_hank->task); - - uint32_t enable_profiling_flag = get_aicore_profiling_flag(); - bool chip_swimlane_enabled = SIMPLER_GET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_CHIP_SWIMLANE); - bool dump_args_enabled = SIMPLER_GET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_DUMP_ARGS); - bool pmu_enabled = SIMPLER_GET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_PMU); - - // Per-core ChipSwimlaneActiveHead channel. AICPU completes - // `chip_swimlane_aicpu_init` (in pre_handshake_init) before any thread opens a - // register window in `handshake_partition`, and Phase 2 above has already - // observed our window open, so the rotation-table slot is populated and the - // first deref is safe here — off the dispatch→start critical path. - __gm__ ChipSwimlaneActiveHead *chip_swimlane_head = - chip_swimlane_enabled ? get_chip_swimlane_aicore_head() : nullptr; - // cached_buf_seq must start != AICPU's initial head.current_buf_seq (0) - // so the first reservation observes a mismatch and loads the buffer ptr. - ChipSwimlaneAicoreLocalState chip_swimlane_local = {nullptr, UINT32_MAX, 0}; - - // Phase 4: Main execution loop - poll register for tasks until exit signal - // Register encoding: AICPU_IDLE_TASK_ID=idle, task_id=task, AICORE_EXIT_SIGNAL=exit - uint32_t reg_val = AICPU_IDLE_TASK_ID; - uint32_t last_reg_val = AICPU_IDLE_TASK_ID; - bool exiting = false; + scheduler_publish_cache_line(&trace->kernel_start_cycles); + scheduler_publish_cache_line(&trace->ready_transition_cycles); + trace->valid = 1; + scheduler_publish_cache_line(trace); +} + +__aicore__ __attribute__((always_inline)) void commit_task_timing_trace( + __gm__ void *scheduler_state_base, __gm__ SchedulerWorkerContext *context, int64_t task_id, uint64_t kernel_start, + uint64_t kernel_end +) { + __gm__ SchedulerTaskTrace *cells = + scheduler_state_at(scheduler_state_base, context->trace_cells_offset); + __gm__ SchedulerTaskTrace *trace = &cells[task_id]; + trace->kernel_start_cycles = kernel_start; + trace->kernel_end_cycles = kernel_end; + scheduler_publish_cache_line(&trace->kernel_start_cycles); +} + +__aicore__ bool bootstrap_ready_graph( + const SchedulerGraphView &graph, __gm__ void *scheduler_state_base, __gm__ SchedulerWorkerContext *resolver, + __gm__ SchedulerRunControl *run_control, uint64_t resolver_count, SchedulerWorkerStats *stats, bool trace_enabled, + SchedulerDeferredAivQueue *deferred_aiv, __gm__ SchedulerReadyOwnerState *ready_owner +) { + if (resolver_count == 0 || resolver->inbox_index >= resolver_count || ready_owner == nullptr) return false; + if (trace_enabled) stats->bootstrap_start_cycles = scheduler_cycles(); + SchedulerReadyBatch batches[SCHEDULER_CORE_TYPE_COUNT]{}; + uint64_t tasks_per_resolver = graph.task_count / resolver_count; + uint64_t remainder = graph.task_count % resolver_count; + uint64_t task_begin = resolver->inbox_index * tasks_per_resolver + + (resolver->inbox_index < remainder ? resolver->inbox_index : remainder); + uint64_t task_end = task_begin + tasks_per_resolver + (resolver->inbox_index < remainder ? 1 : 0); + for (uint64_t task_id = task_begin; task_id < task_end; ++task_id) { + __gm__ SchedulerTaskMetadata *metadata = + scheduler_task_metadata_at(scheduler_state_base, resolver, static_cast(task_id)); + scheduler_observe_cache_line(metadata); + if (!scheduler_task_is_executable(metadata->flags)) continue; + SchedulerRouteResult route = + scheduler_task_has_fanin(metadata->flags) ? + scheduler_bootstrap_route_task( + graph, scheduler_state_base, resolver, run_control, static_cast(task_id), &stats->wake + ) : + SchedulerRouteResult::READY_TO_ENQUEUE; + if (route == SchedulerRouteResult::ERROR) return false; + if (route == SchedulerRouteResult::READY_TO_ENQUEUE && + !scheduler_bootstrap_ready_batch_append( + scheduler_state_base, resolver, static_cast(task_id), + &batches[scheduler_metadata_core_type_index(scheduler_metadata_single_subtask_slot(metadata + ->active_mask))], + &stats->ready, trace_enabled + )) { + return false; + } + ++stats->bootstrap_task_count; + } + scheduler_cache_barrier(); + uint64_t ready_types = 0; + for (uint32_t type = 0; type < SCHEDULER_CORE_TYPE_COUNT; ++type) { + if (!scheduler_bootstrap_ready_batch_publish( + scheduler_state_base, resolver, type, resolver->inbox_index, &batches[type], &stats->ready, &ready_types + )) + return false; + } + __gm__ SchedulerReadyDirectory *ready_directory = scheduler_ready_directory_at(scheduler_state_base, resolver); + scheduler_gm_store(ready_directory->bootstrap_ready_types[resolver->inbox_index], ready_types); + if (trace_enabled) stats->bootstrap_scan_end_cycles = scheduler_cycles(); + + uint64_t arrived = scheduler_gm_fetch_add(run_control->bootstrap_scan_arrived_count, UINT64_C(1)) + 1; + if (arrived == resolver_count) { + scheduler_bootstrap_ready_directory_publish(scheduler_state_base, resolver, resolver_count); + scheduler_gm_publish(run_control->bootstrap_scan_complete, UINT64_C(1)); + } else { + uint32_t barrier_backoff = kInitialBackoffIterations; + const uint64_t barrier_start = scheduler_cycles(); + while (scheduler_gm_query(run_control->bootstrap_scan_complete) == 0) { + if (scheduler_gm_query(run_control->scheduler_error) != 0) return false; + if (scheduler_watchdog_expired( + barrier_start, scheduler_cycles(), scheduler_gm_query(run_control->scheduler_timeout_cycles) + )) { + scheduler_record_error( + run_control, SCHEDULER_TASK_ID_INVALID, SchedulerGraphResult::TIMEOUT, &graph, resolver, + SchedulerErrorSite::BOOTSTRAP_SCAN_TIMEOUT + ); + return false; + } + local_backoff(barrier_backoff); + if (barrier_backoff < kMaximumBackoffIterations) barrier_backoff <<= 1; + } + } + for (uint32_t type = 0; type < SCHEDULER_CORE_TYPE_COUNT; ++type) { + scheduler_gm_store( + ready_owner->queues[type].advertised, (ready_types & (UINT64_C(1) << type)) != 0 ? UINT64_C(1) : UINT64_C(0) + ); + } + if (trace_enabled) stats->target_bootstrap_start_cycles = scheduler_cycles(); + for (uint32_t cluster_lane = 0; cluster_lane < PLATFORM_CORES_PER_BLOCKDIM; ++cluster_lane) { + const uint64_t worker_id = resolver->cluster_worker_ids[cluster_lane]; + uint64_t target_start = trace_enabled ? scheduler_cycles() : 0; + __gm__ SchedulerWorkerContext *target = scheduler_worker_context_at(scheduler_state_base, resolver, worker_id); + scheduler_observe_cache_line(target); + scheduler_observe_cache_line(&target->task_metadata_offset); + if (target->active == 0) continue; + uint32_t type = scheduler_core_type_index(target->core_type); + for (uint32_t slot_index = 0; slot_index < SCHEDULER_PENDING_SLOT_COUNT; ++slot_index) { + __gm__ SchedulerDispatchSlot *slot = + scheduler_dispatch_slot_at(scheduler_state_base, resolver, worker_id, slot_index); + scheduler_observe_cache_line(slot); + scheduler_initialize_free_slot(slot); + } + scheduler_gm_publish(target->bootstrap_done, UINT64_C(1)); + if (trace_enabled) stats->bootstrap_target_cycles[type] += scheduler_cycles() - target_start; + } + if (trace_enabled) stats->target_bootstrap_end_cycles = scheduler_cycles(); + + // Prepare the first executable wave while the sole DMB launch gate is + // still closed. + uint64_t ready_victim_cursors[SCHEDULER_CORE_TYPE_COUNT]{ + (resolver->inbox_index + 1) % resolver_count, + (resolver->inbox_index + 1) % resolver_count, + }; + bool fill_failed = false; + (void)scheduler_fill_cluster_normal_slots( + graph, scheduler_state_base, resolver, run_control, ready_victim_cursors, &stats->ready, trace_enabled, 0, + nullptr, deferred_aiv, ready_owner, &fill_failed + ); + if (fill_failed) return false; + // No peer can make progress while the launch gate is closed. Materialize + // every Resolver reservation now so local deferred state and FILLING slots + // never cross the bootstrap boundary. + while (deferred_aiv != nullptr && deferred_aiv->count != 0) { + if (!scheduler_publish_deferred_aiv_to_resolver( + graph, scheduler_state_base, resolver, run_control, deferred_aiv, trace_enabled, nullptr + )) + return false; + } + + // This completion publication is observed by AICPU before it emits the + // one and only DMB release. Resolvers do not wait on another barrier. + arrived = scheduler_gm_fetch_add(run_control->bootstrap_arrived_count, UINT64_C(1)) + 1; + if (arrived == resolver_count) scheduler_gm_publish(run_control->bootstrap_complete, UINT64_C(1)); + if (trace_enabled) stats->bootstrap_end_cycles = scheduler_cycles(); + return true; +} + +__aicore__ bool run_ready_dispatch_loop( + const SchedulerGraphView &graph, __gm__ void *scheduler_state_base, __gm__ SchedulerWorkerContext *context, + __gm__ SchedulerRunControl *run_control, SchedulerWorkerStats *stats, bool trace_enabled, + uint64_t aicore_entry_cycles, uint64_t handshake_publish_cycles, uint64_t register_release_cycles, + uint64_t descriptor_cache_observed_cycles, SchedulerDeferredAivQueue *deferred_aiv +) { + uint64_t resolver_count = scheduler_gm_query(run_control->resolver_count); + bool resolver_worker = context->is_resolver != 0; + if (resolver_count == 0) return false; + __gm__ SchedulerReadyOwnerState *ready_owner = nullptr; + if (resolver_worker) { + ready_owner = scheduler_ready_owner_state_at(scheduler_state_base, context); + } + uint64_t ready_victim_cursors[SCHEDULER_CORE_TYPE_COUNT]{ + resolver_worker ? (context->inbox_index + 1) % resolver_count : 0, + resolver_worker ? (context->inbox_index + 1) % resolver_count : 0, + }; + uint64_t seen_publication[SCHEDULER_PENDING_SLOT_COUNT]{}; + uint64_t previous_trace_commit_end = 0; + uint64_t inter_task_start_cycles = register_release_cycles; + uint32_t scan_start = 0; + uint32_t backoff_iterations = kInitialBackoffIterations; + uint32_t scheduler_error_poll_count = 0; + SchedulerInterTaskTiming inter_task_timing{}; while (true) { - reg_val = static_cast(read_reg(RegId::DATA_MAIN_BASE)); - if (reg_val == AICORE_EXIT_SIGNAL) { - // Signal exit acknowledgment to AICPU - write_reg(RegId::COND, AICORE_EXITED_VALUE); + if (static_cast(read_reg(RegId::DATA_MAIN_BASE)) == AICORE_EXIT_SIGNAL) { + if (trace_enabled) { + stats->exit_observed_cycles = get_sys_cnt_aicore(); + publish_scheduler_tail_trace( + context, inter_task_start_cycles, stats->exit_observed_cycles, inter_task_timing + ); + } break; } + if (++scheduler_error_poll_count == kSchedulerErrorPollInterval) { + scheduler_error_poll_count = 0; + if (scheduler_gm_query(run_control->scheduler_error) != 0) return false; + } - // Execute task if new (reg_val encoding: AICPU_IDLE_TASK_ID=idle, task_id=task) - if (reg_val == AICPU_IDLE_TASK_ID || reg_val == last_reg_val) { - SPIN_WAIT_HINT(); - continue; + bool scheduler_progress = false; + if (resolver_worker && !scheduler_ready_owner_maintain(scheduler_state_base, context, ready_owner)) { + scheduler_record_error( + run_control, SCHEDULER_TASK_ID_INVALID, SchedulerGraphResult::INVALID_ARGUMENTS, &graph, context, + SchedulerErrorSite::READY_OWNER_MAINTENANCE_FAILED + ); + return false; + } + uint32_t preferred_ready_slot = UINT32_MAX; + if (resolver_worker && deferred_aiv != nullptr && deferred_aiv->count != 0) { + const uint32_t deferred_before = deferred_aiv->count; + if (!scheduler_drain_deferred_aiv_to_peer( + graph, scheduler_state_base, context, run_control, deferred_aiv, &stats->wake, &stats->ready, + &stats->completion, trace_enabled, nullptr, nullptr, ready_owner + )) + return false; + scheduler_progress = deferred_aiv->count != deferred_before; + if (deferred_aiv->count != 0 && !scheduler_publish_deferred_aiv_to_resolver( + graph, scheduler_state_base, context, run_control, deferred_aiv, + trace_enabled, &preferred_ready_slot + )) + return false; + } + if (resolver_worker && preferred_ready_slot == UINT32_MAX) { + uint64_t operation_start = trace_enabled ? get_sys_cnt_aicore() : 0; + uint64_t direct_refilled_slot_mask = 0; + SchedulerCompletionServiceTiming completion_timing{}; + const bool completion_progress = scheduler_service_cluster_completions( + graph, scheduler_state_base, context, run_control, &stats->wake, &stats->ready, &stats->completion, + ready_victim_cursors, trace_enabled, &direct_refilled_slot_mask, + trace_enabled ? &completion_timing : nullptr, ready_owner + ); + if (trace_enabled) { + uint64_t completion_total = get_sys_cnt_aicore() - operation_start; + uint64_t completion_detail = completion_timing.consume_cycles + completion_timing.resolve_cycles + + completion_timing.ready_publish_cycles + completion_timing.refill_cycles + + completion_timing.finalize_cycles; + completion_timing.scan_cycles += + completion_total > completion_detail ? completion_total - completion_detail : 0; + inter_task_timing.completion_service_cycles += completion_total; + inter_task_timing.completion.scan_cycles += completion_timing.scan_cycles; + inter_task_timing.completion.consume_cycles += completion_timing.consume_cycles; + inter_task_timing.completion.resolve_cycles += completion_timing.resolve_cycles; + inter_task_timing.completion.ready_publish_cycles += completion_timing.ready_publish_cycles; + inter_task_timing.completion.refill_cycles += completion_timing.refill_cycles; + inter_task_timing.completion.finalize_cycles += completion_timing.finalize_cycles; + } + scheduler_progress = completion_progress; + operation_start = trace_enabled ? get_sys_cnt_aicore() : 0; + SchedulerNormalDispatchTiming dispatch_timing{}; + bool fill_failed = false; + const bool dispatch_progress = scheduler_fill_cluster_normal_slots( + graph, scheduler_state_base, context, run_control, ready_victim_cursors, &stats->ready, trace_enabled, + direct_refilled_slot_mask, trace_enabled ? &dispatch_timing : nullptr, deferred_aiv, ready_owner, + &fill_failed + ); + scheduler_progress = dispatch_progress || scheduler_progress; + if (fill_failed) return false; + if (trace_enabled) { + inter_task_timing.dispatch_cycles[0] += get_sys_cnt_aicore() - operation_start; + for (uint32_t type = 0; type < SCHEDULER_CORE_TYPE_COUNT; ++type) { + inter_task_timing.dispatch.probe_cycles[type] += dispatch_timing.probe_cycles[type]; + inter_task_timing.dispatch.claim_cycles[type] += dispatch_timing.claim_cycles[type]; + inter_task_timing.dispatch.prepare_cycles[type] += dispatch_timing.prepare_cycles[type]; + inter_task_timing.dispatch.materialize_cycles[type] += dispatch_timing.materialize_cycles[type]; + inter_task_timing.dispatch.publish_cycles[type] += dispatch_timing.publish_cycles[type]; + } + } + if (deferred_aiv != nullptr && deferred_aiv->count != 0) { + const uint32_t deferred_before = deferred_aiv->count; + if (!scheduler_drain_deferred_aiv_to_peer( + graph, scheduler_state_base, context, run_control, deferred_aiv, &stats->wake, &stats->ready, + &stats->completion, trace_enabled, nullptr, nullptr, ready_owner + )) + return false; + scheduler_progress = scheduler_progress || deferred_aiv->count != deferred_before; + if (deferred_aiv->count != 0 && !scheduler_publish_deferred_aiv_to_resolver( + graph, scheduler_state_base, context, run_control, deferred_aiv, + trace_enabled, &preferred_ready_slot + )) + return false; + } } - { - // receive_time marks the moment AICPU's full "task is ready to - // execute" signal landed on this core. Paired with start_time - // (captured after the per-task dcci + ack pair) it lets DFX split - // head_OH into the AICPU→AICore-ready propagation (dispatch_ts → - // receive_time, hardware + scheduling-bound) and the AICore-local - // critical-path prep (receive_time → start_time, software-tunable). - // Stored in the record as a 32-bit delta `start_time - receive_time`. - // - // For the common path (src_payload == 0) the new task_id on - // DATA_MAIN_BASE is itself the ready signal, so receive_time is - // stamped immediately and local_setup covers dcci + ack. - // - // For the speculative early-dispatch path (src_payload != 0) the - // dcci ran BEFORE the dependency-wait spin, so its cost is hidden - // behind the doorbell-wait — not on the critical path between - // "task genuinely ready" and "kernel begins". receive_time is - // re-stamped after the doorbell arrives, so propagation absorbs - // both the original NoC delivery AND any speculation overshoot, - // while local_setup stays the pure ack-on-critical-path cost. This - // makes local_setup the clean "AICore prep we can't hide" figure - // for both paths. - uint64_t receive_time = get_sys_cnt_aicore(); - - uint32_t task_id = reg_val; // Decode: register holds task_id directly - - // Select dual-buffer slot: same bit as AICPU used when writing payload - __gm__ DispatchPayload *exec_payload = payload + (task_id & 1u); - - // Invalidate payload buffer (AICPU updates its content each dispatch) - dcci(exec_payload, ENTIRE_DATA_CACHE); - - // Speculative early-dispatch gate. A not-ready task was staged on - // this core before its dependencies resolved; wait until AICPU rings - // the doorbell (DATA_MAIN_BASE high 32 == task_id) before executing. - // The ACK is deferred until AFTER the gate so the scheduler keeps the - // core off-limits (pending_occupied stays set, no ACK->pending_freed) - // while the task is gated — preventing a real task from being - // dual-issued behind it. The kernel's own input dcci runs inside - // execute_task() below — strictly AFTER this gate — so predecessor - // outputs are visible. src_payload == 0 (the common path) skips this; - // a non-zero src_payload is both the gate flag and the source - // TaskPayload. - if (exec_payload->src_payload != 0) { - // AICPU staged only src_payload, not the arg vector — fill - // args[0..num_args) ourselves now, while we are idle waiting for - // the doorbell. The whole-cache dcci(ENTIRE_DATA_CACHE) above - // already invalidated src's lines, so tensor_count/scalar_count/ - // scalars read coherently with the orchestrator's submit writes. - // args[SPMD_LOCAL_CONTEXT_INDEX]/[SPMD_GLOBAL_CONTEXT_INDEX] are - // pre-filled once at init() and never changed; tensor and scalar - // counts are set by the orchestrator, so num_args never reaches them. - __gm__ char *src = reinterpret_cast<__gm__ char *>(exec_payload->src_payload); - int32_t tensor_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_TENSOR_COUNT_OFFSET); - int32_t scalar_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_SCALAR_COUNT_OFFSET); - // Each region is named by an int32 delta from the naming field's own - // address, so resolve the field, then add what it holds. - __gm__ char *tensors_field = src + TASKPAYLOAD_TENSORS_DELTA_OFFSET; - __gm__ char *src_tensors = tensors_field + *reinterpret_cast<__gm__ int32_t *>(tensors_field); - __gm__ char *scalars_field = src + TASKPAYLOAD_SCALARS_DELTA_OFFSET; - __gm__ uint64_t *src_scalars = reinterpret_cast<__gm__ uint64_t *>( - scalars_field + *reinterpret_cast<__gm__ int32_t *>(scalars_field) + int32_t ready_slot = -1; + uint64_t ready_publication = 0; + uint64_t ready_scan_start = trace_enabled ? get_sys_cnt_aicore() : 0; + if (preferred_ready_slot != UINT32_MAX) { + __gm__ SchedulerDispatchSlot *slot = + scheduler_dispatch_slot_at(scheduler_state_base, context, context->worker_index, preferred_ready_slot); + const uint64_t publication = scheduler_gm_query(slot->publication); + ++stats->task_state_poll_count; + if (publication == seen_publication[preferred_ready_slot] || + scheduler_dispatch_state(publication) != SchedulerDispatchSlotState::READY) { + scheduler_record_error( + run_control, slot->task_id, SchedulerGraphResult::INVALID_ARGUMENTS, &graph, context, + SchedulerErrorSite::EXECUTOR_PREFERRED_SLOT_INVALID ); - int n = 0; - for (int32_t i = 0; i < tensor_count; i++) { - exec_payload->args[n++] = reinterpret_cast(src_tensors + i * TASKPAYLOAD_TENSOR_STRIDE); - } - for (int32_t i = 0; i < scalar_count; i++) { - exec_payload->args[n++] = src_scalars[i]; - } - OUT_OF_ORDER_STORE_BARRIER(); - while (true) { - // Honor teardown: shutdown overwrites the low half with EXIT. - // Check it on the doorbell-match iteration too, so an EXIT that - // races in right after the matching doorbell still wins over - // executing the gated task. - if (read_dmb_high32() == task_id) { - if (static_cast(read_reg(RegId::DATA_MAIN_BASE)) == AICORE_EXIT_SIGNAL) { - exiting = true; - } - break; - } - if (static_cast(read_reg(RegId::DATA_MAIN_BASE)) == AICORE_EXIT_SIGNAL) { - exiting = true; - break; - } - SPIN_WAIT_HINT(); - } - if (exiting) { - write_reg(RegId::COND, AICORE_EXITED_VALUE); + return false; + } + ready_slot = static_cast(preferred_ready_slot); + ready_publication = publication; + } else { + for (uint32_t offset = 0; offset < SCHEDULER_PENDING_SLOT_COUNT; ++offset) { + uint32_t slot_index = (scan_start + offset) % SCHEDULER_PENDING_SLOT_COUNT; + __gm__ SchedulerDispatchSlot *slot = + scheduler_dispatch_slot_at(scheduler_state_base, context, context->worker_index, slot_index); + uint64_t publication = scheduler_gm_query(slot->publication); + ++stats->task_state_poll_count; + if (publication != seen_publication[slot_index] && + scheduler_dispatch_state(publication) == SchedulerDispatchSlotState::READY) { + ready_slot = static_cast(slot_index); + ready_publication = publication; break; } - // Re-stamp receive_time at the moment the doorbell landed: the - // dcci above ran during the speculative-staging window - // (overlapped with the dependency wait, off the critical path). - // Propagation now absorbs the speculation overshoot; local_setup - // = start - receive stays the pure ack-on-critical-path cost. - receive_time = get_sys_cnt_aicore(); } + } + uint64_t ready_poll_end = trace_enabled ? get_sys_cnt_aicore() : 0; - // Bind this task to the currently-published buffer generation - // before ACK makes progress visible to AICPU. - __gm__ ChipSwimlaneAicoreTaskRecord *chip_swimlane_record = nullptr; - if (chip_swimlane_enabled) { - chip_swimlane_record = - chip_swimlane_aicore_reserve_task_record(chip_swimlane_head, &chip_swimlane_local); + if (ready_slot >= 0) { + uint32_t slot_index = static_cast(ready_slot); + __gm__ SchedulerDispatchSlot *slot = + scheduler_dispatch_slot_at(scheduler_state_base, context, context->worker_index, slot_index); + __gm__ DispatchPayload *payload = scheduler_state_at( + scheduler_state_base, + context->dispatch_payload_offset + static_cast(slot_index) * sizeof(DispatchPayload) + ); + uint64_t ready_observe = get_sys_cnt_aicore(); + scheduler_invalidate_cache_line(slot); + scheduler_observe_dispatch_payload_control(payload); + scheduler_observe_dispatch_payload_arguments(payload); + scheduler_observe_dispatch_payload_barrier(); + if (slot->pending_slot != slot_index || slot->task_id < 0 || + static_cast(slot->task_id) >= graph.task_count || + slot->generation != scheduler_dispatch_generation(ready_publication)) { + scheduler_record_error( + run_control, slot->task_id, SchedulerGraphResult::INVALID_ARGUMENTS, &graph, context, + SchedulerErrorSite::EXECUTOR_INVALID_DISPATCH_SLOT + ); + return false; } + seen_publication[slot_index] = ready_publication; + SchedulerExecutionRecord record{ + slot->task_id, + slot->claim_worker_id, + slot->claim_start_cycles, + slot->claim_end_cycles, + static_cast(slot->ready_source), + }; + const bool commit_scheduler_trace = + trace_enabled && should_commit_scheduler_trace(scheduler_state_base, context, slot); + __gm__ SchedulerTaskMetadata *task_metadata = + scheduler_task_metadata_at(scheduler_state_base, context, record.task_id); + scheduler_observe_cache_line(task_metadata); + const bool commit_task_timing = task_metadata->timing_slot >= 0 && + task_metadata->timing_slot < SCHEDULER_TASK_TIMING_SLOT_COUNT && + should_commit_scheduler_trace(scheduler_state_base, context, slot); + OUT_OF_ORDER_STORE_BARRIER(); + uint64_t kernel_start = get_sys_cnt_aicore(); + if (trace_enabled) { + __gm__ SchedulerTaskControl *control = + scheduler_task_control_at(scheduler_state_base, context, record.task_id); + scheduler_observe_cache_line(&control->next_waiter); + if (control->ready_publish_cycles != 0 && kernel_start >= control->ready_publish_cycles) { + uint64_t lag = kernel_start - control->ready_publish_cycles; + stats->completion.ready_to_kernel_cycles += lag; + if (lag > stats->completion.ready_to_kernel_max_cycles) + stats->completion.ready_to_kernel_max_cycles = lag; + } + } + execute_task(payload); + uint64_t kernel_end = get_sys_cnt_aicore(); + scheduler_publish_dispatch_payload(payload); + uint64_t completion_start = get_sys_cnt_aicore(); + uint64_t local_completion_index = stats->completion.enqueue_count; + uint64_t completion_id = scheduler_completion_id(context, local_completion_index); + uint64_t completion_inbox_index = context->resolver_index; + __gm__ SchedulerCompletionInbox *completion_line = + scheduler_completion_inbox_at(scheduler_state_base, context, context->worker_index); + scheduler_gm_store(completion_line->completed_generations[slot_index], slot->generation); + ++stats->completion.enqueue_count; + uint64_t completion_end = get_sys_cnt_aicore(); + ++stats->executed_task_count; + stats->payload_cycles += kernel_start - ready_observe; + stats->kernel_cycles += kernel_end - kernel_start; + stats->completion_enqueue_cycles += completion_end - completion_start; + scan_start = (slot_index + 1) % SCHEDULER_PENDING_SLOT_COUNT; + backoff_iterations = kInitialBackoffIterations; + if (commit_scheduler_trace) { + uint64_t bookkeeping_end = get_sys_cnt_aicore(); + commit_task_trace( + scheduler_state_base, context, record, ready_scan_start, ready_observe, kernel_start, kernel_end, + completion_end, bookkeeping_end, previous_trace_commit_end, aicore_entry_cycles, + handshake_publish_cycles, register_release_cycles, descriptor_cache_observed_cycles, completion_id, + completion_inbox_index, inter_task_timing + ); + previous_trace_commit_end = get_sys_cnt_aicore(); + inter_task_start_cycles = previous_trace_commit_end; + } else if (commit_task_timing) { + commit_task_timing_trace(scheduler_state_base, context, record.task_id, kernel_start, kernel_end); + } else if (trace_enabled) { + inter_task_start_cycles = get_sys_cnt_aicore(); + } + inter_task_timing.reset(); + continue; + } - write_reg(RegId::COND, MAKE_ACK_VALUE(task_id)); - - // Performance profiling: record start time - uint64_t start_time = get_sys_cnt_aicore(); + if (trace_enabled) inter_task_timing.ready_poll_cycles += ready_poll_end - ready_scan_start; - // PMU: start counting window around kernel execution - if (pmu_enabled) { - pmu_aicore_begin(); - } + if (scheduler_progress) { + backoff_iterations = kInitialBackoffIterations; + continue; + } + ++stats->idle_iteration_count; + uint64_t backoff_start = get_sys_cnt_aicore(); + local_backoff(backoff_iterations); + uint64_t backoff_end = get_sys_cnt_aicore(); + stats->backoff_cycles += backoff_end - backoff_start; + if (trace_enabled) inter_task_timing.backoff_cycles += backoff_end - backoff_start; + if (backoff_iterations < kMaximumBackoffIterations) backoff_iterations <<= 1; + } + return true; +} - // Execute the task - execute_task(exec_payload); +} // namespace - // Keep start_time -> end_time scoped to AICore execution. - uint64_t end_time = chip_swimlane_enabled ? get_sys_cnt_aicore() : 0; +__aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, int block_idx, CoreType core_type) { + __gm__ Handshake *handshake = (__gm__ Handshake *)(&runtime->workers[block_idx]); + const uint32_t profiling_flag = get_aicore_profiling_flag(); + scheduler_observe_cache_line(handshake); + if ((handshake->aicpu_ready != SCHEDULER_RUNTIME_MODE_RESIDENT_PENDING && + handshake->aicpu_ready != SCHEDULER_RUNTIME_MODE_RESIDENT_READY)) { + legacy_aicore_execute(runtime, block_idx, core_type); + return; + } + bool trace_enabled = SIMPLER_GET_DFX_FLAG(profiling_flag, SIMPLER_DFX_FLAG_CHIP_SWIMLANE); + uint64_t aicore_entry_cycles = trace_enabled ? get_sys_cnt_aicore() : 0; + handshake->physical_core_id = get_physical_core_id(); + handshake->core_type = core_type; + OUT_OF_ORDER_STORE_BARRIER(); + handshake->aicore_done = block_idx + 1; + dcci(handshake, SINGLE_CACHE_LINE, CACHELINE_OUT); + dsb((mem_dsb_t)0); + uint64_t handshake_publish_cycles = trace_enabled ? get_sys_cnt_aicore() : 0; - if (pmu_enabled) { - pmu_aicore_end(); - } + // AICPU publishes the fully configured context through GM. This lets + // AICore perform all pre-kernel work without consuming a DMB launch. + // Host publishes enough immutable scheduler-state addressing for the READY + // watchdog. AICPU fills the topology-dependent context fields before READY, + // so this pre-publish view must not be reused for execution. + __gm__ SchedulerWorkerContext *pending_context = reinterpret_cast<__gm__ SchedulerWorkerContext *>(handshake->task); + scheduler_observe_cache_line(pending_context); + __gm__ void *pending_scheduler_state_base = + reinterpret_cast<__gm__ void *>(pending_context->scheduler_state_base_address); + __gm__ SchedulerRunControl *pending_run_control = + scheduler_state_at(pending_scheduler_state_base, pending_context->run_control_offset); + scheduler_observe_cache_line(pending_run_control); + uint32_t startup_signal = 0; + const uint64_t context_wait_start = get_sys_cnt_aicore(); + while (handshake->aicpu_ready != SCHEDULER_RUNTIME_MODE_RESIDENT_READY && startup_signal != AICORE_EXIT_SIGNAL) { + scheduler_observe_cache_line(handshake); + if (scheduler_gm_query(pending_run_control->scheduler_error) != 0) { + startup_signal = AICORE_EXIT_SIGNAL; + break; + } + if (scheduler_watchdog_expired( + context_wait_start, get_sys_cnt_aicore(), + scheduler_gm_query(pending_run_control->scheduler_timeout_cycles) + )) { + scheduler_record_error( + pending_run_control, SCHEDULER_TASK_ID_INVALID, SchedulerGraphResult::TIMEOUT, nullptr, pending_context, + SchedulerErrorSite::CONTEXT_READY_TIMEOUT + ); + startup_signal = AICORE_EXIT_SIGNAL; + break; + } + startup_signal = static_cast(read_reg(RegId::DATA_MAIN_BASE)); + SPIN_WAIT_HINT(); + } + if (startup_signal == AICORE_EXIT_SIGNAL) { + write_reg(RegId::COND, AICORE_EXITED_VALUE); + return; + } - if (dump_args_enabled) { - pipe_barrier(PIPE_ALL); - } + // READY publishes AICPU's topology-dependent context updates. Re-observe + // them after the acquire point instead of using the pre-publish cache view. + scheduler_observe_cache_line(handshake); + __gm__ SchedulerWorkerContext *context = reinterpret_cast<__gm__ SchedulerWorkerContext *>(handshake->task); + scheduler_observe_cache_line(context); + scheduler_observe_cache_line(&context->task_metadata_offset); + __gm__ void *scheduler_state_base = reinterpret_cast<__gm__ void *>(context->scheduler_state_base_address); + __gm__ SchedulerRunControl *run_control = + scheduler_state_at(scheduler_state_base, context->run_control_offset); + scheduler_observe_cache_line(run_control); + SchedulerGraphView graph{ + context->graph_storage_address, + context->graph_reserved_address, + context->graph_task_count, + context->task_window_last_index, + }; + SchedulerWorkerStats stats{}; + SchedulerDeferredAivQueue deferred_aiv{}; + __gm__ SchedulerReadyOwnerState *ready_owner = nullptr; + uint64_t descriptor_cache_observed_cycles = 0; + if (context->active != 0) { + scheduler_observe_data_cache(reinterpret_cast<__gm__ void *>(graph.storage_address)); + descriptor_cache_observed_cycles = trace_enabled ? get_sys_cnt_aicore() : 0; + if (context->is_resolver != 0) { + ready_owner = scheduler_ready_owner_state_at(scheduler_state_base, context); + scheduler_ready_owner_init(ready_owner); + } + if (context->is_resolver != 0 && + !bootstrap_ready_graph( + graph, scheduler_state_base, context, run_control, scheduler_gm_query(run_control->resolver_count), + &stats, trace_enabled, &deferred_aiv, ready_owner + )) { + scheduler_record_error( + run_control, SCHEDULER_TASK_ID_INVALID, SchedulerGraphResult::INVALID_ARGUMENTS, &graph, context, + SchedulerErrorSite::BOOTSTRAP_FAILED + ); + } + } - // Performance profiling: record task execution. - // Two identity fields go into the record (different roles): - // - task_token_raw (the full TaskId) is pulled from the dispatch - // payload's LocalContext.async_ctx — already in AICore cache - // from the just-completed task, no extra GM load. Host uses - // it as the canonical task identity for JSON output / task-id - // decoding. - // - reg_task_id is `task_id` (= reg_val, the per-core dispatch - // token AICore just read from DATA_MAIN_BASE). Per-dispatch - // unique within this core; host uses it as the join key - // against the AICPU record stream. Required for correctness - // under SPMD (block_num > num_cores) and MIX cluster spread, - // where multiple dispatches of the same task share the same - // task_token_raw. - last_reg_val = reg_val; - write_reg(RegId::COND, MAKE_FIN_VALUE(task_id)); - - if (chip_swimlane_enabled) { - uint64_t task_token_raw = exec_payload->local_context.async_ctx.task_token.raw; - chip_swimlane_aicore_commit_task_record( - chip_swimlane_record, task_token_raw, task_id, receive_time, start_time, end_time - ); - } + // Sole AICore-wide DMB: after this point a prepared slot may execute. + startup_signal = 0; + bool register_release_timed_out = false; + const uint64_t register_wait_start = get_sys_cnt_aicore(); + while (startup_signal != AICPU_IDLE_TASK_ID && startup_signal != AICORE_EXIT_SIGNAL) { + startup_signal = static_cast(read_reg(RegId::DATA_MAIN_BASE)); + if (scheduler_watchdog_expired( + register_wait_start, get_sys_cnt_aicore(), scheduler_gm_query(run_control->scheduler_timeout_cycles) + )) { + scheduler_record_error( + run_control, SCHEDULER_TASK_ID_INVALID, SchedulerGraphResult::TIMEOUT, &graph, context, + SchedulerErrorSite::REGISTER_RELEASE_TIMEOUT + ); + register_release_timed_out = true; + break; + } + SPIN_WAIT_HINT(); + } + if (startup_signal == AICORE_EXIT_SIGNAL || register_release_timed_out) { + if (trace_enabled) stats.exit_observed_cycles = get_sys_cnt_aicore(); + } else { + uint64_t register_release_cycles = trace_enabled ? get_sys_cnt_aicore() : 0; + write_reg(RegId::COND, AICORE_IDLE_VALUE); + if (context->active != 0) { + (void)run_ready_dispatch_loop( + graph, scheduler_state_base, context, run_control, &stats, trace_enabled, aicore_entry_cycles, + handshake_publish_cycles, register_release_cycles, descriptor_cache_observed_cycles, &deferred_aiv + ); } } - // Flush all dirty cache lines to HBM before kernel exit. - dcci(my_hank, SINGLE_CACHE_LINE, CACHELINE_OUT); + if (trace_enabled && stats.exit_wait_start_cycles == 0) stats.exit_wait_start_cycles = get_sys_cnt_aicore(); + const uint64_t exit_wait_start = get_sys_cnt_aicore(); + while (stats.exit_observed_cycles == 0 && + static_cast(read_reg(RegId::DATA_MAIN_BASE)) != AICORE_EXIT_SIGNAL) { + if (scheduler_watchdog_expired( + exit_wait_start, get_sys_cnt_aicore(), scheduler_gm_query(run_control->scheduler_timeout_cycles) + )) { + scheduler_record_error( + run_control, SCHEDULER_TASK_ID_INVALID, SchedulerGraphResult::TIMEOUT, &graph, context, + SchedulerErrorSite::EXIT_WAIT_TIMEOUT + ); + break; + } + SPIN_WAIT_HINT(); + } + if (trace_enabled && stats.exit_observed_cycles == 0) stats.exit_observed_cycles = get_sys_cnt_aicore(); + if (trace_enabled) stats.final_stats_publish_start_cycles = get_sys_cnt_aicore(); + publish_worker_stats(context, stats); + if (trace_enabled) { + stats.final_stats_publish_end_cycles = get_sys_cnt_aicore(); + stats.exit_ack_publish_cycles = get_sys_cnt_aicore(); + context->final_stats_publish_end_cycles = stats.final_stats_publish_end_cycles; + context->exit_ack_publish_cycles = stats.exit_ack_publish_cycles; + scheduler_publish_cache_line(&context->completion_enqueue_cycles); + } + write_reg(RegId::COND, AICORE_EXITED_VALUE); } diff --git a/src/a5/runtime/host_build_graph/aicore/aicore_legacy_executor.cpp b/src/a5/runtime/host_build_graph/aicore/aicore_legacy_executor.cpp new file mode 100644 index 0000000000..febaf9b038 --- /dev/null +++ b/src/a5/runtime/host_build_graph/aicore/aicore_legacy_executor.cpp @@ -0,0 +1,303 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "aicore/aicore.h" +#include "aicore/aicore_profiling_state.h" +#include "aicore/chip_swimlane_collector_aicore.h" +#include "aicore/pmu_collector_aicore.h" +#include "common/chip_swimlane_profiling.h" +#include "common/platform_config.h" // Register-based communication +#include "dispatch_payload.h" +#include "runtime.h" + +/** + * Unified function pointer type for kernel dispatch + * + * All kernels follow the same signature: void kernel(__gm__ int64_t* args) + * This enables simple, switch-free dispatch. + */ +typedef void (*UnifiedKernelFunc)(__gm__ int64_t *); + +/** + * Execute task from DispatchPayload. + * + * Reads function_bin_addr and args from the dispatch payload. + * + * @param payload Pointer to DispatchPayload in global memory + */ +__aicore__ __attribute__((always_inline)) static void execute_task(__gm__ DispatchPayload *payload) { + if (payload == nullptr || payload->function_bin_addr == 0) { + return; + } + + UnifiedKernelFunc kernel = (UnifiedKernelFunc)payload->function_bin_addr; + kernel(reinterpret_cast<__gm__ int64_t *>(payload->args)); + OUT_OF_ORDER_STORE_BARRIER(); +} + +/** + * AICore main execution loop + * + * Implements the AICPU-AICore register-based dispatch protocol: + * 1. Report physical core ID and core type, signal aicore_done (no AICPU wait) + * 2. Wait for the AICPU to open our register window (DATA_MAIN_BASE != 0) + * 3. Cache per-core DispatchPayload pointer from my_hank->task + * 4. Poll DATA_MAIN_BASE register for task dispatch until exit signal + * + * AICore reports on launch; the AICPU writes &s_payload_per_core[i] to + * my_hank->task and then opens the register window (DATA_MAIN_BASE = IDLE), which + * is itself the acknowledgement. AICore caches this pointer and reads + * function_bin_addr + args pointer from it on each dispatch. reg_val is a + * monotonically increasing task ID used only for dispatch signaling and + * ACK/FIN protocol. + * + * Profiling state (enable flag, chip swimlane rotation channel) is published into the platform + * via set_aicore_profiling_flag / set_chip_swimlane_aicore_head_slot at kernel entry — + * this routine reads it through the matching getters, so neither Handshake + * nor this signature carry profiling fields. + * + * @param runtime Pointer to Runtime in global memory + * @param block_idx Block index (core ID) + * @param core_type Core type (AIC or AIV) + */ +__aicore__ __attribute__((weak)) void +legacy_aicore_execute(__gm__ Runtime *runtime, int block_idx, CoreType core_type) { + __gm__ Handshake *my_hank = (__gm__ Handshake *)(&runtime->workers[block_idx]); + + // Phase 1: report physical core ID + core type and signal done in one write, + // with no wait for the AICPU — both fields are self-known. The AICPU opens + // this core's register window only after it observes aicore_done, so a single + // report suffices. The host clears aicore_done before this kernel launches, + // so the value the AICPU reads is this run's report, never a stale prior one. + my_hank->physical_core_id = get_physical_core_id(); + my_hank->core_type = core_type; + OUT_OF_ORDER_STORE_BARRIER(); + my_hank->aicore_done = block_idx + 1; // Signal ready (use block_idx + 1 to avoid 0) + dcci(my_hank, SINGLE_CACHE_LINE, CACHELINE_OUT); + + // Phase 2: Wait for the AICPU to open our register window. A kernel launch + // resets DATA_MAIN_BASE to 0 (verified on a2a3 silicon); the AICPU writes + // DATA_MAIN_BASE = AICPU_IDLE_TASK_ID (non-zero) as it opens FAST_PATH, so a + // non-zero read means the window is open and reads/writes are valid. The + // AICPU runs assign_cores_to_threads (µs) between opening the window and the + // first dispatch, so this IDLE is observed long before any task_id lands — + // the poll cannot miss it and mistake a later task for the reset value. + // Window-open is the sync point for everything the AICPU publishes (task + // pointer, swimlane head): the AICPU writes those before opening the window. + while (read_reg(RegId::DATA_MAIN_BASE) == 0) { + SPIN_WAIT_HINT(); + } + // Report initial idle status via register (FAST_PATH is now open). + write_reg(RegId::COND, AICORE_IDLE_VALUE); + + // The AICPU writes task after observing our report (so our CACHELINE_OUT flush + // above cannot clobber it) and before opening the window; dcci to read its + // fresh value here. + dcci(my_hank, SINGLE_CACHE_LINE); + __gm__ DispatchPayload *payload = reinterpret_cast<__gm__ DispatchPayload *>(my_hank->task); + + uint32_t enable_profiling_flag = get_aicore_profiling_flag(); + bool chip_swimlane_enabled = SIMPLER_GET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_CHIP_SWIMLANE); + bool dump_args_enabled = SIMPLER_GET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_DUMP_ARGS); + bool pmu_enabled = SIMPLER_GET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_PMU); + + // Per-core ChipSwimlaneActiveHead channel. AICPU completes + // `chip_swimlane_aicpu_init` (in pre_handshake_init) before any thread opens a + // register window in `handshake_partition`, and Phase 2 above has already + // observed our window open, so the rotation-table slot is populated and the + // first deref is safe here — off the dispatch→start critical path. + __gm__ ChipSwimlaneActiveHead *chip_swimlane_head = + chip_swimlane_enabled ? get_chip_swimlane_aicore_head() : nullptr; + // cached_buf_seq must start != AICPU's initial head.current_buf_seq (0) + // so the first reservation observes a mismatch and loads the buffer ptr. + ChipSwimlaneAicoreLocalState chip_swimlane_local = {nullptr, UINT32_MAX, 0}; + + // Phase 4: Main execution loop - poll register for tasks until exit signal + // Register encoding: AICPU_IDLE_TASK_ID=idle, task_id=task, AICORE_EXIT_SIGNAL=exit + uint32_t reg_val = AICPU_IDLE_TASK_ID; + uint32_t last_reg_val = AICPU_IDLE_TASK_ID; + bool exiting = false; + + while (true) { + reg_val = static_cast(read_reg(RegId::DATA_MAIN_BASE)); + if (reg_val == AICORE_EXIT_SIGNAL) { + // Signal exit acknowledgment to AICPU + write_reg(RegId::COND, AICORE_EXITED_VALUE); + break; + } + + // Execute task if new (reg_val encoding: AICPU_IDLE_TASK_ID=idle, task_id=task) + if (reg_val == AICPU_IDLE_TASK_ID || reg_val == last_reg_val) { + SPIN_WAIT_HINT(); + continue; + } + + { + // receive_time marks the moment AICPU's full "task is ready to + // execute" signal landed on this core. Paired with start_time + // (captured after the per-task dcci + ack pair) it lets DFX split + // head_OH into the AICPU→AICore-ready propagation (dispatch_ts → + // receive_time, hardware + scheduling-bound) and the AICore-local + // critical-path prep (receive_time → start_time, software-tunable). + // Stored in the record as a 32-bit delta `start_time - receive_time`. + // + // For the common path (src_payload == 0) the new task_id on + // DATA_MAIN_BASE is itself the ready signal, so receive_time is + // stamped immediately and local_setup covers dcci + ack. + // + // For the speculative early-dispatch path (src_payload != 0) the + // dcci ran BEFORE the dependency-wait spin, so its cost is hidden + // behind the doorbell-wait — not on the critical path between + // "task genuinely ready" and "kernel begins". receive_time is + // re-stamped after the doorbell arrives, so propagation absorbs + // both the original NoC delivery AND any speculation overshoot, + // while local_setup stays the pure ack-on-critical-path cost. This + // makes local_setup the clean "AICore prep we can't hide" figure + // for both paths. + uint64_t receive_time = get_sys_cnt_aicore(); + + uint32_t task_id = reg_val; // Decode: register holds task_id directly + + // Select dual-buffer slot: same bit as AICPU used when writing payload + __gm__ DispatchPayload *exec_payload = payload + (task_id & 1u); + + // Invalidate payload buffer (AICPU updates its content each dispatch) + dcci(exec_payload, ENTIRE_DATA_CACHE); + + // Speculative early-dispatch gate. A not-ready task was staged on + // this core before its dependencies resolved; wait until AICPU rings + // the doorbell (DATA_MAIN_BASE high 32 == task_id) before executing. + // The ACK is deferred until AFTER the gate so the scheduler keeps the + // core off-limits (pending_occupied stays set, no ACK->pending_freed) + // while the task is gated — preventing a real task from being + // dual-issued behind it. The kernel's own input dcci runs inside + // execute_task() below — strictly AFTER this gate — so predecessor + // outputs are visible. src_payload == 0 (the common path) skips this; + // a non-zero src_payload is both the gate flag and the source + // TaskPayload. + if (exec_payload->src_payload != 0) { + // AICPU staged only src_payload, not the arg vector — fill + // args[0..num_args) ourselves now, while we are idle waiting for + // the doorbell. The whole-cache dcci(ENTIRE_DATA_CACHE) above + // already invalidated src's lines, so tensor_count/scalar_count/ + // scalars read coherently with the orchestrator's submit writes. + // args[SPMD_LOCAL_CONTEXT_INDEX]/[SPMD_GLOBAL_CONTEXT_INDEX] are + // pre-filled once at init() and never changed; tensor and scalar + // counts are set by the orchestrator, so num_args never reaches them. + __gm__ char *src = reinterpret_cast<__gm__ char *>(exec_payload->src_payload); + int32_t tensor_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_TENSOR_COUNT_OFFSET); + int32_t scalar_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_SCALAR_COUNT_OFFSET); + // Each region is named by an int32 delta from the naming field's own + // address, so resolve the field, then add what it holds. + __gm__ char *tensors_field = src + TASKPAYLOAD_TENSORS_DELTA_OFFSET; + __gm__ char *src_tensors = tensors_field + *reinterpret_cast<__gm__ int32_t *>(tensors_field); + __gm__ char *scalars_field = src + TASKPAYLOAD_SCALARS_DELTA_OFFSET; + __gm__ uint64_t *src_scalars = reinterpret_cast<__gm__ uint64_t *>( + scalars_field + *reinterpret_cast<__gm__ int32_t *>(scalars_field) + ); + int n = 0; + for (int32_t i = 0; i < tensor_count; i++) { + exec_payload->args[n++] = reinterpret_cast(src_tensors + i * TASKPAYLOAD_TENSOR_STRIDE); + } + for (int32_t i = 0; i < scalar_count; i++) { + exec_payload->args[n++] = src_scalars[i]; + } + OUT_OF_ORDER_STORE_BARRIER(); + while (true) { + // Honor teardown: shutdown overwrites the low half with EXIT. + // Check it on the doorbell-match iteration too, so an EXIT that + // races in right after the matching doorbell still wins over + // executing the gated task. + if (read_dmb_high32() == task_id) { + if (static_cast(read_reg(RegId::DATA_MAIN_BASE)) == AICORE_EXIT_SIGNAL) { + exiting = true; + } + break; + } + if (static_cast(read_reg(RegId::DATA_MAIN_BASE)) == AICORE_EXIT_SIGNAL) { + exiting = true; + break; + } + SPIN_WAIT_HINT(); + } + if (exiting) { + write_reg(RegId::COND, AICORE_EXITED_VALUE); + break; + } + // Re-stamp receive_time at the moment the doorbell landed: the + // dcci above ran during the speculative-staging window + // (overlapped with the dependency wait, off the critical path). + // Propagation now absorbs the speculation overshoot; local_setup + // = start - receive stays the pure ack-on-critical-path cost. + receive_time = get_sys_cnt_aicore(); + } + + // Bind this task to the currently-published buffer generation + // before ACK makes progress visible to AICPU. + __gm__ ChipSwimlaneAicoreTaskRecord *chip_swimlane_record = nullptr; + if (chip_swimlane_enabled) { + chip_swimlane_record = + chip_swimlane_aicore_reserve_task_record(chip_swimlane_head, &chip_swimlane_local); + } + + write_reg(RegId::COND, MAKE_ACK_VALUE(task_id)); + + // Performance profiling: record start time + uint64_t start_time = get_sys_cnt_aicore(); + + // PMU: start counting window around kernel execution + if (pmu_enabled) { + pmu_aicore_begin(); + } + + // Execute the task + execute_task(exec_payload); + + // Keep start_time -> end_time scoped to AICore execution. + uint64_t end_time = chip_swimlane_enabled ? get_sys_cnt_aicore() : 0; + + if (pmu_enabled) { + pmu_aicore_end(); + } + + if (dump_args_enabled) { + pipe_barrier(PIPE_ALL); + } + + // Performance profiling: record task execution. + // Two identity fields go into the record (different roles): + // - task_token_raw (ring/local) is pulled from the dispatch + // payload's LocalContext.async_ctx — already in AICore cache + // from the just-completed task, no extra GM load. Host uses + // it as the canonical task identity for JSON output / ring + // decoding. + // - reg_task_id is `task_id` (= reg_val, the per-core dispatch + // token AICore just read from DATA_MAIN_BASE). Per-dispatch + // unique within this core; host uses it as the join key + // against the AICPU record stream. Required for correctness + // under SPMD (block_num > num_cores) and MIX cluster spread, + // where multiple dispatches of the same task share the same + // task_token_raw. + last_reg_val = reg_val; + write_reg(RegId::COND, MAKE_FIN_VALUE(task_id)); + + if (chip_swimlane_enabled) { + uint64_t task_token_raw = exec_payload->local_context.async_ctx.task_token.raw; + chip_swimlane_aicore_commit_task_record( + chip_swimlane_record, task_token_raw, task_id, receive_time, start_time, end_time + ); + } + } + } + + // Flush all dirty cache lines to HBM before kernel exit. + dcci(my_hank, SINGLE_CACHE_LINE, CACHELINE_OUT); +} diff --git a/src/a5/runtime/host_build_graph/aicpu/aicore_lifecycle.cpp b/src/a5/runtime/host_build_graph/aicpu/aicore_lifecycle.cpp new file mode 100644 index 0000000000..296e35c423 --- /dev/null +++ b/src/a5/runtime/host_build_graph/aicpu/aicore_lifecycle.cpp @@ -0,0 +1,442 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "aicore_lifecycle.h" +#include "aicore_scheduler_error.h" +#include "aicore_scheduler_state.h" + +#include "scheduler/scheduler_types.h" +#include "scheduler/scheduler_topology.h" +#include "scheduler/scheduler_watchdog.h" + +#include +#include +#include + +#include "aicpu/args_dump_aicpu.h" +#include "aicpu/aicpu_device_config.h" +#include "aicpu/cache_maintenance.h" +#include "aicpu/chip_swimlane_collector_aicpu.h" +#include "aicpu/device_time.h" +#include "aicpu/platform_regs.h" +#include "aicpu/pmu_collector_aicpu.h" +#include "common/memory_barrier.h" +#include "common/unified_log.h" +#include "runtime.h" +#include "spin_hint.h" + +namespace { + +uint64_t resident_scheduler_timeout_cycles() { + const int32_t timeout_ms = get_scheduler_timeout_ms(); + return timeout_ms > 0 ? static_cast(timeout_ms) * (PLATFORM_PROF_SYS_CNT_FREQ / 1000) : + SCHEDULER_TIMEOUT_CYCLES; +} + +void record_lifecycle_timeout(Runtime *runtime, SchedulerErrorSite error_site) { + SchedulerWorkerContext *context = aicore_scheduler_bootstrap_context(runtime); + if (context == nullptr) return; + cache_invalidate_range(context, 128); + SchedulerRunControl *run_control = aicore_scheduler_run_control(context); + if (run_control == nullptr || + !record_aicore_scheduler_runtime_error(run_control, SchedulerGraphResult::TIMEOUT, error_site)) + return; + cache_flush_range(reinterpret_cast(run_control) + offsetof(SchedulerRunControl, error_claimed), 128); + wmb(); +} + +} // namespace + +int32_t AicoreLifecycle::pre_handshake_init(Runtime *runtime, int32_t aicpu_thread_num, uint64_t regs_base) { + if (runtime == nullptr || runtime->worker_count <= 0 || runtime->worker_count > kMaxWorkers) { + LOG_ERROR("Invalid AICore lifecycle worker count"); + return -1; + } + + std::memset(cores_, 0, sizeof(cores_)); + std::memset(physical_core_ids_, 0, sizeof(physical_core_ids_)); + core_count_ = runtime->worker_count; + aicpu_thread_num_ = aicpu_thread_num; + regs_base_ = regs_base; + handshake_failed_.store(false, std::memory_order_release); + + const bool chip_swimlane_enabled = is_chip_swimlane_enabled(); + if (chip_swimlane_enabled || is_pmu_enabled() || is_dump_args_enabled()) { + LOG_WARN( + "A5 HBG resident AICore diagnostics are best-effort: artifacts may be absent or incomplete and do not " + "yet describe Resolver scheduling" + ); + } + if (chip_swimlane_enabled) chip_swimlane_aicpu_init(core_count_); + return 0; +} + +void AicoreLifecycle::handshake_partition(Runtime *runtime, int32_t tidx, int32_t nthreads) { + Handshake *handshakes = runtime->workers; + const int32_t lo = static_cast((static_cast(tidx) * core_count_) / nthreads); + const int32_t hi = static_cast((static_cast(tidx + 1) * core_count_) / nthreads); + const uint32_t physical_core_count = platform_get_physical_cores_count(); + uint64_t *regs = reinterpret_cast(regs_base_); + + struct ReadyCore { + int32_t worker_id; + uint32_t physical_core_id; + uint64_t reg_addr; + CoreType core_type; + uint64_t handshake_observed_cycles; + }; + ReadyCore ready[kMaxWorkers]{}; + bool observed[kMaxWorkers]{}; + int32_t ready_count = 0; + const uint64_t wait_start = get_sys_cnt_aicpu(); + const uint64_t timeout_cycles = resident_scheduler_timeout_cycles(); + + for (int32_t remaining = hi - lo; remaining > 0;) { + for (int32_t i = lo; i < hi; ++i) { + if (observed[i]) continue; + Handshake *handshake = &handshakes[i]; + cache_invalidate_range(handshake, sizeof(*handshake)); + if (handshake->aicore_done == 0) { + SPIN_WAIT_HINT(); + continue; + } + observed[i] = true; + --remaining; + uint32_t physical_core_id = handshake->physical_core_id; + if (physical_core_id >= physical_core_count) { + LOG_ERROR( + "Core %d reported invalid physical_core_id=%u (platform max=%u)", i, physical_core_id, + physical_core_count + ); + handshake_failed_.store(true, std::memory_order_release); + continue; + } + ready[ready_count++] = { + i, physical_core_id, regs[physical_core_id], handshake->core_type, + is_chip_swimlane_enabled() ? get_sys_cnt_aicpu() : 0 + }; + } + if (scheduler_watchdog_expired(wait_start, get_sys_cnt_aicpu(), timeout_cycles)) { + LOG_ERROR("A5 HBG resident scheduler handshake timeout thread=%d remaining=%d", tidx, remaining); + record_lifecycle_timeout(runtime, SchedulerErrorSite::AICPU_HANDSHAKE_TIMEOUT); + handshake_failed_.store(true, std::memory_order_release); + return; + } + } + + for (int32_t i = 0; i < ready_count; ++i) { + const ReadyCore &core = ready[i]; + cores_[core.worker_id] = {core.reg_addr, core.physical_core_id, core.core_type, + nullptr, core.handshake_observed_cycles, 0}; + physical_core_ids_[core.worker_id] = core.physical_core_id; + } + const uint64_t partition_complete_cycles = is_chip_swimlane_enabled() ? get_sys_cnt_aicpu() : 0; + for (int32_t i = lo; i < hi; ++i) + cores_[i].handshake_partition_complete_cycles = partition_complete_cycles; +} + +int32_t AicoreLifecycle::post_handshake_init(Runtime *runtime) { + SchedulerWorkerContext *bootstrap_context = aicore_scheduler_bootstrap_context(runtime); + if (bootstrap_context == nullptr || runtime->host_total_tasks < 0 || + handshake_failed_.load(std::memory_order_acquire)) + return -1; + cache_invalidate_range(bootstrap_context, 256); + void *scheduler_state_base = aicore_scheduler_state_base(bootstrap_context); + const bool trace_enabled = is_chip_swimlane_enabled(); + const uint64_t config_start_cycles = trace_enabled ? get_sys_cnt_aicpu() : 0; + + auto *run_control = aicore_scheduler_run_control(bootstrap_context); + if (scheduler_state_base == nullptr || run_control == nullptr) return -1; + auto *contexts = + scheduler_state_at(scheduler_state_base, bootstrap_context->worker_contexts_offset); + auto *lifecycle_traces = scheduler_state_at( + scheduler_state_base, bootstrap_context->aicpu_lifecycle_traces_offset + ); + cache_invalidate_range(run_control, sizeof(*run_control)); + cache_invalidate_range(contexts, static_cast(core_count_) * sizeof(*contexts)); + cache_invalidate_range(lifecycle_traces, static_cast(core_count_) * sizeof(*lifecycle_traces)); + + int32_t aic_count = 0; + int32_t aiv_count = 0; + for (int32_t i = 0; i < core_count_; ++i) { + if (cores_[i].core_type == CoreType::AIC) { + contexts[i].type_rank = aic_count++; + } else if (cores_[i].core_type == CoreType::AIV) { + contexts[i].type_rank = aiv_count++; + } else { + LOG_ERROR("Core %d reported invalid core type", i); + return -1; + } + contexts[i].core_type = static_cast(cores_[i].core_type); + contexts[i].physical_core_id = static_cast(cores_[i].physical_core_id); + contexts[i].active = 0; + cores_[i].trace = &lifecycle_traces[i]; + lifecycle_traces[i].worker_id = static_cast(i); + lifecycle_traces[i].core_type = static_cast(cores_[i].core_type); + lifecycle_traces[i].physical_core_id = static_cast(cores_[i].physical_core_id); + lifecycle_traces[i].handshake_observed_cycles = cores_[i].handshake_observed_cycles; + lifecycle_traces[i].handshake_partition_complete_cycles = cores_[i].handshake_partition_complete_cycles; + lifecycle_traces[i].config_start_cycles = config_start_cycles; + } + LOG_INFO("Core discovery complete: %d AIC, %d AIV", aic_count, aiv_count); + + if (aic_count <= 0 || aic_count > static_cast(SCHEDULER_CLUSTER_CAPACITY) || + aiv_count != aic_count * PLATFORM_AIV_CORES_PER_BLOCKDIM) { + LOG_ERROR("A5 HBG AICore scheduler: incomplete cluster topology AIC=%d AIV=%d", aic_count, aiv_count); + return -1; + } + int32_t cluster_workers[SCHEDULER_CLUSTER_CAPACITY][PLATFORM_CORES_PER_BLOCKDIM]; + for (auto &cluster : cluster_workers) + for (int32_t &worker : cluster) + worker = -1; + for (int32_t worker = 0; worker < core_count_; ++worker) { + const int32_t physical_core_id = static_cast(cores_[worker].physical_core_id); + SchedulerClusterCoordinate coordinate{-1, -1}; + const bool mapped = scheduler_cluster_coordinate_from_worker( + worker, cores_[worker].core_type == CoreType::AIC, aic_count, PLATFORM_AIV_CORES_PER_BLOCKDIM, &coordinate + ); + const int32_t cluster = coordinate.cluster_index; + const int32_t lane = coordinate.cluster_lane; + if (!mapped || lane >= PLATFORM_CORES_PER_BLOCKDIM || cluster_workers[cluster][lane] != -1) { + LOG_ERROR( + "A5 HBG AICore scheduler: invalid discovered topology worker=%d physical=%d cluster=%d lane=%d", worker, + physical_core_id, cluster, lane + ); + return -1; + } + cluster_workers[cluster][lane] = worker; + } + static_assert(PLATFORM_CORES_PER_BLOCKDIM == 3, "Resolver selection assumes one AIC and two AIV lanes"); + static_assert(PLATFORM_AIV_CORES_PER_BLOCKDIM == 2, "Resolver selection assumes two AIV lanes per cluster"); + for (int32_t cluster = 0; cluster < aic_count; ++cluster) { + for (int32_t lane = 0; lane < PLATFORM_CORES_PER_BLOCKDIM; ++lane) { + if (cluster_workers[cluster][lane] < 0) { + LOG_ERROR("A5 HBG AICore scheduler: cluster %d is missing lane %d", cluster, lane); + return -1; + } + } + } + + const uint64_t executable_task_count = run_control->expected_task_count - run_control->inline_completed_count; + const int32_t requested_aic = static_cast( + std::min(aic_count, std::max(run_control->aic_worker_demand, run_control->aic_task_count)) + ); + const int32_t requested_aiv = static_cast( + std::min(aiv_count, std::max(run_control->aiv_worker_demand, run_control->aiv_task_count)) + ); + uint64_t required_clusters = std::max( + requested_aic, + (static_cast(requested_aiv) + PLATFORM_AIV_CORES_PER_BLOCKDIM - 1) / PLATFORM_AIV_CORES_PER_BLOCKDIM + ); + if (executable_task_count != 0) required_clusters = std::max(required_clusters, 1); + const int32_t active_clusters = static_cast(std::min(required_clusters, aic_count)); + if (run_control->aic_worker_demand > static_cast(aic_count) || + run_control->aiv_worker_demand > static_cast(aiv_count) || + (executable_task_count != 0 && active_clusters == 0)) { + LOG_ERROR( + "A5 HBG AICore scheduler: topology cannot execute graph (demand AIC=%" PRIu64 " AIV=%" PRIu64 + ", cores AIC=%d AIV=%d)", + run_control->aic_worker_demand, run_control->aiv_worker_demand, aic_count, aiv_count + ); + return -1; + } + + for (int32_t cluster = 0; cluster < aic_count; ++cluster) { + const int32_t aiv0_worker = cluster_workers[cluster][1]; + const int32_t aiv1_worker = cluster_workers[cluster][2]; + // Do not bind Resolver ownership to a runtime worker rank. Pick the + // lower physical AIV in each discovered hardware Cluster. + const uint64_t resolver_worker = static_cast( + cores_[aiv0_worker].physical_core_id <= cores_[aiv1_worker].physical_core_id ? aiv0_worker : aiv1_worker + ); + for (int32_t lane = 0; lane < PLATFORM_CORES_PER_BLOCKDIM; ++lane) { + const int32_t worker = cluster_workers[cluster][lane]; + const bool resolver_lane = static_cast(worker) == resolver_worker; + const bool additional_aiv_lane = + lane != 0 && !resolver_lane && cluster * PLATFORM_AIV_CORES_PER_BLOCKDIM + 1 < requested_aiv; + const bool active_lane = cluster < active_clusters && + (resolver_lane || (lane == 0 && cluster < requested_aic) || additional_aiv_lane); + contexts[worker].active = active_lane ? 1 : 0; + contexts[worker].cluster_count = static_cast(active_clusters); + contexts[worker].cluster_index = static_cast(cluster); + contexts[worker].resolver_index = cluster < active_clusters ? static_cast(cluster) : UINT64_MAX; + contexts[worker].resolver_worker_id = resolver_worker; + contexts[worker].is_resolver = resolver_lane && cluster < active_clusters ? 1 : 0; + contexts[worker].inbox_index = + contexts[worker].is_resolver != 0 ? static_cast(cluster) : UINT64_MAX; + contexts[worker].resolver_count = static_cast(active_clusters); + for (int32_t member = 0; member < PLATFORM_CORES_PER_BLOCKDIM; ++member) + contexts[worker].cluster_worker_ids[member] = static_cast(cluster_workers[cluster][member]); + } + } + int32_t active_aic = 0; + int32_t active_aiv = 0; + for (int32_t worker = 0; worker < core_count_; ++worker) { + if (contexts[worker].active == 0) continue; + if (cores_[worker].core_type == CoreType::AIC) ++active_aic; + else ++active_aiv; + } + run_control->active_worker_count = static_cast(active_aic) + static_cast(active_aiv); + run_control->aic_active_worker_count = static_cast(active_aic); + run_control->aiv_active_worker_count = static_cast(active_aiv); + run_control->resolver_count = static_cast(active_clusters); + run_control->scheduler_timeout_cycles = resident_scheduler_timeout_cycles(); + if (executable_task_count == 0) { + run_control->bootstrap_scan_arrived_count = static_cast(active_clusters); + run_control->bootstrap_scan_complete = 1; + run_control->bootstrap_arrived_count = static_cast(active_clusters); + run_control->bootstrap_complete = 1; + } + + const uint64_t topology_complete_cycles = trace_enabled ? get_sys_cnt_aicpu() : 0; + for (int32_t i = 0; i < core_count_; ++i) + lifecycle_traces[i].topology_complete_cycles = topology_complete_cycles; + + if (is_pmu_enabled()) pmu_aicpu_init(physical_core_ids_, core_count_); + cache_flush_range(contexts, static_cast(core_count_) * sizeof(*contexts)); + cache_flush_range(run_control, sizeof(*run_control)); + cache_flush_range(lifecycle_traces, static_cast(core_count_) * sizeof(*lifecycle_traces)); + wmb(); + return 0; +} + +void AicoreLifecycle::publish_context_partition(Runtime *runtime, int32_t thread_idx) { + const int32_t lo = static_cast((static_cast(thread_idx) * core_count_) / aicpu_thread_num_); + const int32_t hi = static_cast((static_cast(thread_idx + 1) * core_count_) / aicpu_thread_num_); + Handshake *handshakes = runtime->workers; + SchedulerWorkerContext *bootstrap_context = aicore_scheduler_bootstrap_context(runtime); + if (bootstrap_context == nullptr) return; + cache_invalidate_range(bootstrap_context, 256); + void *scheduler_state_base = aicore_scheduler_state_base(bootstrap_context); + for (int32_t i = lo; i < hi; ++i) { + handshakes[i].task = reinterpret_cast(scheduler_state_at( + scheduler_state_base, + bootstrap_context->worker_contexts_offset + static_cast(i) * sizeof(SchedulerWorkerContext) + )); + handshakes[i].aicpu_ready = SCHEDULER_RUNTIME_MODE_RESIDENT_READY; + } + if (hi > lo) cache_flush_range(&handshakes[lo], static_cast(hi - lo) * sizeof(Handshake)); + wmb(); + const uint64_t publish_complete_cycles = is_chip_swimlane_enabled() ? get_sys_cnt_aicpu() : 0; + for (int32_t i = lo; i < hi; ++i) { + if (cores_[i].trace != nullptr) cores_[i].trace->context_publish_complete_cycles = publish_complete_cycles; + } +} + +int32_t AicoreLifecycle::wait_bootstrap_complete(Runtime *runtime) { + SchedulerWorkerContext *context = aicore_scheduler_bootstrap_context(runtime); + if (context == nullptr) return -1; + cache_invalidate_range(context, 128); + auto *run_control = aicore_scheduler_run_control(context); + if (run_control == nullptr) return -1; + const bool trace_enabled = is_chip_swimlane_enabled(); + const uint64_t wait_start_cycles = trace_enabled ? get_sys_cnt_aicpu() : 0; + const uint64_t watchdog_start = get_sys_cnt_aicpu(); + const uint64_t timeout_cycles = resident_scheduler_timeout_cycles(); + uint32_t error_poll_count = 0; + while (true) { + cache_invalidate_range( + reinterpret_cast(run_control) + offsetof(SchedulerRunControl, executed_task_count), 128 + ); + if (run_control->bootstrap_complete != 0) break; + if (++error_poll_count == 64) { + error_poll_count = 0; + cache_invalidate_range( + reinterpret_cast(run_control) + offsetof(SchedulerRunControl, error_claimed), 128 + ); + if (run_control->scheduler_error != 0) return -1; + } + if (scheduler_watchdog_expired(watchdog_start, get_sys_cnt_aicpu(), timeout_cycles)) { + LOG_ERROR("%s", "A5 HBG resident scheduler bootstrap timeout"); + record_lifecycle_timeout(runtime, SchedulerErrorSite::BOOTSTRAP_COMPLETE_TIMEOUT); + return -1; + } + SPIN_WAIT_HINT(); + } + const uint64_t complete_cycles = trace_enabled ? get_sys_cnt_aicpu() : 0; + for (int32_t i = 0; i < core_count_; ++i) { + if (cores_[i].trace == nullptr) continue; + cores_[i].trace->bootstrap_wait_start_cycles = wait_start_cycles; + cores_[i].trace->bootstrap_complete_cycles = complete_cycles; + } + return 0; +} + +int32_t AicoreLifecycle::release_partition(int32_t thread_idx, bool start_execution) { + const int32_t lo = static_cast((static_cast(thread_idx) * core_count_) / aicpu_thread_num_); + const int32_t hi = static_cast((static_cast(thread_idx + 1) * core_count_) / aicpu_thread_num_); + int32_t rc = 0; + wmb(); + for (int32_t i = lo; i < hi; ++i) { + if (cores_[i].reg_addr == 0) continue; + if (start_execution) { + if (cores_[i].trace != nullptr) { + cores_[i].trace->aicpu_thread_id = static_cast(thread_idx); + cores_[i].trace->register_release_cycles = get_sys_cnt_aicpu(); + } + platform_init_aicore_regs(cores_[i].reg_addr); + } else { + if (platform_deinit_aicore_regs(cores_[i].reg_addr) != 0) rc = -1; + } + } + return rc; +} + +void AicoreLifecycle::signal_shutdown_partition(int32_t thread_idx) { + const int32_t lo = static_cast((static_cast(thread_idx) * core_count_) / aicpu_thread_num_); + const int32_t hi = static_cast((static_cast(thread_idx + 1) * core_count_) / aicpu_thread_num_); + for (int32_t i = lo; i < hi; ++i) { + if (cores_[i].reg_addr == 0) continue; + if (cores_[i].trace != nullptr) { + cores_[i].trace->aicpu_thread_id = static_cast(thread_idx); + cores_[i].trace->exit_signal_cycles = get_sys_cnt_aicpu(); + } + write_reg(cores_[i].reg_addr, RegId::DATA_MAIN_BASE, AICORE_EXIT_SIGNAL); + } +} + +int32_t AicoreLifecycle::finish_shutdown_partition(int32_t thread_idx, Runtime *runtime) { + (void)runtime; + const int32_t lo = static_cast((static_cast(thread_idx) * core_count_) / aicpu_thread_num_); + const int32_t hi = static_cast((static_cast(thread_idx + 1) * core_count_) / aicpu_thread_num_); + int32_t rc = 0; + for (int32_t i = lo; i < hi; ++i) { + if (cores_[i].reg_addr == 0) continue; + if (platform_deinit_aicore_regs(cores_[i].reg_addr) != 0) { + rc = -1; + } else if (cores_[i].trace != nullptr) { + cores_[i].trace->exit_ack_cycles = get_sys_cnt_aicpu(); + } + } + rmb(); + if (hi > lo && cores_[lo].trace != nullptr) { + cache_flush_range(cores_[lo].trace, static_cast(hi - lo) * sizeof(AicpuCoreLifecycleTrace)); + } + + int32_t core_ids[kMaxWorkers]{}; + int32_t count = 0; + + for (int32_t i = lo; i < hi; ++i) + core_ids[count++] = i; + + if (is_chip_swimlane_enabled()) chip_swimlane_aicpu_flush(thread_idx, core_ids, count); + if (is_pmu_enabled()) pmu_aicpu_finalize(core_ids, count); + return rc; +} + +void AicoreLifecycle::deinit() { + std::memset(cores_, 0, sizeof(cores_)); + std::memset(physical_core_ids_, 0, sizeof(physical_core_ids_)); + handshake_failed_.store(false, std::memory_order_release); + core_count_ = 0; + aicpu_thread_num_ = 0; + regs_base_ = 0; +} diff --git a/src/a5/runtime/host_build_graph/aicpu/aicore_lifecycle.h b/src/a5/runtime/host_build_graph/aicpu/aicore_lifecycle.h new file mode 100644 index 0000000000..e6f2c193e9 --- /dev/null +++ b/src/a5/runtime/host_build_graph/aicpu/aicore_lifecycle.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include +#include + +#include "common/core_type.h" + +class Runtime; +struct AicpuCoreLifecycleTrace; + +class AicoreLifecycle { +public: + int32_t pre_handshake_init(Runtime *runtime, int32_t aicpu_thread_num, uint64_t regs_base); + void handshake_partition(Runtime *runtime, int32_t tidx, int32_t nthreads); + int32_t post_handshake_init(Runtime *runtime); + void publish_context_partition(Runtime *runtime, int32_t thread_idx); + int32_t wait_bootstrap_complete(Runtime *runtime); + int32_t release_partition(int32_t thread_idx, bool start_execution); + void signal_shutdown_partition(int32_t thread_idx); + int32_t finish_shutdown_partition(int32_t thread_idx, Runtime *runtime); + void deinit(); + +private: + static constexpr int32_t kMaxWorkers = 108; + + struct CoreState { + uint64_t reg_addr; + uint32_t physical_core_id; + CoreType core_type; + AicpuCoreLifecycleTrace *trace; + uint64_t handshake_observed_cycles; + uint64_t handshake_partition_complete_cycles; + }; + + CoreState cores_[kMaxWorkers]{}; + uint32_t physical_core_ids_[kMaxWorkers]{}; + std::atomic handshake_failed_{false}; + int32_t core_count_{0}; + int32_t aicpu_thread_num_{0}; + uint64_t regs_base_{0}; +}; diff --git a/src/a5/runtime/host_build_graph/aicpu/aicore_scheduler_error.h b/src/a5/runtime/host_build_graph/aicpu/aicore_scheduler_error.h new file mode 100644 index 0000000000..ead53be397 --- /dev/null +++ b/src/a5/runtime/host_build_graph/aicpu/aicore_scheduler_error.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include +#include + +#include "host_build_graph/runtime_status.h" +#include "scheduler/scheduler_graph.h" +#include "scheduler/scheduler_types.h" + +// This bridge is intentionally AICPU/host-only. AICore code publishes its +// detailed first error through SchedulerRunControl; AICPU maps that internal +// diagnostic to the existing host runtime status ABI without making this +// std::atomic helper visible to either AICore compiler. +inline int32_t aicore_scheduler_runtime_error_code(uint64_t scheduler_error) { + if (scheduler_error == 0) return SIMPLER_ERROR_NONE; + if (scheduler_error == static_cast(SchedulerGraphResult::TIMEOUT)) { + return SIMPLER_ERROR_SCHEDULER_TIMEOUT; + } + return SIMPLER_ERROR_INVALID_ARGS; +} + +inline bool latch_aicore_scheduler_runtime_error(std::atomic *sched_error_code, uint64_t scheduler_error) { + if (sched_error_code == nullptr || scheduler_error == 0) return false; + int32_t expected = SIMPLER_ERROR_NONE; + return sched_error_code->compare_exchange_strong( + expected, aicore_scheduler_runtime_error_code(scheduler_error), std::memory_order_acq_rel, + std::memory_order_acquire + ); +} + +inline bool record_aicore_scheduler_runtime_error( + SchedulerRunControl *run_control, SchedulerGraphResult scheduler_error, SchedulerErrorSite error_site +) { + if (run_control == nullptr || scheduler_error == SchedulerGraphResult::OK) return false; + uint64_t expected = 0; + if (!__atomic_compare_exchange_n( + &run_control->error_claimed, &expected, UINT64_C(1), false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE + )) + return false; + __atomic_store_n(&run_control->error_site, static_cast(error_site), __ATOMIC_RELAXED); + __atomic_store_n(&run_control->scheduler_error, static_cast(scheduler_error), __ATOMIC_RELEASE); + return true; +} diff --git a/src/a5/runtime/host_build_graph/aicpu/aicore_scheduler_state.h b/src/a5/runtime/host_build_graph/aicpu/aicore_scheduler_state.h new file mode 100644 index 0000000000..4bbcc679be --- /dev/null +++ b/src/a5/runtime/host_build_graph/aicpu/aicore_scheduler_state.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include "runtime.h" +#include "scheduler/scheduler_types.h" + +inline bool aicore_scheduler_runtime_mode_is_resident(uint32_t mode) { + return mode == SCHEDULER_RUNTIME_MODE_RESIDENT_PENDING || mode == SCHEDULER_RUNTIME_MODE_RESIDENT_READY; +} + +inline bool aicore_scheduler_runtime_mode_is_explicit_legacy(uint32_t mode) { + return mode == SCHEDULER_RUNTIME_MODE_LEGACY_GRAPH || mode == SCHEDULER_RUNTIME_MODE_LEGACY_UNSUPPORTED_SHAPE; +} + +inline bool aicore_scheduler_runtime_enabled(const Runtime *runtime) { + return runtime != nullptr && runtime->get_worker_count() > 0 && + aicore_scheduler_runtime_mode_is_resident(runtime->workers[0].aicpu_ready); +} + +inline bool aicore_scheduler_explicit_legacy_enabled(const Runtime *runtime) { + return runtime != nullptr && runtime->get_worker_count() > 0 && + aicore_scheduler_runtime_mode_is_explicit_legacy(runtime->workers[0].aicpu_ready); +} + +inline SchedulerWorkerContext *aicore_scheduler_bootstrap_context(Runtime *runtime) { + if (!aicore_scheduler_runtime_enabled(runtime) || runtime->workers[0].task == 0) return nullptr; + return reinterpret_cast(runtime->workers[0].task); +} + +inline void *aicore_scheduler_state_base(SchedulerWorkerContext *context) { + return context == nullptr ? nullptr : reinterpret_cast(context->scheduler_state_base_address); +} + +inline SchedulerRunControl *aicore_scheduler_run_control(SchedulerWorkerContext *context) { + void *state_base = aicore_scheduler_state_base(context); + return state_base == nullptr ? nullptr : + scheduler_state_at(state_base, context->run_control_offset); +} diff --git a/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp b/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp index bf15c91bcd..0ef9291175 100644 --- a/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp +++ b/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -20,63 +21,32 @@ #include #endif -#include "aicpu/device_time.h" +#include "aicore_lifecycle.h" +#include "aicore_scheduler_error.h" +#include "aicore_scheduler_state.h" +#include "aicpu/aicpu_device_config.h" +#include "aicpu/cache_maintenance.h" +#include "aicpu/args_dump_aicpu.h" +#include "aicpu/chip_swimlane_collector_aicpu.h" #include "aicpu/device_phase_aicpu.h" -#include "callable_protocol.h" -#include "dispatch_payload.h" +#include "aicpu/device_time.h" +#include "aicpu/pmu_collector_aicpu.h" +#include "host_build_graph/runtime_status.h" +#include "host_build_graph/shared_memory.h" #include "runtime.h" +#include "scheduler/scheduler_types.h" +#include "scheduler/scheduler_watchdog.h" #include "spin_hint.h" -// Runtime headers (full struct definition for create/destroy + SIMPLER_SCOPE) -#include "host_build_graph/runtime_core.h" -#include "host_build_graph/runtime_types.h" -#include "host_build_graph/shared_memory.h" - -// Performance profiling headers -#include "aicpu/chip_swimlane_collector_aicpu.h" -#include "aicpu/args_dump_aicpu.h" -#include "common/chip_swimlane_profiling.h" #include "common/unified_log.h" // Register-based communication #include "aicpu/platform_aicpu_affinity.h" #include "aicpu/platform_regs.h" #include "common/platform_config.h" +#include "common/memory_barrier.h" #include "utils/thread_completion_gate.h" -// Core type definitions -#include "common/core_type.h" - -// CoreCallable for resolved dispatch address -#include "callable.h" - -// Scheduler data structures (CoreExecState, CoreTracker, etc.) -#include "scheduler/scheduler_types.h" - -// Scheduler context class -#include "scheduler/scheduler_context.h" - -// From orchestration/common.cpp linked into this DSO — updates g_current_runtime -// here (cleared on teardown before runtime_destroy). -extern "C" void framework_bind_runtime(RuntimeContext *rt); - -static int32_t read_runtime_status(Runtime *runtime) { - if (runtime == nullptr) { - return 0; - } - - void *sm = runtime->get_gm_sm_ptr(); - if (sm == nullptr) { - return 0; - } - - auto *header = static_cast(sm); - int32_t sched_error_code = header->sched_error_code.load(std::memory_order_acquire); - return runtime_status_from_error_code(sched_error_code); -} - -static RuntimeContext *rt{nullptr}; - struct AicpuExecutor { // ===== Thread management state ===== std::atomic thread_idx_{0}; @@ -90,46 +60,127 @@ struct AicpuExecutor { // exposes no affinity idx (sim, where platform_aicpu_affinity_thread_idx() // is -1 during init) so the threads don't all collapse to leader 0. std::atomic hs_setup_done_{false}; + std::atomic hs_config_done_{false}; + std::atomic hs_bootstrap_done_{false}; std::atomic hs_arrived_{0}; + std::atomic hs_release_arrived_{0}; std::atomic hs_thread_seq_{0}; - - // Parallel-boot-classify coordination (see AicpuExecutor::run). classify_ready_ - // is published by the boot leader once its leader-only orchestration setup is - // visible; classify_arrived_ is the barrier counting threads that finished - // their slice of the initial classify. Both are one-shot per run and reset in - // deinit(). - std::atomic classify_ready_{false}; - std::atomic classify_arrived_{0}; + std::atomic init_failure_arrived_{0}; int32_t aicpu_thread_num_{0}; - // ===== Task queue state (managed by scheduler ready queues) ===== - simpler::ThreadCompletionGate completion_gate_; - std::atomic runtime_init_ready_{false}; + std::atomic shutdown_ready_{false}; + std::atomic shutdown_signaled_{0}; + std::atomic run_status_{0}; - // Per-Worker arena backing the RuntimeContext + sm_handle + orch/sched/mailbox - // sub-regions (created in runtime_create_from_sm, released in runtime_destroy). - // Default-constructed: libc-backed backend, no ctx. - DeviceArena runtime_arena_; - - // ===== Scheduler context (owns all dispatch/completion/drain state) ===== - SchedulerContext sched_ctx_; + // AICPU owns only the AICore launch/teardown lifecycle. AICore owns graph + // dependency resolution, Ready scheduling, and task execution. + AicoreLifecycle aicore_lifecycle_; // ===== Methods ===== int32_t init(Runtime *runtime); int32_t run(Runtime *runtime); void deinit(Runtime *runtime); + int32_t finish_failed_init(Runtime *runtime); }; static AicpuExecutor g_aicpu_executor; +extern "C" int32_t legacy_aicpu_execute(Runtime *runtime); + +static uint64_t resident_scheduler_timeout_cycles() { + const int32_t timeout_ms = get_scheduler_timeout_ms(); + return timeout_ms > 0 ? static_cast(timeout_ms) * (PLATFORM_PROF_SYS_CNT_FREQ / 1000) : + SCHEDULER_TIMEOUT_CYCLES; +} + +template +static bool wait_with_scheduler_timeout(Predicate predicate) { + const uint64_t start = get_sys_cnt_aicpu(); + const uint64_t timeout_cycles = resident_scheduler_timeout_cycles(); + while (!predicate()) { + if (scheduler_watchdog_expired(start, get_sys_cnt_aicpu(), timeout_cycles)) return false; + SPIN_WAIT_HINT(); + } + return true; +} + +static void record_resident_timeout(Runtime *runtime, SchedulerErrorSite error_site) { + SchedulerWorkerContext *context = aicore_scheduler_bootstrap_context(runtime); + if (context != nullptr) { + cache_invalidate_range(context, 128); + SchedulerRunControl *run_control = aicore_scheduler_run_control(context); + if (run_control != nullptr && + record_aicore_scheduler_runtime_error(run_control, SchedulerGraphResult::TIMEOUT, error_site)) { + cache_flush_range( + reinterpret_cast(run_control) + offsetof(SchedulerRunControl, error_claimed), 128 + ); + wmb(); + } + } + if (runtime != nullptr && runtime->get_gm_sm_ptr() != nullptr) { + auto *header = static_cast(runtime->get_gm_sm_ptr()); + (void)latch_aicore_scheduler_runtime_error( + &header->sched_error_code, static_cast(SchedulerGraphResult::TIMEOUT) + ); + } + LOG_ERROR("A5 HBG resident scheduler timeout site=%" PRIu64, static_cast(error_site)); +} + +static int32_t read_runtime_status(Runtime *runtime) { + if (runtime == nullptr || runtime->get_gm_sm_ptr() == nullptr) return 0; + auto *header = static_cast(runtime->get_gm_sm_ptr()); + int32_t sched_error_code = header->sched_error_code.load(std::memory_order_acquire); + return runtime_status_from_error_code(sched_error_code); +} + +static int32_t publish_aicore_scheduler_runtime_error(Runtime *runtime) { + if (runtime == nullptr || runtime->get_gm_sm_ptr() == nullptr) return 0; + SchedulerWorkerContext *context = aicore_scheduler_bootstrap_context(runtime); + if (context == nullptr) return 0; + cache_invalidate_range(context, 128); + auto *run_control = aicore_scheduler_run_control(context); + if (run_control == nullptr) return 0; + cache_invalidate_range( + reinterpret_cast(run_control) + offsetof(SchedulerRunControl, error_claimed), 128 + ); + auto *header = static_cast(runtime->get_gm_sm_ptr()); + (void)latch_aicore_scheduler_runtime_error(&header->sched_error_code, run_control->scheduler_error); + return read_runtime_status(runtime); +} + +static void publish_aicore_task_timing(Runtime *runtime) { + SchedulerWorkerContext *context = aicore_scheduler_bootstrap_context(runtime); + if (context == nullptr) return; + cache_invalidate_range(context, 256); + void *scheduler_state_base = aicore_scheduler_state_base(context); + if (scheduler_state_base == nullptr) return; + TaskTimingRecord *records = aicpu_task_timing_records(get_platform_phase_base(), 0); + if (records == nullptr) return; + + auto *metadata = scheduler_state_at(scheduler_state_base, context->task_metadata_offset); + auto *traces = scheduler_state_at(scheduler_state_base, context->trace_cells_offset); + cache_invalidate_range(metadata, static_cast(context->graph_task_count) * sizeof(*metadata)); + cache_invalidate_range(traces, static_cast(context->graph_task_count) * sizeof(*traces)); + + for (uint64_t task_id = 0; task_id < context->graph_task_count; ++task_id) { + const int32_t slot = metadata[task_id].timing_slot; + if (slot < 0 || slot >= NUM_TASK_TIMING_SLOTS) continue; + const uint64_t start = traces[task_id].kernel_start_cycles; + const uint64_t end = traces[task_id].kernel_end_cycles; + if (start == 0 || end <= start) continue; + if (start < records[slot].dispatch_cycle) records[slot].dispatch_cycle = start; + if (end > records[slot].finish_cycle) records[slot].finish_cycle = end; + } + aicpu_publish_task_timing_tail_usage(1); +} + // ===== AicpuExecutor Method Implementations ===== int32_t AicpuExecutor::init(Runtime *runtime) { if (runtime == nullptr) { LOG_ERROR("runtime is nullptr"); - init_failed_.store(true, std::memory_order_release); return -1; } @@ -137,12 +188,12 @@ int32_t AicpuExecutor::init(Runtime *runtime) { // dominant preamble cost (serial MMIO, ~217 µs of ~283 µs for 72 cores), so // it is parallelized: the leader (tidx 0) does the shared setup, every // thread handshakes a disjoint slice of cores, then the leader finishes init - // after a barrier. Non-leaders spin on init_done_. + // across two barriers: discovery precedes configuration, and all register + // releases precede init completion. int32_t nthreads = runtime->aicpu_thread_num; if (nthreads == 0) nthreads = 1; - if (nthreads < 1 || nthreads > MAX_AICPU_THREADS) { + if (nthreads < 1 || nthreads > PLATFORM_MAX_AICPU_THREADS) { LOG_ERROR("Invalid aicpu_thread_num: %d", nthreads); - init_failed_.store(true, std::memory_order_release); return -1; } // Each thread needs a distinct index in [0, nthreads) to pick the leader and @@ -151,7 +202,7 @@ int32_t AicpuExecutor::init(Runtime *runtime) { // thread — hand those a distinct index from a counter (mirrors run()'s // thread_idx_++ fallback) instead of collapsing them all to leader 0, which // would run pre_/post_handshake_init on every thread and race the shared - // scheduler state. Exactly nthreads threads reach init (the gate drops the + // lifecycle state. Exactly nthreads threads reach init (the gate drops the // rest), so the counter yields a gap-free [0, nthreads). int32_t tidx = platform_aicpu_affinity_thread_idx(); if (tidx < 0) tidx = hs_thread_seq_.fetch_add(1, std::memory_order_acq_rel); @@ -173,41 +224,130 @@ int32_t AicpuExecutor::init(Runtime *runtime) { aicpu_thread_num_ = nthreads; hs_arrived_.store(0, std::memory_order_relaxed); - if (sched_ctx_.pre_handshake_init(runtime, aicpu_thread_num_, get_platform_regs()) != 0) { + hs_release_arrived_.store(0, std::memory_order_relaxed); + hs_bootstrap_done_.store(false, std::memory_order_relaxed); + if (aicore_lifecycle_.pre_handshake_init(runtime, aicpu_thread_num_, get_platform_regs()) != 0) { init_failed_.store(true, std::memory_order_release); - hs_setup_done_.store(true, std::memory_order_release); - return -1; } hs_setup_done_.store(true, std::memory_order_release); } else { - while (!hs_setup_done_.load(std::memory_order_acquire)) { - if (init_failed_.load(std::memory_order_acquire)) return -1; + if (!wait_with_scheduler_timeout([this] { + return hs_setup_done_.load(std::memory_order_acquire) || init_failed_.load(std::memory_order_acquire); + })) { + record_resident_timeout(runtime, SchedulerErrorSite::AICPU_INIT_BARRIER_TIMEOUT); + init_failed_.store(true, std::memory_order_release); } - if (init_failed_.load(std::memory_order_acquire)) return -1; } // All threads: handshake this thread's slice of cores in parallel. - sched_ctx_.handshake_partition(runtime, tidx, nthreads); + if (!init_failed_.load(std::memory_order_acquire)) { + aicore_lifecycle_.handshake_partition(runtime, tidx, nthreads); + } - // Barrier: leader waits for every slice to finish, then completes init. + // Barrier 1: every thread discovers its core slice before the leader builds + // and publishes the topology-dependent execution configuration. hs_arrived_.fetch_add(1, std::memory_order_acq_rel); if (is_leader) { - while (hs_arrived_.load(std::memory_order_acquire) < nthreads) {} + if (!wait_with_scheduler_timeout([this, nthreads] { + return hs_arrived_.load(std::memory_order_acquire) >= nthreads; + })) { + record_resident_timeout(runtime, SchedulerErrorSite::AICPU_INIT_BARRIER_TIMEOUT); + init_failed_.store(true, std::memory_order_release); + } completion_gate_.reset(); - if (sched_ctx_.post_handshake_init(runtime) != 0) { + if (!init_failed_.load(std::memory_order_acquire) && aicore_lifecycle_.post_handshake_init(runtime) != 0) { + init_failed_.store(true, std::memory_order_release); + } + hs_config_done_.store(true, std::memory_order_release); + } else { + if (!wait_with_scheduler_timeout([this] { + return hs_config_done_.load(std::memory_order_acquire) || init_failed_.load(std::memory_order_acquire); + })) { + record_resident_timeout(runtime, SchedulerErrorSite::AICPU_INIT_BARRIER_TIMEOUT); + init_failed_.store(true, std::memory_order_release); + } + } + + // Publish the configured context through GM. AICore performs descriptor + // observation, Ready bootstrap, and initial slot fill while the DMB gate + // remains closed. This is a one-way publication, not a second launch gate. + if (!init_failed_.load(std::memory_order_acquire)) { + aicore_lifecycle_.publish_context_partition(runtime, tidx); + } + + // Only the leader polls the shared bootstrap line. Peers wait in AICPU + // local memory, avoiding six concurrent cache invalidates on the same GM + // cache line. The following release is the sole AICore-wide DMB gate. + if (is_leader) { + if (!init_failed_.load(std::memory_order_acquire) && aicore_lifecycle_.wait_bootstrap_complete(runtime) != 0) { + init_failed_.store(true, std::memory_order_release); + } + hs_bootstrap_done_.store(true, std::memory_order_release); + } else { + if (!wait_with_scheduler_timeout([this] { + return hs_bootstrap_done_.load(std::memory_order_acquire) || + init_failed_.load(std::memory_order_acquire); + })) { + record_resident_timeout(runtime, SchedulerErrorSite::AICPU_INIT_BARRIER_TIMEOUT); + init_failed_.store(true, std::memory_order_release); + } + } + + // The DMB register is the only execution gate. On failure the same + // partitioned path sends EXIT and waits for every ACK. + if (aicore_lifecycle_.release_partition(tidx, !init_failed_.load(std::memory_order_acquire)) != 0) { + init_failed_.store(true, std::memory_order_release); + } + + // Barrier 2: init cannot return until every successful slice is released, + // or every failed slice has acknowledged EXIT. + hs_release_arrived_.fetch_add(1, std::memory_order_acq_rel); + if (is_leader) { + if (!wait_with_scheduler_timeout([this, nthreads] { + return hs_release_arrived_.load(std::memory_order_acquire) >= nthreads; + })) { + record_resident_timeout(runtime, SchedulerErrorSite::AICPU_INIT_BARRIER_TIMEOUT); init_failed_.store(true, std::memory_order_release); - init_done_.store(true, std::memory_order_release); - return -1; } init_done_.store(true, std::memory_order_release); - LOG_INFO("AicpuExecutor: Init complete"); + if (!init_failed_.load(std::memory_order_acquire)) LOG_INFO("AicpuExecutor: Init complete"); } else { - while (!init_done_.load(std::memory_order_acquire)) { - if (init_failed_.load(std::memory_order_acquire)) return -1; + if (!wait_with_scheduler_timeout([this] { + return init_done_.load(std::memory_order_acquire) || init_failed_.load(std::memory_order_acquire); + })) { + record_resident_timeout(runtime, SchedulerErrorSite::AICPU_INIT_BARRIER_TIMEOUT); + init_failed_.store(true, std::memory_order_release); } - if (init_failed_.load(std::memory_order_acquire)) return -1; } - return 0; + return init_failed_.load(std::memory_order_acquire) ? finish_failed_init(runtime) : 0; +} + +int32_t AicpuExecutor::finish_failed_init(Runtime *runtime) { + const int32_t nthreads = aicpu_thread_num_; + const bool last = init_failure_arrived_.fetch_add(1, std::memory_order_acq_rel) + 1 == nthreads; + if (last) { + aicore_lifecycle_.deinit(); + completion_gate_.reset(); + shutdown_ready_.store(false, std::memory_order_relaxed); + shutdown_signaled_.store(0, std::memory_order_relaxed); + run_status_.store(0, std::memory_order_relaxed); + init_done_.store(false, std::memory_order_relaxed); + hs_setup_done_.store(false, std::memory_order_relaxed); + hs_config_done_.store(false, std::memory_order_relaxed); + hs_bootstrap_done_.store(false, std::memory_order_relaxed); + hs_arrived_.store(0, std::memory_order_relaxed); + hs_release_arrived_.store(0, std::memory_order_relaxed); + hs_thread_seq_.store(0, std::memory_order_relaxed); + thread_idx_.store(0, std::memory_order_relaxed); + aicpu_thread_num_ = 0; + init_failure_arrived_.store(0, std::memory_order_relaxed); + init_failed_.store(false, std::memory_order_release); + } else if (!wait_with_scheduler_timeout([this] { + return !init_failed_.load(std::memory_order_acquire); + })) { + record_resident_timeout(runtime, SchedulerErrorSite::AICPU_INIT_BARRIER_TIMEOUT); + } + return -1; } /** @@ -216,188 +356,151 @@ int32_t AicpuExecutor::init(Runtime *runtime) { int32_t AicpuExecutor::run(Runtime *runtime) { int32_t affinity_exec_idx = platform_aicpu_affinity_thread_idx(); int32_t thread_idx = (affinity_exec_idx >= 0) ? affinity_exec_idx : (thread_idx_++); - if (thread_idx < 0 || thread_idx >= aicpu_thread_num_ || thread_idx >= MAX_AICPU_THREADS) { + if (thread_idx < 0 || thread_idx >= aicpu_thread_num_ || thread_idx >= PLATFORM_MAX_AICPU_THREADS) { LOG_ERROR( "Thread index %d out of bounds (active=%d max=%d exec_idx=%d)", thread_idx, aicpu_thread_num_, - MAX_AICPU_THREADS, affinity_exec_idx + PLATFORM_MAX_AICPU_THREADS, affinity_exec_idx ); return -1; } - int32_t run_rc = 0; - - // Boot: the last AICPU thread (aicpu_thread_num_ - 1) performs the one-time - // host-orch attach. host_build_graph's orchestrator already ran on the host, - // and every cross-task reference it wrote is an offset from its own block, so - // the SM/arena this thread sees need no address fixup. This thread attaches - // the prebuilt arena, points the SM - // handle's task-header pointers at the device SM WITHOUT resetting the - // host-populated data, hands the host-computed task count to the scheduler, - // and releases the other threads. It then falls through and schedules its own - // cores like every other thread — host_build_graph has no device-side - // orchestrator, so there is no orch/sched split. + bool shutdown_wait_timed_out = false; if (thread_idx == aicpu_thread_num_ - 1) { - void *prebuilt_arena = runtime->get_prebuilt_arena_base(); - size_t off_runtime = runtime->get_prebuilt_runtime_offset(); - - // A boot failure falls through to the common teardown at the end of - // run() — it must NOT return early. This thread owns a core slice - // (handshake_partition assigns [lo, total) to the last thread), so an - // early return would skip shutdown(thread_idx) — leaving its AICore - // cores spinning on an unclosed register window — and the completion - // gate never opens, so the host hangs into the op-execute - // timeout (507018) instead of seeing the failure. On failure: record it - // in run_rc, leave rt null so the dispatch block below skips, and still - // publish runtime_init_ready_ (single point at the block's end) so the - // peer threads stop spinning. - bool boot_ok = (prebuilt_arena != nullptr); - if (!boot_ok) { - LOG_ERROR("Thread %d: host-orch: prebuilt_arena_base is null", thread_idx); - rt = nullptr; - run_rc = -1; - } - - if (boot_ok) { - runtime_arena_.attach(prebuilt_arena, DeviceArena::kDefaultBaseAlign); - rt = reinterpret_cast(static_cast(prebuilt_arena) + off_runtime); - runtime_wire_arena_pointers(runtime_arena_, rt->prebuilt_layout, rt); - - void *sm_ptr = runtime->get_gm_sm_ptr(); - // The image the host shipped is pitched to the submitted task count, - // not to the count the table was dimensioned for, and the device region - // holds exactly that image — so its size comes from the same pitch. - // attach_populated rejects a pitch outside (0, task_capacity] and a - // region too small for it. - const uint64_t live_slots = sm_layout::live_slot_pitch(static_cast(runtime->host_total_tasks)); - const uint64_t sm_size = runtime->sm_image_bytes; - // sm_handle and the scheduler state are the device-only zone: their - // bytes never travel, so they start as whatever the pooled arena last - // held. Zeroing the handle first is what makes attach_populated's - // assignment of every field checkable here rather than by inspecting - // attach_populated. - memset(rt->sm_handle, 0, sizeof(*rt->sm_handle)); - if (!rt->sm_handle->attach_populated( - sm_ptr, sm_size, rt->prebuilt_layout.task_capacity, live_slots, runtime->sm_image_bytes - )) { - LOG_ERROR("Thread %d: host-orch: sm_handle->attach_populated failed", thread_idx); - rt = nullptr; - run_rc = -1; - boot_ok = false; - } else if (!rt->scheduler->init_data_from_layout(rt->prebuilt_layout.sched, runtime_arena_, sm_ptr)) { - LOG_ERROR("Thread %d: host-orch: scheduler init_data_from_layout failed", thread_idx); - rt = nullptr; - run_rc = -1; - boot_ok = false; + int32_t supervisor_rc = 0; + SchedulerWorkerContext *context = aicore_scheduler_bootstrap_context(runtime); + if (context == nullptr || runtime->host_total_tasks < 0) { + LOG_ERROR("A5 HBG AICore scheduler requires an initialized graph"); + supervisor_rc = -1; + } else { + cache_invalidate_range(context, 128); + auto *run_control = aicore_scheduler_run_control(context); + if (run_control == nullptr) { + LOG_ERROR("A5 HBG AICore scheduler is missing run control"); + supervisor_rc = -1; } else { - // Queue headers are set, so the slot arrays can take their ramp, - // and the mailbox ring gets its cursors and publication gates. - // All of it precedes runtime_init_ready_, which is what releases - // the peer threads into the dispatch loop, so neither a push nor a - // completion message sees an uninitialized region. - // - // Both regions sit in the device-only zone, so their bytes are - // whatever the pooled arena last held, and each one's empty state is - // whatever its own initializer writes — a ready queue's is a - // sequence ramp (slot i holds i), the mailbox's is zeroed cursors - // and publication gates. Zeroing the region is a substitute for - // neither call. - rt->scheduler->seed_queue_slots(); - rt->aicore_mailbox->init_empty(); + const uint64_t executable_task_count = + run_control->expected_task_count - run_control->inline_completed_count; + const bool trace_enabled = is_chip_swimlane_enabled(); + const uint64_t completion_wait_start_cycles = trace_enabled ? get_sys_cnt_aicpu() : 0; + uint64_t bootstrap_complete_cycles = 0; + uint64_t all_tasks_resolved_cycles = 0; + uint64_t completion_poll_count = 0; + uint64_t completion_poll_cycles = 0; + uint64_t error_poll_total = 0; + uint32_t error_poll_count = 0; + uint64_t last_progress_cycles = get_sys_cnt_aicpu(); + uint64_t last_resolved_task_count = run_control->resolved_task_count; + bool bootstrap_was_complete = run_control->bootstrap_complete != 0; + const uint64_t timeout_cycles = resident_scheduler_timeout_cycles(); + while (true) { + const uint64_t poll_start = trace_enabled ? get_sys_cnt_aicpu() : 0; + cache_invalidate_range( + reinterpret_cast(run_control) + offsetof(SchedulerRunControl, executed_task_count), + 128 + ); + const bool bootstrap_complete = run_control->bootstrap_complete != 0; + const bool tasks_resolved = run_control->resolved_task_count == executable_task_count; + const uint64_t now = get_sys_cnt_aicpu(); + if (run_control->resolved_task_count != last_resolved_task_count || + bootstrap_complete != bootstrap_was_complete) { + last_resolved_task_count = run_control->resolved_task_count; + bootstrap_was_complete = bootstrap_complete; + last_progress_cycles = now; + } + if (trace_enabled) { + const uint64_t poll_end = get_sys_cnt_aicpu(); + ++completion_poll_count; + completion_poll_cycles += poll_end - poll_start; + if (bootstrap_complete && bootstrap_complete_cycles == 0) { + bootstrap_complete_cycles = poll_end; + } + if (bootstrap_complete && tasks_resolved && all_tasks_resolved_cycles == 0) { + all_tasks_resolved_cycles = poll_end; + } + } + if (bootstrap_complete && tasks_resolved) { + break; + } + if (++error_poll_count == 64) { + error_poll_count = 0; + ++error_poll_total; + cache_invalidate_range( + reinterpret_cast(run_control) + offsetof(SchedulerRunControl, error_claimed), 128 + ); + if (run_control->scheduler_error != 0) break; + } + if (scheduler_watchdog_expired(last_progress_cycles, now, timeout_cycles)) { + record_resident_timeout(runtime, SchedulerErrorSite::EXECUTION_PROGRESS_TIMEOUT); + break; + } + SPIN_WAIT_HINT(); + } + cache_invalidate_range( + reinterpret_cast(run_control) + offsetof(SchedulerRunControl, executed_task_count), 256 + ); + if (run_control->scheduler_error != 0) { + LOG_ERROR( + "A5 HBG AICore scheduler: graph execution failed at task=%" PRIu64 " status=%" PRIu64 + " core=%" PRIu64 " type=%" PRIu64 " graph_tasks=%" PRIu64 " storage=0x%" PRIx64 + " reserved=0x%" PRIx64 " window_last_index=0x%" PRIx64 " site=%" PRIu64, + run_control->error_task_id, run_control->scheduler_error, run_control->error_core_id, + run_control->error_core_type, run_control->error_graph_task_count, + run_control->error_storage_address, run_control->error_reserved_address, + run_control->error_task_window_last_index, run_control->error_site + ); + supervisor_rc = publish_aicore_scheduler_runtime_error(runtime); + if (supervisor_rc == 0) supervisor_rc = -1; + } + if (supervisor_rc == 0 && run_control->resolved_task_count + run_control->inline_completed_count != + run_control->expected_task_count) { + LOG_ERROR( + "A5 HBG AICore scheduler: count mismatch expected=%" PRIu64 " resolved=%" PRIu64 + " inline=%" PRIu64, + run_control->expected_task_count, run_control->resolved_task_count, + run_control->inline_completed_count + ); + supervisor_rc = -1; + } + if (trace_enabled) { + run_control->completion_wait_start_cycles = completion_wait_start_cycles; + run_control->bootstrap_complete_cycles = bootstrap_complete_cycles; + run_control->all_tasks_resolved_cycles = all_tasks_resolved_cycles; + run_control->shutdown_ready_cycles = get_sys_cnt_aicpu(); + run_control->completion_poll_count = completion_poll_count; + run_control->completion_poll_cycles = completion_poll_cycles; + run_control->error_poll_count = error_poll_total; + cache_flush_range( + reinterpret_cast(run_control) + offsetof(SchedulerRunControl, executed_task_count), + 128 + ); + } } } - - if (boot_ok) { - sched_ctx_.bind_runtime(rt); - - // Latch the host-built task count (on_graph_attached sets total_tasks_) - // BEFORE the runtime_init_ready_ release below — that store is the barrier - // that unblocks the scheduler threads. Otherwise they would acquire - // runtime_init_ready_ with total_tasks_=0 and race to an early exit before - // the host task count is visible (host-orch has no concurrent orchestrator - // to keep them alive). - // NOTE: do NOT call rt_orchestration_done(rt) here. The HOST already - // called it in run_host_orchestration; the orchestrator's own - // task-allocator pointers name host memory the device never reads, so - // mark_done()'s active_count() read would dereference it and fault the - // AICPU. on_graph_attached only needs total_tasks and the scalar - // orchestrator.inline_completed_tasks, both already valid. - sched_ctx_.on_graph_attached(rt, thread_idx, runtime->host_total_tasks); - LOG_INFO("Thread %d: host-orch boot complete (%d tasks)", thread_idx, runtime->host_total_tasks); - } - - // Publish "leader setup done" (SM attached, task count latched, queues - // allocated). Every thread then classifies its slice below before any of - // them may dispatch — the leader holds runtime_init_ready_ until then. - classify_ready_.store(true, std::memory_order_release); - } - - // Parallel initial classify. Every AICPU thread waits for the leader's - // orchestration setup, seeds its disjoint slice of the whole graph's ready - // set + wake lists, then barriers. Only once all slices are done does the - // leader publish runtime_init_ready_, so no thread dispatches against a - // half-seeded graph. - while (!classify_ready_.load(std::memory_order_acquire)) { - SPIN_WAIT_HINT(); - } - if (!sched_ctx_.is_completed() && rt != nullptr) { - sched_ctx_.classify_partition(thread_idx, aicpu_thread_num_); - } - classify_arrived_.fetch_add(1, std::memory_order_acq_rel); - if (thread_idx == aicpu_thread_num_ - 1) { - while (classify_arrived_.load(std::memory_order_acquire) < aicpu_thread_num_) { - SPIN_WAIT_HINT(); - } - runtime_init_ready_.store(true, std::memory_order_release); + run_status_.store(supervisor_rc, std::memory_order_release); + shutdown_ready_.store(true, std::memory_order_release); } else { - while (!runtime_init_ready_.load(std::memory_order_acquire)) { - SPIN_WAIT_HINT(); + if (!wait_with_scheduler_timeout([this] { + return shutdown_ready_.load(std::memory_order_acquire); + })) { + record_resident_timeout(runtime, SchedulerErrorSite::AICPU_SHUTDOWN_BARRIER_TIMEOUT); + shutdown_wait_timed_out = true; } } - // Every AICPU thread schedules its assigned cores. - if (!sched_ctx_.is_completed()) { - if (rt == nullptr) { - LOG_ERROR("Thread %d: rt is null after a failed boot, skipping dispatch", thread_idx); - } else { - sched_ctx_.bind_runtime(rt); - // 3S+1P: the last thread is the core-less resolution (P) thread; the - // rest are core-owning schedulers (S). - int32_t completed = (thread_idx == sched_ctx_.p_thread_idx()) ? - sched_ctx_.run_resolution_thread(runtime, thread_idx) : - sched_ctx_.resolve_and_dispatch(runtime, thread_idx); - if (completed < 0) { - LOG_ERROR("Thread %d: Scheduler failed with rc=%d", thread_idx, completed); - run_rc = completed; - } else { - LOG_INFO("Thread %d: Executed %d tasks from runtime", thread_idx, completed); - } - } + int32_t run_rc = + shutdown_wait_timed_out ? -SIMPLER_ERROR_SCHEDULER_TIMEOUT : run_status_.load(std::memory_order_acquire); + aicore_lifecycle_.signal_shutdown_partition(thread_idx); + shutdown_signaled_.fetch_add(1, std::memory_order_acq_rel); + if (!wait_with_scheduler_timeout([this] { + return shutdown_signaled_.load(std::memory_order_acquire) >= aicpu_thread_num_; + })) { + record_resident_timeout(runtime, SchedulerErrorSite::AICPU_SHUTDOWN_BARRIER_TIMEOUT); + if (run_rc == 0) run_rc = -SIMPLER_ERROR_SCHEDULER_TIMEOUT; } - - // Always shutdown AICore — even if sched_ctx_.completed_ was already true. - // platform_deinit_aicore_regs is idempotent. - int32_t shutdown_rc = sched_ctx_.shutdown(thread_idx); - if (shutdown_rc != 0 && run_rc == 0) { - run_rc = shutdown_rc; - } - - LOG_INFO("Thread %d: Completed", thread_idx); - - completion_gate_.arrive_and_finalize_if_last(aicpu_thread_num_, [&] { - aicpu_publish_task_timing_tail_usage(aicpu_thread_num_); - // Destroy the host_build_graph runtime. sm_handle / rt are recreated - // every run, so always tear them down here. - if (rt != nullptr) { - rt->scheduler->print_queues(); - // Clear g_current_runtime in this DSO before destroying rt. - framework_bind_runtime(nullptr); - // A Graph's expansion storage is the tail of its outer task's heap - // allocation, so it retires with that allocation; nothing here owns - // a separate block to release. - runtime_destroy(rt, runtime_arena_); - rt = nullptr; - } + int32_t shutdown_rc = aicore_lifecycle_.finish_shutdown_partition(thread_idx, runtime); + if (shutdown_rc != 0 && run_rc == 0) run_rc = shutdown_rc; + completion_gate_.arrive_and_finalize_if_last(aicpu_thread_num_, [runtime] { + publish_aicore_task_timing(runtime); }); - return run_rc; } @@ -407,26 +510,25 @@ void AicpuExecutor::deinit(Runtime *runtime) { // bypasses this cache. Invalidating now ensures next round reads from HBM. cache_invalidate_range(runtime, sizeof(Runtime)); - // Reset all SchedulerContext-owned state in one place. - sched_ctx_.deinit(); + aicore_lifecycle_.deinit(); completion_gate_.reset(); - runtime_init_ready_.store(false, std::memory_order_release); + shutdown_ready_.store(false, std::memory_order_release); + shutdown_signaled_.store(0, std::memory_order_release); + run_status_.store(0, std::memory_order_release); aicpu_thread_num_ = 0; - // Clear the file-scope runtime pointer (freed by the last scheduler thread before deinit). - rt = nullptr; - LOG_INFO("DeInit: Runtime execution state reset"); init_done_.store(false, std::memory_order_release); init_failed_.store(false, std::memory_order_release); hs_setup_done_.store(false, std::memory_order_release); + hs_config_done_.store(false, std::memory_order_release); + hs_bootstrap_done_.store(false, std::memory_order_release); hs_arrived_.store(0, std::memory_order_release); + hs_release_arrived_.store(0, std::memory_order_release); hs_thread_seq_.store(0, std::memory_order_release); - classify_ready_.store(false, std::memory_order_release); - classify_arrived_.store(0, std::memory_order_release); thread_idx_.store(0, std::memory_order_release); LOG_INFO("DeInit: AicpuExecutor reset complete"); @@ -464,6 +566,17 @@ extern "C" int32_t aicpu_execute(Runtime *runtime) { LOG_ERROR("%s", "Invalid argument: null Runtime pointer"); return -1; } + if (!aicore_scheduler_runtime_enabled(runtime)) { + const bool explicit_legacy = aicore_scheduler_explicit_legacy_enabled(runtime); + // Legacy execution also owns the cleanup handshake for the AICores that + // were already launched. Run it before rejecting an unmarked fallback. + const int32_t legacy_rc = legacy_aicpu_execute(runtime); + if (!explicit_legacy && legacy_rc == 0) { + LOG_ERROR("A5 HBG: ordinary run reached legacy scheduling without an explicit fallback mode"); + return -1; + } + return legacy_rc; + } LOG_INFO("%s", "aicpu_execute: Starting AICPU kernel execution"); @@ -472,7 +585,8 @@ extern "C" int32_t aicpu_execute(Runtime *runtime) { // return is authoritative on all threads and no extra spin is needed. if (g_aicpu_executor.init(runtime) != 0) { LOG_ERROR("%s", "aicpu_execute: Initialization failed, aborting execution"); - return -1; + const int32_t runtime_rc = publish_aicore_scheduler_runtime_error(runtime); + return runtime_rc != 0 ? runtime_rc : -1; } int32_t rc = g_aicpu_executor.run(runtime); @@ -489,7 +603,7 @@ extern "C" int32_t aicpu_execute(Runtime *runtime) { } if (runtime_rc != 0) { - LOG_ERROR("aicpu_execute: simpler runtime failed with rc=%d", runtime_rc); + LOG_ERROR("aicpu_execute: runtime failed with rc=%d", runtime_rc); return runtime_rc; } diff --git a/src/a5/runtime/host_build_graph/aicpu/aicpu_legacy_executor.cpp b/src/a5/runtime/host_build_graph/aicpu/aicpu_legacy_executor.cpp new file mode 100644 index 0000000000..ba1524f66a --- /dev/null +++ b/src/a5/runtime/host_build_graph/aicpu/aicpu_legacy_executor.cpp @@ -0,0 +1,505 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#include + +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif + +#include "aicpu/device_time.h" +#include "aicpu/device_phase_aicpu.h" +#include "callable_protocol.h" +#include "dispatch_payload.h" +#include "runtime.h" +#include "spin_hint.h" + +// Runtime headers (full struct definition for create/destroy + SIMPLER_SCOPE) +#include "runtime_core.h" +#include "runtime_types.h" +#include "shared_memory.h" + +// Performance profiling headers +#include "aicpu/chip_swimlane_collector_aicpu.h" +#include "aicpu/args_dump_aicpu.h" +#include "common/chip_swimlane_profiling.h" +#include "common/unified_log.h" + +// Register-based communication +#include "aicpu/platform_aicpu_affinity.h" +#include "aicpu/platform_regs.h" +#include "common/platform_config.h" +#include "utils/thread_completion_gate.h" + +// Core type definitions +#include "common/core_type.h" + +// CoreCallable for resolved dispatch address +#include "callable.h" + +// Scheduler data structures (CoreExecState, CoreTracker, etc.) +#include "scheduler/scheduler_types.h" + +// Scheduler context class +#include "scheduler/scheduler_context.h" + +// From orchestration/common.cpp linked into this DSO — updates g_current_runtime +// here (cleared on teardown before runtime_destroy). +extern "C" void framework_bind_runtime(RuntimeContext *rt); + +static int32_t read_runtime_status(Runtime *runtime) { + if (runtime == nullptr) { + return 0; + } + + void *sm = runtime->get_gm_sm_ptr(); + if (sm == nullptr) { + return 0; + } + + auto *header = static_cast(sm); + int32_t sched_error_code = header->sched_error_code.load(std::memory_order_acquire); + return runtime_status_from_error_code(sched_error_code); +} + +static RuntimeContext *rt{nullptr}; + +struct LegacyAicpuExecutor { + // ===== Thread management state ===== + std::atomic thread_idx_{0}; + std::atomic init_done_{false}; + std::atomic init_failed_{false}; + + // Parallel-handshake coordination (see LegacyAicpuExecutor::init). hs_setup_done_ + // is published by the leader once the shared pre-handshake setup is visible; + // hs_arrived_ is the barrier counting threads that finished their core slice. + // hs_thread_seq_ hands out a distinct [0, nthreads) index when the platform + // exposes no affinity idx (sim, where platform_aicpu_affinity_thread_idx() + // is -1 during init) so the threads don't all collapse to leader 0. + std::atomic hs_setup_done_{false}; + std::atomic hs_arrived_{0}; + std::atomic hs_thread_seq_{0}; + + // Parallel-boot-classify coordination (see LegacyAicpuExecutor::run). classify_ready_ + // is published by the boot leader once its leader-only orchestration setup is + // visible; classify_arrived_ is the barrier counting threads that finished + // their slice of the initial classify. Both are one-shot per run and reset in + // deinit(). + std::atomic classify_ready_{false}; + std::atomic classify_arrived_{0}; + + int32_t aicpu_thread_num_{0}; + + // ===== Task queue state (managed by scheduler ready queues) ===== + + simpler::ThreadCompletionGate completion_gate_; + std::atomic runtime_init_ready_{false}; + + // Per-Worker arena backing the RuntimeContext + sm_handle + orch/sched/mailbox + // sub-regions (created in runtime_create_from_sm, released in runtime_destroy). + // Default-constructed: libc-backed backend, no ctx. + DeviceArena runtime_arena_; + + // ===== Scheduler context (owns all dispatch/completion/drain state) ===== + SchedulerContext sched_ctx_; + + // ===== Methods ===== + int32_t init(Runtime *runtime); + int32_t run(Runtime *runtime); + void deinit(Runtime *runtime); +}; + +static LegacyAicpuExecutor g_aicpu_executor; + +// ===== LegacyAicpuExecutor Method Implementations ===== + +int32_t LegacyAicpuExecutor::init(Runtime *runtime) { + if (runtime == nullptr) { + LOG_ERROR("runtime is nullptr"); + init_failed_.store(true, std::memory_order_release); + return -1; + } + + // All AICPU threads enter init. The per-core AICore handshake is the + // dominant preamble cost (serial MMIO, ~217 µs of ~283 µs for 72 cores), so + // it is parallelized: the leader (tidx 0) does the shared setup, every + // thread handshakes a disjoint slice of cores, then the leader finishes init + // after a barrier. Non-leaders spin on init_done_. + int32_t nthreads = runtime->aicpu_thread_num; + if (nthreads == 0) nthreads = 1; + if (nthreads < 1 || nthreads > MAX_AICPU_THREADS) { + LOG_ERROR("Invalid aicpu_thread_num: %d", nthreads); + init_failed_.store(true, std::memory_order_release); + return -1; + } + // Each thread needs a distinct index in [0, nthreads) to pick the leader and + // partition the cores. Onboard the gate filter assigns it (exec_idx); sim's + // gate does not, so platform_aicpu_affinity_thread_idx() is -1 here for every + // thread — hand those a distinct index from a counter (mirrors run()'s + // thread_idx_++ fallback) instead of collapsing them all to leader 0, which + // would run pre_/post_handshake_init on every thread and race the shared + // scheduler state. Exactly nthreads threads reach init (the gate drops the + // rest), so the counter yields a gap-free [0, nthreads). + int32_t tidx = platform_aicpu_affinity_thread_idx(); + if (tidx < 0) tidx = hs_thread_seq_.fetch_add(1, std::memory_order_acq_rel); + // A thread whose index still falls outside [0, nthreads) owns no core slice: + // handshake_partition would compute lo/hi past cores_total_num_ and index + // all_handshakes[]/core_exec_states_ out of bounds. Reject it here (mirrors + // the bounds guard already in run()). Fail only this thread and do NOT set + // init_failed_ — that would make the valid peers abort before their + // hs_arrived_ increment and hang the leader at the barrier below. + if (tidx >= nthreads) { + LOG_ERROR("AICPU affinity thread idx %d out of range [0,%d) in init", tidx, nthreads); + return -1; + } + const bool is_leader = (tidx == 0); + + if (is_leader) { + LOG_INFO("LegacyAicpuExecutor: Initializing"); + // The 0 → 1 fixup already applied above. + aicpu_thread_num_ = nthreads; + + hs_arrived_.store(0, std::memory_order_relaxed); + if (sched_ctx_.pre_handshake_init(runtime, aicpu_thread_num_, get_platform_regs()) != 0) { + init_failed_.store(true, std::memory_order_release); + hs_setup_done_.store(true, std::memory_order_release); + return -1; + } + hs_setup_done_.store(true, std::memory_order_release); + } else { + while (!hs_setup_done_.load(std::memory_order_acquire)) { + if (init_failed_.load(std::memory_order_acquire)) return -1; + SPIN_WAIT_HINT(); + } + if (init_failed_.load(std::memory_order_acquire)) return -1; + } + + // All threads: handshake this thread's slice of cores in parallel. + sched_ctx_.handshake_partition(runtime, tidx, nthreads); + + // Barrier: leader waits for every slice to finish, then completes init. + hs_arrived_.fetch_add(1, std::memory_order_acq_rel); + if (is_leader) { + while (hs_arrived_.load(std::memory_order_acquire) < nthreads) + SPIN_WAIT_HINT(); + completion_gate_.reset(); + if (sched_ctx_.post_handshake_init(runtime) != 0) { + init_failed_.store(true, std::memory_order_release); + init_done_.store(true, std::memory_order_release); + return -1; + } + init_done_.store(true, std::memory_order_release); + LOG_INFO("LegacyAicpuExecutor: Init complete"); + } else { + while (!init_done_.load(std::memory_order_acquire)) { + if (init_failed_.load(std::memory_order_acquire)) return -1; + } + if (init_failed_.load(std::memory_order_acquire)) return -1; + } + return 0; +} + +/** + * Shutdown AICore - Send exit signal via registers to all AICore kernels + */ +int32_t LegacyAicpuExecutor::run(Runtime *runtime) { + int32_t affinity_exec_idx = platform_aicpu_affinity_thread_idx(); + int32_t thread_idx = (affinity_exec_idx >= 0) ? affinity_exec_idx : (thread_idx_++); + if (thread_idx < 0 || thread_idx >= aicpu_thread_num_ || thread_idx >= MAX_AICPU_THREADS) { + LOG_ERROR( + "Thread index %d out of bounds (active=%d max=%d exec_idx=%d)", thread_idx, aicpu_thread_num_, + MAX_AICPU_THREADS, affinity_exec_idx + ); + return -1; + } + int32_t run_rc = 0; + + // Boot: the last AICPU thread (aicpu_thread_num_ - 1) performs the one-time + // host-orch attach. host_build_graph's orchestrator already ran on the host, + // and every cross-task reference it wrote is an offset from its own block, so + // the SM/arena this thread sees need no address fixup. This thread attaches + // the prebuilt arena, points the SM + // handle's task-header pointers at the device SM WITHOUT resetting the + // host-populated data, hands the host-computed task count to the scheduler, + // and releases the other threads. It then falls through and schedules its own + // cores like every other thread — host_build_graph has no device-side + // orchestrator, so there is no orch/sched split. + if (thread_idx == aicpu_thread_num_ - 1) { + void *prebuilt_arena = runtime->get_prebuilt_arena_base(); + size_t off_runtime = runtime->get_prebuilt_runtime_offset(); + + // A boot failure falls through to the common teardown at the end of + // run() — it must NOT return early. This thread owns a core slice + // (handshake_partition assigns [lo, total) to the last thread), so an + // early return would skip shutdown(thread_idx) — leaving its AICore + // cores spinning on an unclosed register window — and the completion + // gate never opens, so the host hangs into the op-execute + // timeout (507018) instead of seeing the failure. On failure: record it + // in run_rc, leave rt null so the dispatch block below skips, and still + // publish runtime_init_ready_ (single point at the block's end) so the + // peer threads stop spinning. + bool boot_ok = (prebuilt_arena != nullptr); + if (!boot_ok) { + LOG_ERROR("Thread %d: host-orch: prebuilt_arena_base is null", thread_idx); + rt = nullptr; + run_rc = -1; + } + + if (boot_ok) { + runtime_arena_.attach(prebuilt_arena, DeviceArena::kDefaultBaseAlign); + rt = reinterpret_cast(static_cast(prebuilt_arena) + off_runtime); + runtime_wire_arena_pointers(runtime_arena_, rt->prebuilt_layout, rt); + + void *sm_ptr = runtime->get_gm_sm_ptr(); + // The image the host shipped is pitched to the submitted task count, + // not to the count the table was dimensioned for, and the device region + // holds exactly that image — so its size comes from the same pitch. + // attach_populated rejects a pitch outside (0, task_capacity] and a + // region too small for it. + const uint64_t live_slots = sm_layout::live_slot_pitch(static_cast(runtime->host_total_tasks)); + const uint64_t sm_size = runtime->sm_image_bytes; + // sm_handle and the scheduler state are the device-only zone: their + // bytes never travel, so they start as whatever the pooled arena last + // held. Zeroing the handle first is what makes attach_populated's + // assignment of every field checkable here rather than by inspecting + // attach_populated. + memset(rt->sm_handle, 0, sizeof(*rt->sm_handle)); + if (!rt->sm_handle->attach_populated( + sm_ptr, sm_size, rt->prebuilt_layout.task_capacity, live_slots, runtime->sm_image_bytes + )) { + LOG_ERROR("Thread %d: host-orch: sm_handle->attach_populated failed", thread_idx); + rt = nullptr; + run_rc = -1; + boot_ok = false; + } else if (!rt->scheduler->init_data_from_layout(rt->prebuilt_layout.sched, runtime_arena_, sm_ptr)) { + LOG_ERROR("Thread %d: host-orch: scheduler init_data_from_layout failed", thread_idx); + rt = nullptr; + run_rc = -1; + boot_ok = false; + } else { + // Queue headers are set, so the slot arrays can take their ramp, + // and the mailbox ring gets its cursors and publication gates. + // All of it precedes runtime_init_ready_, which is what releases + // the peer threads into the dispatch loop, so neither a push nor a + // completion message sees an uninitialized region. + // + // Both regions sit in the device-only zone, so their bytes are + // whatever the pooled arena last held, and each one's empty state is + // whatever its own initializer writes — a ready queue's is a + // sequence ramp (slot i holds i), the mailbox's is zeroed cursors + // and publication gates. Zeroing the region is a substitute for + // neither call. + rt->scheduler->seed_queue_slots(); + rt->aicore_mailbox->init_empty(); + } + } + + if (boot_ok) { + sched_ctx_.bind_runtime(rt); + + // Latch the host-built task count (on_graph_attached sets total_tasks_) + // BEFORE the runtime_init_ready_ release below — that store is the barrier + // that unblocks the scheduler threads. Otherwise they would acquire + // runtime_init_ready_ with total_tasks_=0 and race to an early exit before + // the host task count is visible (host-orch has no concurrent orchestrator + // to keep them alive). + // NOTE: do NOT call rt_orchestration_done(rt) here. The HOST already + // called it in run_host_orchestration; the orchestrator's own + // task-allocator pointers name host memory the device never reads, so + // mark_done()'s active_count() read would dereference it and fault the + // AICPU. on_graph_attached only needs total_tasks and the scalar + // orchestrator.inline_completed_tasks, both already valid. + sched_ctx_.on_graph_attached(rt, thread_idx, runtime->host_total_tasks); + LOG_INFO("Thread %d: host-orch boot complete (%d tasks)", thread_idx, runtime->host_total_tasks); + } + + // Publish "leader setup done" (SM attached, task count latched, queues + // allocated). Every thread then classifies its slice below before any of + // them may dispatch — the leader holds runtime_init_ready_ until then. + classify_ready_.store(true, std::memory_order_release); + } + + // Parallel initial classify. Every AICPU thread waits for the leader's + // orchestration setup, seeds its disjoint slice of the whole graph's ready + // set + wake lists, then barriers. Only once all slices are done does the + // leader publish runtime_init_ready_, so no thread dispatches against a + // half-seeded graph. This replaces the O(total_tasks) serial classify the + // leader used to run alone while the others idle-waited. + while (!classify_ready_.load(std::memory_order_acquire)) { + SPIN_WAIT_HINT(); + } + if (!sched_ctx_.is_completed() && rt != nullptr) { + sched_ctx_.classify_partition(thread_idx, aicpu_thread_num_); + } + classify_arrived_.fetch_add(1, std::memory_order_acq_rel); + if (thread_idx == aicpu_thread_num_ - 1) { + while (classify_arrived_.load(std::memory_order_acquire) < aicpu_thread_num_) { + SPIN_WAIT_HINT(); + } + runtime_init_ready_.store(true, std::memory_order_release); + } else { + while (!runtime_init_ready_.load(std::memory_order_acquire)) { + SPIN_WAIT_HINT(); + } + } + + // Every AICPU thread schedules its assigned cores. + if (!sched_ctx_.is_completed()) { + if (rt == nullptr) { + LOG_ERROR("Thread %d: rt is null after a failed boot, skipping dispatch", thread_idx); + } else { + sched_ctx_.bind_runtime(rt); + // 3S+1P: the last thread is the core-less resolution (P) thread; the + // rest are core-owning schedulers (S). + int32_t completed = (thread_idx == sched_ctx_.p_thread_idx()) ? + sched_ctx_.run_resolution_thread(runtime, thread_idx) : + sched_ctx_.resolve_and_dispatch(runtime, thread_idx); + if (completed < 0) { + LOG_ERROR("Thread %d: Scheduler failed with rc=%d", thread_idx, completed); + run_rc = completed; + } else { + LOG_INFO("Thread %d: Executed %d tasks from runtime", thread_idx, completed); + } + } + } + + // Always shutdown AICore — even if sched_ctx_.completed_ was already true. + // platform_deinit_aicore_regs is idempotent. + int32_t shutdown_rc = sched_ctx_.shutdown(thread_idx); + if (shutdown_rc != 0 && run_rc == 0) { + run_rc = shutdown_rc; + } + + LOG_INFO("Thread %d: Completed", thread_idx); + + completion_gate_.arrive_and_finalize_if_last(aicpu_thread_num_, [&] { + aicpu_publish_task_timing_tail_usage(aicpu_thread_num_); + // Destroy the host_build_graph runtime. sm_handle / rt are recreated + // every run, so always tear them down here. + if (rt != nullptr) { + rt->scheduler->print_queues(); + // Clear g_current_runtime in this DSO before destroying rt. + framework_bind_runtime(nullptr); + // A Graph's expansion storage is the tail of its outer task's heap + // allocation, so it retires with that allocation; nothing here owns + // a separate block to release. + runtime_destroy(rt, runtime_arena_); + rt = nullptr; + } + }); + + return run_rc; +} + +void LegacyAicpuExecutor::deinit(Runtime *runtime) { + // 1. Invalidate AICPU cache for Runtime address range. + // Next round's Host DMA (rtMemcpy) writes fresh Runtime to HBM but + // bypasses this cache. Invalidating now ensures next round reads from HBM. + cache_invalidate_range(runtime, sizeof(Runtime)); + + // Reset all SchedulerContext-owned state in one place. + sched_ctx_.deinit(); + + completion_gate_.reset(); + runtime_init_ready_.store(false, std::memory_order_release); + + aicpu_thread_num_ = 0; + + // Clear the file-scope runtime pointer (freed by the last scheduler thread before deinit). + rt = nullptr; + + LOG_INFO("DeInit: Runtime execution state reset"); + + init_done_.store(false, std::memory_order_release); + init_failed_.store(false, std::memory_order_release); + hs_setup_done_.store(false, std::memory_order_release); + hs_arrived_.store(0, std::memory_order_release); + hs_thread_seq_.store(0, std::memory_order_release); + classify_ready_.store(false, std::memory_order_release); + classify_arrived_.store(0, std::memory_order_release); + thread_idx_.store(0, std::memory_order_release); + + LOG_INFO("DeInit: LegacyAicpuExecutor reset complete"); +} + +// ===== Public Entry Point ===== + +extern "C" int32_t legacy_aicpu_prewarm_callable(Runtime *runtime) { + // host_build_graph host-orch: the orchestration .so is dlopen'd on the HOST + // during prepare_callable_impl and the whole task graph is built host-side, + // so there is no device-side orchestrator .so to pre-load — prewarm is a + // no-op. The symbol is retained because the platform onboard kernel + // (src/a2a3/platform/onboard/aicpu/kernel.cpp) links it strongly via + // simpler_aicpu_prewarm_callable; removing it would break the onboard link. + (void)runtime; + return 0; +} + +/** + * aicpu_execute - Main AICPU kernel execution entry point + * + * This is called by DynTileFwkBackendKernelServer in kernel.cpp. + * Orchestrates the complete task runtime execution: + * 1. Initialize executor: all threads enter init(), which handshakes the cores + * in parallel and barriers internally until init is complete (or a thread + * failed); its return value is authoritative on every thread. + * 2. Execute tasks on managed cores + * 3. Cleanup when last thread finishes + * + * @param runtime Pointer to Runtime structure + * @return 0 on success, non-zero on error + */ +extern "C" int32_t legacy_aicpu_execute(Runtime *runtime) { + if (runtime == nullptr) { + LOG_ERROR("%s", "Invalid argument: null Runtime pointer"); + return -1; + } + + LOG_INFO("%s", "aicpu_execute: Starting AICPU kernel execution"); + + // init() barriers every thread internally until init is complete on the + // leader (or a thread failed), then returns the status — so a non-zero + // return is authoritative on all threads and no extra spin is needed. + if (g_aicpu_executor.init(runtime) != 0) { + LOG_ERROR("%s", "aicpu_execute: Initialization failed, aborting execution"); + return -1; + } + + int32_t rc = g_aicpu_executor.run(runtime); + if (rc != 0) { + LOG_ERROR("aicpu_execute: Thread execution failed with rc=%d", rc); + } + + int32_t runtime_rc = read_runtime_status(runtime); + + // The finalizer publishes cleanup eligibility only after runtime destruction. + if (g_aicpu_executor.completion_gate_.claim_cleanup()) { + LOG_INFO("aicpu_execute: All threads finished, cleaning up"); + g_aicpu_executor.deinit(runtime); + } + + if (runtime_rc != 0) { + LOG_ERROR("aicpu_execute: simpler runtime failed with rc=%d", runtime_rc); + return runtime_rc; + } + + if (rc != 0) { + return rc; + } + + LOG_INFO("%s", "aicpu_execute: Kernel execution completed successfully"); + return 0; +} diff --git a/src/a5/runtime/host_build_graph/build_config.py b/src/a5/runtime/host_build_graph/build_config.py index d673db4a28..5a93f120df 100644 --- a/src/a5/runtime/host_build_graph/build_config.py +++ b/src/a5/runtime/host_build_graph/build_config.py @@ -9,14 +9,9 @@ # host_build_graph Runtime build configuration # All paths are relative to this file's directory (src/runtime/host_build_graph/) # -# This is the host-orchestration variant of tensormap_and_ringbuffer: it shares -# the same scheduler, ring buffers, and shared-memory layout, differing only in -# WHEN the orchestrator runs. -# - Host runs the orchestrator to completion, populating SM + arena, then H2Ds -# the image to device (vs tensormap, where AICPU thread N-1 orchestrates on -# device concurrently with the scheduler threads) -# - AICPU threads 0..N-1 all run schedulers (no on-device orchestrator thread) -# - AICore executes tasks via an aligned DispatchPayload + pre-built dispatch_args +# The host materializes the graph and uploads a read-only view. AICore workers +# resolve dependencies and execute kernels. AICPU participates only in worker +# initialization, run completion waiting, and teardown. # # The "orchestration" directory contains source files compiled into both # runtime targets AND the orchestration .so (e.g., tensor methods needed diff --git a/src/a5/runtime/host_build_graph/docs/profiling_levels.md b/src/a5/runtime/host_build_graph/docs/profiling_levels.md index 5bac75aa6d..4a19611367 100644 --- a/src/a5/runtime/host_build_graph/docs/profiling_levels.md +++ b/src/a5/runtime/host_build_graph/docs/profiling_levels.md @@ -6,6 +6,13 @@ This document describes the profiling macro hierarchy and logging control in the The runtime uses a hierarchical profiling system with compile-time macros to control profiling code compilation and log output. The `enable_chip_swimlane` runtime flag (integer perf_level 0–4) controls data collection granularity (performance buffers, shared memory writes) but does NOT control log output. +> **A5 HBG scheduler selection.** Diagnostic flags never select the scheduler. +> Ordinary DAGs remain on the resident AICore scheduler, while Graph replay +> remains on its explicit legacy compatibility path. Until resident Resolver +> profiling lands, chip-swimlane, PMU, and argument-dump collection for +> ordinary DAGs is best-effort; artifacts may be absent or incomplete and must +> not be used as evidence of Resolver scheduling behavior or as a profiling-on +> performance baseline. > **host_build_graph (host-orch) note.** The profiling **macros** below > (`SIMPLER_DFX`, `SIMPLER_ORCH_PROFILING`, …) are shared with > `tensormap_and_ringbuffer`. But the orchestrator-timing **device-log lines** diff --git a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp index 478be50b87..c81c35d6dc 100644 --- a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp @@ -45,7 +45,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -57,6 +59,8 @@ #include "host_build_graph/common.h" #include "host_build_graph/dep_gen_host_graph.h" #include "host_build_graph/graph_execution.h" +#include "scheduler/scheduler_graph.h" +#include "scheduler/scheduler_types.h" #include "host_build_graph/host_tensor_access.h" #include "host_build_graph/graph_host_state.h" #include "host_build_graph/host_phase_trace.h" @@ -76,6 +80,7 @@ #include "common/platform_config.h" #include "common/unified_log.h" #include "host_log.h" +#include "host/platform_compile_info.h" #include "host/raii_scope_guard.h" #include "utils/device_arena.h" #include "prepare_callable_common.h" @@ -351,6 +356,16 @@ static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedM namespace { +struct SchedulerStateOwner { + void *allocation; + void *state_base; + uint64_t allocation_size; + AicoreSchedulerLayout layout; +}; + +std::mutex scheduler_state_owners_mutex; +std::unordered_map scheduler_state_owners; + // host_build_graph is host-orchestration-first: the HOST dlopens the // orchestration .so and runs it to completion. Every cross-task reference the // shared memory and arena carry is an offset or an index from its own block, so @@ -590,6 +605,340 @@ struct GraphHostStateBinding { OrchestratorState &orchestrator; }; +void release_scheduler_state(Runtime *runtime, const HostApi *api) { + if (runtime == nullptr || api == nullptr) return; + SchedulerStateOwner owner{}; + { + std::scoped_lock lock(scheduler_state_owners_mutex); + auto it = scheduler_state_owners.find(runtime); + if (it != scheduler_state_owners.end()) { + owner = it->second; + scheduler_state_owners.erase(it); + } + } + if (owner.allocation != nullptr) api->device_free(owner.allocation); + for (int32_t i = 0; i < runtime->get_worker_count(); ++i) { + runtime->workers[i].aicpu_ready = 0; + runtime->workers[i].task = 0; + } +} + +void select_legacy_scheduler(Runtime *runtime, uint32_t mode) { + always_assert( + mode == SCHEDULER_RUNTIME_MODE_LEGACY_GRAPH || mode == SCHEDULER_RUNTIME_MODE_LEGACY_UNSUPPORTED_SHAPE + ); + for (int32_t i = 0; i < runtime->get_worker_count(); ++i) { + runtime->workers[i].aicpu_ready = mode; + runtime->workers[i].task = 0; + } +} + +bool create_scheduler_state( + Runtime *runtime, const HostApi *api, SharedMemoryHandle &host_sm_handle, int32_t total_tasks, + uint64_t task_window_size, const sm_layout::SegmentOffsets &device_segments +) { + release_scheduler_state(runtime, api); + if (total_tasks < 0 || task_window_size == 0 || static_cast(total_tasks) > task_window_size) { + LOG_ERROR( + "A5 HBG AICore scheduler: invalid graph size tasks=%d window=%" PRIu64, total_tasks, task_window_size + ); + return false; + } + SchedulerGraphView host_graph{ + reinterpret_cast(host_sm_handle.header->tasks.task_storage), + 0, + static_cast(total_tasks), + task_window_size - 1, + }; + for (int64_t task_id = 0; task_id < total_tasks; ++task_id) { + ChipTaskSlotState &slot = host_sm_handle.header->tasks.get_slot_state_by_task_id(task_id); + if (slot.task_kind == TaskKind::GRAPH) { + select_legacy_scheduler(runtime, SCHEDULER_RUNTIME_MODE_LEGACY_GRAPH); + LOG_INFO("A5 HBG: retaining AICPU scheduling for a graph-execution run"); + return true; + } + } + std::vector inline_completed_task_ids; + std::vector task_metadata(static_cast(total_tasks)); + uint64_t aic_task_count = 0; + uint64_t aiv_task_count = 0; + uint64_t executable_task_count = 0; + uint64_t executable_subtask_count = 0; + uint64_t gang_task_count = 0; + uint64_t aic_worker_demand = 0; + uint64_t aiv_worker_demand = 0; + int64_t legacy_shape_task_id = -1; + for (int64_t task_id = 0; task_id < total_tasks; ++task_id) { + ChipTaskSlotState &slot = host_sm_handle.header->tasks.get_slot_state_by_task_id(task_id); + SchedulerTaskShape shape{}; + SchedulerGraphResult status = scheduler_classify_task_shape(host_graph, task_id, &shape); + bool inline_dispatch_task = false; + if (status != SchedulerGraphResult::OK) { + bool inline_completed_task = status == SchedulerGraphResult::UNSUPPORTED_SHAPE && + slot.active_mask.raw() == 0 && slot.logical_block_num == 1 && + slot.total_required_subtasks == 0 && + slot.task_state.load(std::memory_order_acquire) == CHIP_TASK_COMPLETED && + slot.task_attrs.allow_early_resolve() && + !slot.task_attrs.requires_sync_start() && !slot.task_attrs.has_predicate(); + if (inline_completed_task) { + inline_completed_task_ids.push_back(task_id); + continue; + } + inline_dispatch_task = status == SchedulerGraphResult::UNSUPPORTED_SHAPE && slot.active_mask.raw() == 0 && + slot.logical_block_num == 1 && slot.total_required_subtasks == 0 && + slot.task_state.load(std::memory_order_acquire) == CHIP_TASK_PENDING && + !slot.task_attrs.requires_sync_start() && !slot.task_attrs.has_predicate(); + if (!inline_dispatch_task) { + LOG_ERROR( + "A5 HBG AICore scheduler: invalid v0 task id=%" PRId64 " status=%" PRIu64, task_id, + static_cast(status) + ); + return false; + } + shape.task_id = task_id; + shape.active_mask = SUBTASK_MASK_AIV0; + } + // Ordinary tasks retain the existing ChipTaskSlotState scheduling + // semantics. Only the legacy inline-dispatch fallback synthesizes its + // AIV0 execution shape because it has no active subtask in the slot. + const uint8_t classified_active_mask = shape.active_mask; + const ActiveMask metadata_active_mask = + inline_dispatch_task ? ActiveMask(classified_active_mask) : slot.active_mask; + const uint8_t active_mask = metadata_active_mask.core_mask(); + const uint32_t active_subtasks = static_cast(__builtin_popcount(active_mask)); + if (slot.logical_block_num <= 0) { + LOG_ERROR( + "A5 HBG AICore scheduler: task id=%" PRId64 " has invalid block_num=%d", task_id, slot.logical_block_num + ); + return false; + } + const uint32_t logical_block_num = static_cast(slot.logical_block_num); + if (active_subtasks == 0 || logical_block_num > UINT16_MAX / active_subtasks) { + LOG_ERROR( + "A5 HBG AICore scheduler: task id=%" PRId64 " block/subtask product exceeds scheduler state capacity", + task_id + ); + return false; + } + const uint32_t expected_subtasks = logical_block_num * active_subtasks; + if ((!inline_dispatch_task && (slot.active_mask.raw() != classified_active_mask || + static_cast(slot.total_required_subtasks) != expected_subtasks)) || + expected_subtasks > UINT16_MAX || slot.task_state.load(std::memory_order_acquire) != CHIP_TASK_PENDING || + (slot.task_attrs.has_predicate() && (active_subtasks != 1 || logical_block_num != 1))) { + LOG_ERROR( + "A5 HBG AICore scheduler: task id=%" PRId64 + " has inconsistent shape/state or uses predicate/early-resolve", + task_id + ); + return false; + } + for (uint32_t subtask_slot = 0; subtask_slot < 3; ++subtask_slot) { + if ((active_mask & (1U << subtask_slot)) == 0 || inline_dispatch_task) continue; + const int32_t kernel_id = shape.kernel_ids[subtask_slot]; + if (kernel_id < 0 || kernel_id >= RUNTIME_MAX_FUNC_ID || runtime->get_function_bin_addr(kernel_id) == 0) { + LOG_ERROR( + "A5 HBG AICore scheduler: task id=%" PRId64 " kernel id %d has no registered callable", task_id, + kernel_id + ); + return false; + } + } + __gm__ uint8_t *payload = scheduler_graph_payload(host_graph, task_id); + const auto *predicate = + reinterpret_cast(payload + SCHEDULER_GRAPH_PREDICATE_OFFSET); + if (!scheduler_dispatch_predicate_metadata_valid(predicate->addr, predicate->elem_size, predicate->op)) { + LOG_ERROR("A5 HBG AICore scheduler: task id=%" PRId64 " has malformed predicate metadata", task_id); + return false; + } + int32_t fanin_count = *reinterpret_cast<__gm__ int32_t *>(payload + SCHEDULER_GRAPH_FANIN_COUNT_OFFSET); + if (fanin_count < 0) { + LOG_ERROR("A5 HBG AICore scheduler: task id=%" PRId64 " has invalid fanin count %d", task_id, fanin_count); + return false; + } + if (slot.task_attrs.requires_sync_start() && logical_block_num == 1) { + LOG_ERROR("A5 HBG AICore scheduler: sync-start task id=%" PRId64 " must have block_num > 1", task_id); + return false; + } + if (!scheduler_resident_v0_task_shape_supported( + active_subtasks, logical_block_num, slot.task_attrs.requires_sync_start() + )) { + if (legacy_shape_task_id < 0) legacy_shape_task_id = task_id; + } + SchedulerTaskMetadata &metadata = task_metadata[static_cast(task_id)]; + for (uint32_t subtask_slot = 0; subtask_slot < 3; ++subtask_slot) { + metadata.kernel_ids[subtask_slot] = (active_mask & (1U << subtask_slot)) != 0 ? + static_cast(shape.kernel_ids[subtask_slot]) : + UINT16_MAX; + } + metadata.active_mask = active_mask; + metadata.flags = scheduler_task_metadata_flags_from_submit_state( + metadata_active_mask, slot.task_attrs, slot.logical_block_num, fanin_count != 0, inline_dispatch_task + ); + metadata.logical_block_num = static_cast(logical_block_num); + metadata.total_required_subtasks = static_cast(expected_subtasks); + metadata.timing_slot = slot.task_attrs.timing_slot(); + if ((active_mask & 1U) != 0) { + ++aic_task_count; + aic_worker_demand = std::max(aic_worker_demand, logical_block_num); + } + const uint32_t active_aiv_subtasks = active_subtasks - ((active_mask & 1U) != 0 ? 1U : 0U); + if (active_aiv_subtasks != 0) { + ++aiv_task_count; + aiv_worker_demand = std::max(aiv_worker_demand, logical_block_num * active_aiv_subtasks); + } + if (scheduler_task_is_gang(metadata.flags)) ++gang_task_count; + executable_subtask_count += expected_subtasks; + ++executable_task_count; + } + + if (executable_task_count + inline_completed_task_ids.size() != static_cast(total_tasks)) { + LOG_ERROR("A5 HBG AICore scheduler: task metadata does not cover the graph"); + return false; + } + if (legacy_shape_task_id >= 0) { + select_legacy_scheduler(runtime, SCHEDULER_RUNTIME_MODE_LEGACY_UNSUPPORTED_SHAPE); + LOG_INFO( + "A5 HBG: retaining AICPU scheduling for task id=%" PRId64 " with MIX, SPMD, or sync-start shape", + legacy_shape_task_id + ); + return true; + } + + AicoreSchedulerLayout layout{}; + if (!scheduler_plan_layout(static_cast(total_tasks), aic_task_count, aiv_task_count, &layout) || + layout.total_size > std::numeric_limits::max() - (SCHEDULER_STATE_ALIGNMENT - 1)) { + LOG_ERROR("A5 HBG AICore scheduler: scheduler state layout overflow"); + return false; + } + layout.executable_task_count = executable_task_count; + layout.executable_subtask_count = executable_subtask_count; + layout.gang_task_count = gang_task_count; + layout.aic_worker_demand = aic_worker_demand; + layout.aiv_worker_demand = aiv_worker_demand; + + const uint64_t allocation_size = layout.total_size + SCHEDULER_STATE_ALIGNMENT - 1; + void *allocation = api->device_malloc(static_cast(allocation_size)); + if (allocation == nullptr) { + LOG_ERROR("A5 HBG AICore scheduler: failed to allocate %" PRIu64 " scheduler state bytes", allocation_size); + return false; + } + const uintptr_t aligned_address = (reinterpret_cast(allocation) + SCHEDULER_STATE_ALIGNMENT - 1) & + ~(static_cast(SCHEDULER_STATE_ALIGNMENT) - 1); + + std::vector storage(static_cast(allocation_size)); + const uintptr_t host_aligned_address = + (reinterpret_cast(storage.data()) + SCHEDULER_STATE_ALIGNMENT - 1) & + ~(static_cast(SCHEDULER_STATE_ALIGNMENT) - 1); + void *host_base = reinterpret_cast(host_aligned_address); + if (!scheduler_init_data_from_layout(host_base, layout)) { + api->device_free(allocation); + LOG_ERROR("A5 HBG AICore scheduler: failed to initialize scheduler state"); + return false; + } + + auto *task_controls = scheduler_state_at(host_base, layout.task_controls_offset); + for (int64_t task_id : inline_completed_task_ids) { + task_controls[task_id].state = static_cast(SchedulerTaskState::DONE); + task_controls[task_id].wake_list_head = SCHEDULER_WAKE_LIST_CLOSED; + } + static_assert( + SCHEDULER_CALLABLE_CAPACITY == RUNTIME_MAX_FUNC_ID, + "scheduler state callable table must cover the runtime table" + ); + auto *callable_addresses = scheduler_state_at(host_base, layout.callable_addresses_offset); + const bool cpu_sim = std::strcmp(get_platform(), "a5sim") == 0; + for (uint32_t func_id = 0; func_id < SCHEDULER_CALLABLE_CAPACITY; ++func_id) { + const uint64_t callable_address = runtime->get_function_bin_addr(static_cast(func_id)); + callable_addresses[func_id] = + callable_address == 0 ? + 0 : + (cpu_sim ? reinterpret_cast(callable_address)->resolved_addr() : + callable_address + CoreCallable::binary_data_offset()); + } + auto *metadata = scheduler_state_at(host_base, layout.task_metadata_offset); + std::copy(task_metadata.begin(), task_metadata.end(), metadata); + + const uint64_t device_sm_address = reinterpret_cast(runtime->get_gm_sm_ptr()); + auto *run_control = scheduler_state_at(host_base, layout.run_control_offset); + run_control->expected_task_count = static_cast(total_tasks); + run_control->inline_completed_count = inline_completed_task_ids.size(); + run_control->aic_task_count = aic_task_count; + run_control->aiv_task_count = aiv_task_count; + run_control->aic_worker_demand = aic_worker_demand; + run_control->aiv_worker_demand = aiv_worker_demand; + run_control->dispatch_payloads_offset = layout.dispatch_payloads_offset; + run_control->task_metadata_offset = layout.task_metadata_offset; + run_control->ready_inboxes_offset = layout.ready_inboxes_offset; + run_control->ready_directory_offset = layout.ready_directory_offset; + run_control->gang_coordinator_offset = layout.gang_coordinator_offset; + run_control->gang_cohorts_offset = layout.gang_cohorts_offset; + run_control->scheduler_timeout_cycles = SCHEDULER_TIMEOUT_CYCLES; + run_control->error_task_id = UINT64_MAX; + run_control->error_core_id = UINT64_MAX; + run_control->error_core_type = UINT64_MAX; + auto *gang_coordinator = scheduler_state_at(host_base, layout.gang_coordinator_offset); + gang_coordinator->gang_task_count = gang_task_count; + + auto *contexts = scheduler_state_at(host_base, layout.worker_contexts_offset); + int32_t aic_rank = 0; + int32_t aiv_rank = 0; + for (int32_t i = 0; i < runtime->get_worker_count(); ++i) { + SchedulerWorkerContext &context = contexts[i]; + context.core_type = static_cast(runtime->workers[i].core_type); + context.physical_core_id = -1; + context.type_rank = context.core_type == static_cast(CoreType::AIC) ? aic_rank++ : aiv_rank++; + context.active = 0; + context.run_control_offset = layout.run_control_offset; + context.task_controls_offset = layout.task_controls_offset; + context.completion_inboxes_offset = layout.completion_inboxes_offset; + context.task_metadata_offset = layout.task_metadata_offset; + context.aicpu_lifecycle_traces_offset = layout.aicpu_lifecycle_traces_offset; + context.ready_inboxes_offset = layout.ready_inboxes_offset; + context.ready_owner_states_offset = layout.ready_owner_states_offset; + context.ready_directory_offset = layout.ready_directory_offset; + context.worker_contexts_offset = layout.worker_contexts_offset; + context.dispatch_slots_offset = layout.dispatch_slots_offset; + context.callable_addresses_offset = layout.callable_addresses_offset; + context.runtime_worker_count = static_cast(runtime->get_worker_count()); + context.bootstrap_done = 0; + context.gang_coordinator_offset = layout.gang_coordinator_offset; + context.gang_cohorts_offset = layout.gang_cohorts_offset; + context.gang_participants_offset = layout.gang_participants_offset; + context.gang_commands_offset = layout.gang_commands_offset; + context.graph_storage_address = device_sm_address + device_segments.storage; + context.graph_reserved_address = 0; + context.scheduler_state_base_address = aligned_address; + context.dispatch_payload_offset = layout.dispatch_payloads_offset + static_cast(i) * + SCHEDULER_PENDING_SLOT_COUNT * + sizeof(DispatchPayload); + context.trace_cells_offset = layout.trace_cells_offset; + context.task_window_last_index = task_window_size - 1; + context.graph_task_count = static_cast(total_tasks); + context.worker_index = static_cast(i); + } + + if (api->copy_to_device( + reinterpret_cast(aligned_address), host_base, static_cast(layout.total_size) + ) != 0) { + api->device_free(allocation); + LOG_ERROR("A5 HBG AICore scheduler: failed to publish scheduler state"); + return false; + } + for (int32_t i = 0; i < runtime->get_worker_count(); ++i) { + runtime->workers[i].aicpu_ready = SCHEDULER_RUNTIME_MODE_RESIDENT_PENDING; + runtime->workers[i].task = + aligned_address + layout.worker_contexts_offset + static_cast(i) * sizeof(SchedulerWorkerContext); + } + { + std::scoped_lock lock(scheduler_state_owners_mutex); + scheduler_state_owners.emplace( + runtime, SchedulerStateOwner{allocation, reinterpret_cast(aligned_address), allocation_size, layout} + ); + } + LOG_INFO("A5 HBG: selected resident AICore scheduling for %d tasks", total_tasks); + return true; +} + int32_t run_host_orchestration( Runtime *runtime, const HostApi *api, HostTensorAccessor &tensor_access, RuntimeContext *rt, DeviceArena &host_arena, const RuntimeArenaLayout &layout, uint64_t sm_size, uint64_t task_capacity, @@ -945,6 +1294,11 @@ int32_t run_host_orchestration( ); always_assert(compacted == image_bytes); + const sm_layout::SegmentOffsets device_segments = sm_layout::segment_offsets(sm_layout::image_extents(bind_usage)); + if (!create_scheduler_state(runtime, api, host_sm_handle, total_tasks, task_capacity, device_segments)) { + return PTO_RUNTIME_ERR_INTERNAL; + } + const BindPhaseMark h2d_phase = bind_phase_begin(); if (api->copy_to_device(arena_dev + layout.off_copied_begin, upload_base, upload_bytes) != 0) { LOG_ERROR("host-orch: H2D of the runtime image failed"); @@ -1438,6 +1792,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e } } LOG_INFO("Freed %d device allocations", tensor_pair_count); + release_scheduler_state(runtime, api); // The dispatch table is owned by bind_callable_to_runtime, which clears it // before replaying the active callable's addresses. The chip-callable device diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_graph.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_graph.h index 867af299a5..44a401895a 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_graph.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_graph.h @@ -52,8 +52,14 @@ enum class SchedulerGraphResult : uint64_t { INVALID_CALLABLE = 7, INVALID_FANIN_COUNT = 8, INVALID_FANIN_ID = 9, + TIMEOUT = 10, }; +inline constexpr bool +scheduler_resident_v0_task_shape_supported(uint32_t active_subtasks, uint32_t logical_block_num, bool sync_start) { + return active_subtasks == 1 && logical_block_num == 1 && !sync_start; +} + struct SchedulerGraphView { // Base of the ChipTaskStorage array. One address, because a task's descriptor // and payload are members of one entry rather than peers in two arrays. @@ -66,7 +72,7 @@ struct SchedulerGraphView { // Kept in the wire view for diagnostics. HBG task ids directly index the // whole-graph-resident tables and never wrap, so this value must not be // used to mask an id (the configured capacity need not be a power of two). - uint64_t task_window_mask; + uint64_t task_window_last_index; }; struct SchedulerTaskInfo { @@ -96,6 +102,13 @@ static_assert(sizeof(SchedulerTaskInfo) == 24, "root classification result layou static_assert(sizeof(SchedulerTaskShape) == 24, "task shape result layout changed"); static_assert(sizeof(SchedulerDispatchPredicate) == 24, "dispatch predicate layout changed"); +inline __aicore__ bool +scheduler_dispatch_predicate_metadata_valid(uint64_t address, uint8_t element_size, uint8_t operation) { + if (operation == 0) return true; + const bool valid_width = element_size == 1 || element_size == 2 || element_size == 4 || element_size == 8; + return operation <= 6 && valid_width && address != 0 && (address & (static_cast(element_size) - 1)) == 0; +} + inline __aicore__ __gm__ uint8_t *scheduler_graph_storage(const SchedulerGraphView &graph, int64_t task_id) { uint64_t slot = static_cast(task_id); return reinterpret_cast<__gm__ uint8_t *>(graph.storage_address) + slot * SCHEDULER_GRAPH_TASK_STORAGE_STRIDE; diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_layout.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_layout.h new file mode 100644 index 0000000000..3f46fb9cb2 --- /dev/null +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_layout.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include + +constexpr uint32_t SCHEDULER_RUNTIME_MODE_RESIDENT_PENDING = 1; +constexpr uint32_t SCHEDULER_RUNTIME_MODE_RESIDENT_READY = 2; +// A zeroed Handshake is not a scheduler choice. Every supported fallback is +// published explicitly so an ordinary run cannot silently drift back to the +// legacy dependency scheduler while continuing to pass result-only tests. +constexpr uint32_t SCHEDULER_RUNTIME_MODE_LEGACY_GRAPH = 3; +constexpr uint32_t SCHEDULER_RUNTIME_MODE_LEGACY_UNSUPPORTED_SHAPE = 4; + +struct AicoreSchedulerLayout { + uint64_t total_size; + uint64_t task_count; + uint64_t aic_task_count; + uint64_t aiv_task_count; + uint64_t run_control_offset; + uint64_t aicpu_lifecycle_traces_offset; + uint64_t worker_contexts_offset; + uint64_t dispatch_payloads_offset; + uint64_t dispatch_slots_offset; + uint64_t callable_addresses_offset; + uint64_t task_metadata_offset; + uint64_t task_controls_offset; + uint64_t completion_inboxes_offset; + uint64_t ready_inboxes_offset; + uint64_t ready_owner_states_offset; + uint64_t ready_directory_offset; + uint64_t trace_cells_offset; + uint64_t gang_coordinator_offset; + uint64_t gang_cohorts_offset; + uint64_t gang_participants_offset; + uint64_t gang_commands_offset; + uint64_t executable_task_count; + uint64_t executable_subtask_count; + uint64_t gang_task_count; + uint64_t aic_worker_demand; + uint64_t aiv_worker_demand; +}; diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_ready.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_ready.h index 59995904b1..8118a07cc2 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_ready.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_ready.h @@ -151,10 +151,7 @@ scheduler_evaluate_task_predicate(const SchedulerGraphView &graph, int64_t task_ reinterpret_cast<__gm__ SchedulerDispatchPredicate *>(payload + SCHEDULER_GRAPH_PREDICATE_OFFSET); scheduler_observe_cache_line(predicate); if (predicate->op == 0) return SchedulerPredicateResult::PASS; - if (predicate->op > 6 || - (predicate->elem_size != 1 && predicate->elem_size != 2 && predicate->elem_size != 4 && - predicate->elem_size != 8) || - predicate->addr == 0 || (predicate->addr & (static_cast(predicate->elem_size) - 1)) != 0) + if (!scheduler_dispatch_predicate_metadata_valid(predicate->addr, predicate->elem_size, predicate->op)) return SchedulerPredicateResult::MALFORMED; __gm__ void *operand = reinterpret_cast<__gm__ void *>(predicate->addr); @@ -311,7 +308,7 @@ inline __aicore__ void scheduler_record_error( if (graph != nullptr) { scheduler_gm_store(run_control->error_graph_task_count, graph->task_count); scheduler_gm_store(run_control->error_storage_address, graph->storage_address); - scheduler_gm_store(run_control->error_task_window_mask, graph->task_window_mask); + scheduler_gm_store(run_control->error_task_window_last_index, graph->task_window_last_index); } if (context != nullptr) { scheduler_gm_store(run_control->error_core_id, static_cast(context->physical_core_id)); diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h index 5ad82c6889..e356e1b872 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h @@ -17,6 +17,7 @@ #include "common/core_type.h" #include "common/platform_config.h" #include "dispatch_payload.h" +#include "scheduler_layout.h" #ifndef __gm__ #define __gm__ @@ -893,6 +894,7 @@ enum class SchedulerErrorSite : uint64_t { DISPATCH_INVALID_CALLABLE = 45, DISPATCH_MATERIALIZE_FAILED = 46, DISPATCH_INVALID_PREDICATE = 47, + EXECUTOR_INVALID_DISPATCH_SLOT = 50, BOOTSTRAP_WAKE_INVALID_HEAD = 60, COMPLETION_TASK_NOT_DONE = 61, COMPLETION_WAKE_ALREADY_CLOSED = 62, @@ -907,21 +909,34 @@ enum class SchedulerErrorSite : uint64_t { DEFERRED_RESERVATION_INVALID_STATE = 76, DEFERRED_PUBLISH_INVALID_RESERVATION = 77, NORMAL_DISPATCH_INVALID_TOPOLOGY = 78, + EXECUTOR_PREFERRED_SLOT_INVALID = 79, + READY_OWNER_MAINTENANCE_FAILED = 80, + BOOTSTRAP_FAILED = 90, + CONTEXT_READY_TIMEOUT = 91, + BOOTSTRAP_SCAN_TIMEOUT = 92, + BOOTSTRAP_COMPLETE_TIMEOUT = 93, + REGISTER_RELEASE_TIMEOUT = 94, + EXECUTION_PROGRESS_TIMEOUT = 95, + EXIT_WAIT_TIMEOUT = 96, + AICPU_HANDSHAKE_TIMEOUT = 100, + AICPU_INIT_BARRIER_TIMEOUT = 101, + AICPU_SHUTDOWN_BARRIER_TIMEOUT = 102, }; struct alignas(128) SchedulerRunControl { - uint64_t config_reserved_prefix[2]; + uint64_t aic_task_count; + uint64_t aiv_task_count; uint64_t active_worker_count; uint64_t expected_task_count; uint64_t inline_completed_count; uint64_t aic_active_worker_count; uint64_t aiv_active_worker_count; - uint64_t dispatch_reserved; + uint64_t aic_worker_demand; volatile uint64_t dispatch_payloads_offset; uint64_t task_metadata_offset; uint64_t ready_inboxes_offset; uint64_t ready_directory_offset; - uint64_t directory_reserved; + uint64_t aiv_worker_demand; uint64_t gang_coordinator_offset; uint64_t gang_cohorts_offset; uint64_t resolver_count; @@ -939,7 +954,8 @@ struct alignas(128) SchedulerRunControl { uint64_t error_poll_count; volatile uint64_t bootstrap_scan_arrived_count; volatile uint64_t bootstrap_scan_complete; - uint64_t lifecycle_reserved[3]; + volatile uint64_t scheduler_timeout_cycles; + uint64_t lifecycle_reserved[2]; volatile uint64_t error_claimed; volatile uint64_t scheduler_error; @@ -949,7 +965,7 @@ struct alignas(128) SchedulerRunControl { volatile uint64_t error_graph_task_count; volatile uint64_t error_storage_address; volatile uint64_t error_reserved_address; - volatile uint64_t error_task_window_mask; + volatile uint64_t error_task_window_last_index; volatile uint64_t error_site; uint64_t error_reserved[6]; }; @@ -1009,13 +1025,13 @@ struct alignas(128) SchedulerWorkerContext { volatile uint64_t scheduler_state_base_address; volatile uint64_t dispatch_payload_offset; volatile uint64_t trace_cells_offset; - volatile uint64_t task_window_mask; + volatile uint64_t task_window_last_index; volatile uint64_t graph_task_count; volatile uint64_t worker_index; volatile uint64_t completion_inboxes_offset; volatile uint64_t inbox_index; volatile uint64_t ready_owner_states_offset; - uint64_t runtime_offset_padding; + volatile uint64_t aicpu_lifecycle_traces_offset; volatile uint64_t task_metadata_offset; volatile uint64_t ready_inboxes_offset; @@ -1141,35 +1157,6 @@ struct alignas(128) SchedulerTaskTrace { uint64_t inter_task_dispatch_publish_cycles[SCHEDULER_CORE_TYPE_COUNT]; }; -struct AicoreSchedulerLayout { - uint64_t total_size; - uint64_t task_count; - uint64_t aic_task_count; - uint64_t aiv_task_count; - uint64_t run_control_offset; - uint64_t aicpu_lifecycle_traces_offset; - uint64_t worker_contexts_offset; - uint64_t dispatch_payloads_offset; - uint64_t dispatch_slots_offset; - uint64_t callable_addresses_offset; - uint64_t task_metadata_offset; - uint64_t task_controls_offset; - uint64_t completion_inboxes_offset; - uint64_t ready_inboxes_offset; - uint64_t ready_owner_states_offset; - uint64_t ready_directory_offset; - uint64_t trace_cells_offset; - uint64_t gang_coordinator_offset; - uint64_t gang_cohorts_offset; - uint64_t gang_participants_offset; - uint64_t gang_commands_offset; - uint64_t executable_task_count; - uint64_t executable_subtask_count; - uint64_t gang_task_count; - uint64_t aic_worker_demand; - uint64_t aiv_worker_demand; -}; - static_assert(sizeof(SchedulerTaskMetadata) == 16, "task metadata layout changed"); static_assert(alignof(SchedulerTaskMetadata) == 16, "task metadata alignment changed"); static_assert(sizeof(SchedulerTaskControl) == 128, "task control layout changed"); @@ -1250,6 +1237,7 @@ inline __aicore__ __gm__ T *scheduler_state_at(__gm__ void *base, uint64_t offse #if !defined(__CCE_AICORE__) #include +static_assert(std::is_standard_layout_v && std::is_trivially_copyable_v); static_assert(std::is_standard_layout_v && std::is_trivially_copyable_v); static_assert(std::is_standard_layout_v && std::is_trivially_copyable_v); static_assert( diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_watchdog.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_watchdog.h new file mode 100644 index 0000000000..665a383051 --- /dev/null +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_watchdog.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include + +#ifndef __aicore__ +#define __aicore__ +#endif + +inline __aicore__ bool scheduler_watchdog_expired(uint64_t start, uint64_t now, uint64_t timeout_cycles) { + return timeout_cycles != 0 && now - start >= timeout_cycles; +} diff --git a/tests/st/a5/host_build_graph/dfx/chip_swimlane/kernels/aiv/kernel_noop.cpp b/tests/st/a5/host_build_graph/dfx/chip_swimlane/kernels/aiv/kernel_noop.cpp index 6a73f725c4..70d8d0a366 100644 --- a/tests/st/a5/host_build_graph/dfx/chip_swimlane/kernels/aiv/kernel_noop.cpp +++ b/tests/st/a5/host_build_graph/dfx/chip_swimlane/kernels/aiv/kernel_noop.cpp @@ -12,6 +12,9 @@ #include #include +#include "intrinsic.h" +#include "tensor.h" + #ifndef __gm__ #define __gm__ #endif @@ -20,4 +23,10 @@ #define __aicore__ [aicore] #endif -extern "C" __aicore__ void kernel_entry(__gm__ int64_t *args) { (void)args; } +extern "C" __aicore__ void kernel_entry(__gm__ int64_t *args) { + __gm__ Tensor *tensor = reinterpret_cast<__gm__ Tensor *>(args[0]); + __gm__ int32_t *output = reinterpret_cast<__gm__ int32_t *>(tensor->buffer.addr) + tensor->start_offset; + output[0] = 1; + dcci(&output[0], cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_OUT); + dsb((mem_dsb_t)0); +} diff --git a/tests/st/a5/host_build_graph/dfx/chip_swimlane/kernels/orchestration/scheduler_phases_orch.cpp b/tests/st/a5/host_build_graph/dfx/chip_swimlane/kernels/orchestration/scheduler_phases_orch.cpp index ec912b9f57..1a95d8d9bf 100644 --- a/tests/st/a5/host_build_graph/dfx/chip_swimlane/kernels/orchestration/scheduler_phases_orch.cpp +++ b/tests/st/a5/host_build_graph/dfx/chip_swimlane/kernels/orchestration/scheduler_phases_orch.cpp @@ -30,7 +30,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip const simpler::hbg::Tensor &input = args.tensor(0).ref(); CoreTaskArgs normal_args; - normal_args.add_input(input); + normal_args.add_inout(input); normal_args.launch_spec.set_block_num(1); const TaskId normal_task = rt_submit_aiv_task(kNoopKernel, normal_args).task_id(); diff --git a/tests/st/a5/host_build_graph/dfx/chip_swimlane/test_scheduler_phases.py b/tests/st/a5/host_build_graph/dfx/chip_swimlane/test_scheduler_phases.py index b874c46d45..63be463304 100644 --- a/tests/st/a5/host_build_graph/dfx/chip_swimlane/test_scheduler_phases.py +++ b/tests/st/a5/host_build_graph/dfx/chip_swimlane/test_scheduler_phases.py @@ -10,14 +10,10 @@ from __future__ import annotations -import time - import torch from simpler.task_interface import ArgDirection as D from simpler_setup import SceneTestCase, TaskArgsBuilder, TensorArg, scene_test -from simpler_setup.scene_test import _outputs_dir, _sanitize_for_filename -from simpler_setup.tools.swimlane_converter import read_perf_data @scene_test(level=2, runtime="host_build_graph") @@ -26,14 +22,14 @@ class TestSchedulerPhases(SceneTestCase): "orchestration": { "source": "kernels/orchestration/scheduler_phases_orch.cpp", "function_name": "aicpu_orchestration_entry", - "signature": [D.IN], + "signature": [D.INOUT], }, "incores": [ { "func_id": 0, "source": "kernels/aiv/kernel_noop.cpp", "core_type": "aiv", - "signature": [D.IN], + "signature": [D.INOUT], }, ], } @@ -53,45 +49,7 @@ def generate_args(self, params): ) def compute_golden(self, args, params): - pass - - def test_run(self, st_platform, st_worker, request): - run_marker = int(time.time()) - super().test_run(st_platform, st_worker, request) - if self._effective_enable_chip_swimlane(request) < 3: - return - - for case in self._matching_cases(st_platform, request): - case_label = _sanitize_for_filename(f"TestSchedulerPhases_{case['name']}") - matches = [p for p in _outputs_dir().glob(f"{case_label}_*") if p.stat().st_mtime >= run_marker] - assert matches, f"no output directory created for {case_label}" - perf_path = max(matches, key=lambda p: p.stat().st_mtime) / "chip_swimlane_records.json" - assert perf_path.exists(), f"missing chip swimlane artifact: {perf_path}" - - data = read_perf_data(perf_path) - phase_threads = data.get("aicpu_scheduler_phases") - assert phase_threads, "scheduler phase records are missing" - assigned_threads = {thread_idx for thread_idx in data.get("core_to_thread", []) if thread_idx >= 0} - resolution_threads = [ - records - for thread_idx, records in enumerate(phase_threads) - if records and thread_idx not in assigned_threads - ] - assert len(resolution_threads) == 1, f"expected one core-less P thread, found {len(resolution_threads)}" - resolution_thread = resolution_threads[0] - required = {"resolve_standalone", "dummy"} - emitted = {record.get("phase") for record in resolution_thread} - assert required <= emitted, f"missing P-thread phases: {sorted(required - emitted)}" - - records = [record for record in resolution_thread if record.get("phase") in required] - assert all(record["loop_iter"] > 0 for record in records) - assert all(record["end_time_us"] >= record["start_time_us"] for record in records) - assert sum(record["tasks_processed"] for record in records if record["phase"] == "resolve_standalone") >= 1 - assert sum(record["tasks_processed"] for record in records if record["phase"] == "dummy") == 1 - assert len(resolution_thread) < 64, "P-thread phase aggregation produced excessive records" - - ordered = sorted(records, key=lambda record: (record["start_time_us"], record["end_time_us"])) - assert all(left["end_time_us"] <= right["start_time_us"] for left, right in zip(ordered, ordered[1:])) + args.input[0] = 1 if __name__ == "__main__": diff --git a/tests/st/a5/host_build_graph/empty_lifecycle/kernels/orchestration/empty_orch.cpp b/tests/st/a5/host_build_graph/empty_lifecycle/kernels/orchestration/empty_orch.cpp new file mode 100644 index 0000000000..a818a50a41 --- /dev/null +++ b/tests/st/a5/host_build_graph/empty_lifecycle/kernels/orchestration/empty_orch.cpp @@ -0,0 +1,23 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "orchestration_api.h" // NOLINT(build/include_subdir) + +extern "C" { + +__attribute__((visibility("default"))) OrchestrationConfig aicpu_orchestration_config(const ChipTaskArgs &args) { + (void)args; + return OrchestrationConfig{.expected_arg_count = 0}; +} + +__attribute__((visibility("default"))) void aicpu_orchestration_entry(const ChipTaskArgs &args) { (void)args; } + +} // extern "C" diff --git a/tests/st/a5/host_build_graph/empty_lifecycle/test_empty_lifecycle.py b/tests/st/a5/host_build_graph/empty_lifecycle/test_empty_lifecycle.py new file mode 100644 index 0000000000..c9de37f7e1 --- /dev/null +++ b/tests/st/a5/host_build_graph/empty_lifecycle/test_empty_lifecycle.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Gate for the A5 HBG empty-graph resident lifecycle.""" + +from simpler_setup import SceneTestCase, TaskArgsBuilder, scene_test + + +@scene_test(level=2, runtime="host_build_graph") +class TestHbgEmptyLifecycle(SceneTestCase): + CALLABLE = { + "orchestration": { + "source": "kernels/orchestration/empty_orch.cpp", + "function_name": "aicpu_orchestration_entry", + "signature": [], + }, + "incores": [], + } + + CASES = [ + { + "name": "empty_lifecycle", + "platforms": ["a5sim", "a5"], + # Exercise the minimum supported AICPU population during a no-task shutdown. + "config": {"aicpu_thread_num": 2}, + "params": {}, + }, + ] + + def generate_args(self, params): + return TaskArgsBuilder() + + def compute_golden(self, args, params): + pass + + +if __name__ == "__main__": + SceneTestCase.run_module(__name__) diff --git a/tests/st/a5/host_build_graph/multi_core_dag/kernels/check_stress.cpp b/tests/st/a5/host_build_graph/multi_core_dag/kernels/check_stress.cpp new file mode 100644 index 0000000000..f5eeb5d478 --- /dev/null +++ b/tests/st/a5/host_build_graph/multi_core_dag/kernels/check_stress.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include + +#include "intrinsic.h" +#include "tensor.h" + +#ifndef __gm__ +#define __gm__ +#endif + +#ifndef __aicore__ +#define __aicore__ [aicore] // NOLINT(whitespace/braces) +#endif + +extern "C" __aicore__ void kernel_entry(__gm__ int64_t *args) { + __gm__ Tensor *tensor = reinterpret_cast<__gm__ Tensor *>(args[0]); + __gm__ int64_t *state = reinterpret_cast<__gm__ int64_t *>(tensor->buffer.addr) + tensor->start_offset; + const int64_t task_id = args[1]; + const int64_t graph_case = args[2]; + const int64_t fanin_count = args[3]; + bool ready = true; + if (graph_case == 1 && task_id >= 32) { + for (int64_t producer = 0; producer < 32; ++producer) + dcci(&state[(producer + 1) * 8], cache_line_t::SINGLE_CACHE_LINE); + dsb((mem_dsb_t)0); + for (int64_t producer = 0; producer < 32; ++producer) { + ready = ready && state[(producer + 1) * 8] == producer + 1; + } + } else { + const int64_t checked = fanin_count < 4 ? fanin_count : 4; + for (int64_t i = 0; i < checked; ++i) { + const int64_t producer = args[4 + i]; + if (producer >= 0) dcci(&state[(producer + 1) * 8], cache_line_t::SINGLE_CACHE_LINE); + } + dsb((mem_dsb_t)0); + for (int64_t i = 0; i < checked; ++i) { + const int64_t producer = args[4 + i]; + ready = ready && producer >= 0 && state[(producer + 1) * 8] == producer + 1; + } + } + state[(task_id + 1) * 8] = ready ? task_id + 1 : -(task_id + 1); + dcci(&state[(task_id + 1) * 8], cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_OUT); + dsb((mem_dsb_t)0); +} diff --git a/tests/st/a5/host_build_graph/multi_core_dag/kernels/orchestration/multi_core_dag_orch.cpp b/tests/st/a5/host_build_graph/multi_core_dag/kernels/orchestration/multi_core_dag_orch.cpp new file mode 100644 index 0000000000..f0834e5fd3 --- /dev/null +++ b/tests/st/a5/host_build_graph/multi_core_dag/kernels/orchestration/multi_core_dag_orch.cpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include + +#include "orchestration_api.h" // NOLINT(build/include_subdir) + +namespace { + +constexpr int32_t kAicFuncId = 0; +constexpr int32_t kAivFuncId = 1; +constexpr int32_t kTaskCapacity = 4096; +constexpr int32_t kKernelCheckedFanin = 4; + +TaskId submit_task( + const simpler::hbg::Tensor &task_state, int64_t graph_case, int64_t logical_id, const TaskId *deps, + int32_t dep_count +) { + CoreTaskArgs task; + task.add_no_dep(task_state); + task.add_scalar(logical_id); + task.add_scalar(graph_case); + task.add_scalar(static_cast(dep_count)); + for (int32_t i = 0; i < kKernelCheckedFanin; ++i) { + task.add_scalar(i < dep_count ? static_cast(deps[i].local_id()) : -1); + } + task.set_dependencies(deps, static_cast(dep_count)); + const bool use_aic = (logical_id & 1) == 0; + return (use_aic ? rt_submit_aic_task(kAicFuncId, task) : rt_submit_aiv_task(kAivFuncId, task)).task_id(); +} + +void build_chain(const simpler::hbg::Tensor &task_state, int64_t task_count) { + TaskId ids[kTaskCapacity]; + ids[0] = submit_task(task_state, 0, 0, nullptr, 0); + for (int64_t task_id = 1; task_id < task_count; ++task_id) { + ids[task_id] = submit_task(task_state, 0, task_id, &ids[task_id - 1], 1); + } +} + +void build_fanin32(const simpler::hbg::Tensor &task_state, int64_t task_count) { + TaskId roots[32]; + const int32_t root_count = static_cast(std::min(task_count, 32)); + for (int32_t task_id = 0; task_id < root_count; ++task_id) { + roots[task_id] = submit_task(task_state, 1, task_id, nullptr, 0); + } + for (int64_t task_id = root_count; task_id < task_count; ++task_id) { + (void)submit_task(task_state, 1, task_id, roots, root_count); + } +} + +void build_multi_root(const simpler::hbg::Tensor &task_state, int64_t task_count) { + for (int64_t task_id = 0; task_id < task_count; ++task_id) { + (void)submit_task(task_state, 2, task_id, nullptr, 0); + } +} + +void build_random(const simpler::hbg::Tensor &task_state, int64_t task_count) { + TaskId ids[kTaskCapacity]; + ids[0] = submit_task(task_state, 3, 0, nullptr, 0); + uint64_t random = UINT64_C(0x9e3779b97f4a7c15); + for (int64_t task_id = 1; task_id < task_count; ++task_id) { + random = random * UINT64_C(6364136223846793005) + 1; + const int32_t requested = static_cast(random % 5); + TaskId deps[kKernelCheckedFanin]; + int32_t count = 0; + for (int32_t edge = 0; edge < requested; ++edge) { + random = random * UINT64_C(6364136223846793005) + 1; + const int64_t producer = static_cast(random % static_cast(task_id)); + bool duplicate = false; + for (int32_t i = 0; i < count; ++i) + duplicate = duplicate || deps[i] == ids[producer]; + if (!duplicate) deps[count++] = ids[producer]; + } + ids[task_id] = submit_task(task_state, 3, task_id, deps, count); + } +} + +} // namespace + +extern "C" { + +__attribute__((visibility("default"))) OrchestrationConfig aicpu_orchestration_config(const ChipTaskArgs &args) { + (void)args; + return OrchestrationConfig{.expected_arg_count = 3}; +} + +__attribute__((visibility("default"))) void aicpu_orchestration_entry(const ChipTaskArgs &args) { + const simpler::hbg::Tensor &task_state = args.tensor(0).ref(); + const int64_t graph_case = static_cast(args.scalar(0)); + const int64_t task_count = static_cast(args.scalar(1)); + if (task_count < 1 || task_count > kTaskCapacity) { + rt_report_fatal( + SIMPLER_ERROR_INVALID_ARGS, "task_count must be in [1, %d], got %ld", kTaskCapacity, task_count + ); + return; + } + switch (graph_case) { + case 0: + build_chain(task_state, task_count); + break; + case 1: + build_fanin32(task_state, task_count); + break; + case 2: + build_multi_root(task_state, task_count); + break; + case 3: + build_random(task_state, task_count); + break; + default: + rt_report_fatal(SIMPLER_ERROR_INVALID_ARGS, "unsupported graph case %ld", graph_case); + break; + } +} + +} // extern "C" diff --git a/tests/st/a5/host_build_graph/multi_core_dag/test_multi_core_dag.py b/tests/st/a5/host_build_graph/multi_core_dag/test_multi_core_dag.py new file mode 100644 index 0000000000..003af83618 --- /dev/null +++ b/tests/st/a5/host_build_graph/multi_core_dag/test_multi_core_dag.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Mixed-core DAG correctness, queue-wrap, and topology pressure matrix.""" + +import ctypes + +import torch +from simpler.task_interface import ArgDirection as D + +from simpler_setup import Scalar, SceneTestCase, TaskArgsBuilder, TensorArg, scene_test + + +def _case(name, graph_case, task_count): + case = { + "name": name, + "platforms": ["a5sim", "a5"], + "params": {"graph_case": graph_case, "task_count": task_count}, + } + if name != "mixed_chain_65": + case["manual"] = True + return case + + +@scene_test(level=2, runtime="host_build_graph") +class TestHbgMultiCoreDag(SceneTestCase): + RTOL = 0 + ATOL = 0 + + CALLABLE = { + "orchestration": { + "source": "kernels/orchestration/multi_core_dag_orch.cpp", + "function_name": "aicpu_orchestration_entry", + "signature": [D.INOUT], + }, + "incores": [ + { + "func_id": 0, + "source": "kernels/check_stress.cpp", + "core_type": "aic", + "signature": [D.INOUT], + }, + { + "func_id": 1, + "source": "kernels/check_stress.cpp", + "core_type": "aiv", + "signature": [D.INOUT], + }, + ], + } + + CASES = [ + *[_case(f"mixed_chain_{count}", 0, count) for count in (1, 2, 63, 65, 1024)], + _case("mixed_fanin32_1024", 1, 1024), + _case("mixed_random_1024", 3, 1024), + _case("mixed_multi_root_4096", 2, 4096), + ] + + def generate_args(self, params): + task_count = params["task_count"] + return TaskArgsBuilder( + TensorArg("task_state", torch.zeros((task_count + 1) * 8, dtype=torch.int64)), + Scalar("graph_case", ctypes.c_int64(params["graph_case"])), + Scalar("task_count", ctypes.c_int64(task_count)), + ) + + def compute_golden(self, args, params): + task_count = params["task_count"] + args.task_state[8 : (task_count + 1) * 8 : 8] = torch.arange(1, task_count + 1, dtype=torch.int64) + + +if __name__ == "__main__": + SceneTestCase.run_module(__name__) diff --git a/tests/st/a5/host_build_graph/paged_attention/test_paged_attention.py b/tests/st/a5/host_build_graph/paged_attention/test_paged_attention.py index 6bb83b3efc..3f4d2fe266 100644 --- a/tests/st/a5/host_build_graph/paged_attention/test_paged_attention.py +++ b/tests/st/a5/host_build_graph/paged_attention/test_paged_attention.py @@ -68,16 +68,14 @@ class TestPagedAttentionHostBuildGraphA5(SceneTestCase): CASES = [ { - # Marked manual for host_build_graph: this batch=256 case submits - # ~64K tasks, and host-orchestration populates the whole task graph - # before the device schedules — nothing is reclaimed mid-orchestration, - # so the table must hold the entire graph at once. That exceeds the - # default (16384 slots). Run it explicitly with a larger - # runtime_env.ring_task_window if needed; the GM heap needs no sizing, - # since it is committed to the size orchestration measured. "name": "Case1", "platforms": ["a5"], "manual": True, + "config": { + "runtime_env": { + "ring_task_window": 131072, + } + }, "params": { "batch": 256, "num_heads": 16, diff --git a/tests/st/a5/host_build_graph/single_core_dag/kernels/check_dag.cpp b/tests/st/a5/host_build_graph/single_core_dag/kernels/check_dag.cpp new file mode 100644 index 0000000000..95fafee88c --- /dev/null +++ b/tests/st/a5/host_build_graph/single_core_dag/kernels/check_dag.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include + +#include "intrinsic.h" +#include "tensor.h" + +#ifndef __gm__ +#define __gm__ +#endif + +#ifndef __aicore__ +#define __aicore__ [aicore] // NOLINT(whitespace/braces) +#endif + +extern "C" __aicore__ void kernel_entry(__gm__ int64_t *args) { + __gm__ Tensor *tensor = reinterpret_cast<__gm__ Tensor *>(args[0]); + __gm__ int64_t *state = reinterpret_cast<__gm__ int64_t *>(tensor->buffer.addr) + tensor->start_offset; + int64_t task_id = args[1]; + uint64_t producer_mask = static_cast(args[2]); + for (int64_t producer = 0; producer < 64; ++producer) { + if ((producer_mask & (UINT64_C(1) << producer)) != 0) + dcci(&state[producer * 8], cache_line_t::SINGLE_CACHE_LINE); + } + dsb((mem_dsb_t)0); + for (int64_t producer = 0; producer < 64; ++producer) { + if ((producer_mask & (UINT64_C(1) << producer)) != 0 && state[producer * 8] != producer + 1) { + state[task_id * 8] = -(producer + 1); + dcci(&state[task_id * 8], cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_OUT); + dsb((mem_dsb_t)0); + return; + } + } + state[task_id * 8] = task_id + 1; + dcci(&state[task_id * 8], cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_OUT); + dsb((mem_dsb_t)0); +} diff --git a/tests/st/a5/host_build_graph/single_core_dag/kernels/orchestration/single_core_dag_orch.cpp b/tests/st/a5/host_build_graph/single_core_dag/kernels/orchestration/single_core_dag_orch.cpp new file mode 100644 index 0000000000..3433e3b48b --- /dev/null +++ b/tests/st/a5/host_build_graph/single_core_dag/kernels/orchestration/single_core_dag_orch.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "orchestration_api.h" // NOLINT(build/include_subdir) + +namespace { + +constexpr int32_t kAicFuncId = 0; +constexpr int32_t kAivFuncId = 1; +constexpr int32_t kTaskCapacity = 64; + +TaskId submit_task( + const simpler::hbg::Tensor &task_state, int64_t core_mode, int64_t logical_id, const TaskId *deps, int32_t dep_count +) { + CoreTaskArgs task; + task.add_no_dep(task_state); + task.add_scalar(logical_id); + uint64_t producer_mask = 0; + for (int32_t i = 0; i < dep_count; ++i) + producer_mask |= UINT64_C(1) << deps[i].local_id(); + task.add_scalar(producer_mask); + task.set_dependencies(deps, static_cast(dep_count)); + const bool use_aic = core_mode == 0 || (core_mode == 2 && (logical_id & 1) == 0); + return (use_aic ? rt_submit_aic_task(kAicFuncId, task) : rt_submit_aiv_task(kAivFuncId, task)).task_id(); +} + +void build_chain(const simpler::hbg::Tensor &task_state, int64_t core_type) { + TaskId ids[kTaskCapacity]; + ids[0] = submit_task(task_state, core_type, 0, nullptr, 0); + for (int64_t task_id = 1; task_id < kTaskCapacity; ++task_id) + ids[task_id] = submit_task(task_state, core_type, task_id, &ids[task_id - 1], 1); +} + +void build_diamonds(const simpler::hbg::Tensor &task_state, int64_t core_type) { + TaskId ids[25]; + ids[0] = submit_task(task_state, core_type, 0, nullptr, 0); + int32_t next = 1; + int32_t prior = 0; + for (int32_t layer = 0; layer < 8; ++layer) { + int32_t left = next++; + int32_t right = next++; + ids[left] = submit_task(task_state, core_type, left, &ids[prior], 1); + ids[right] = submit_task(task_state, core_type, right, &ids[prior], 1); + TaskId deps[2] = {ids[left], ids[right]}; + int32_t join = next++; + ids[join] = submit_task(task_state, core_type, join, deps, 2); + prior = join; + } +} + +void build_fanout(const simpler::hbg::Tensor &task_state, int64_t core_type) { + TaskId root = submit_task(task_state, core_type, 0, nullptr, 0); + for (int64_t task_id = 1; task_id < kTaskCapacity; ++task_id) + (void)submit_task(task_state, core_type, task_id, &root, 1); +} + +void build_dense_fanin(const simpler::hbg::Tensor &task_state, int64_t core_type) { + TaskId roots[32]; + for (int64_t task_id = 0; task_id < 32; ++task_id) + roots[task_id] = submit_task(task_state, core_type, task_id, nullptr, 0); + for (int64_t task_id = 32; task_id < kTaskCapacity; ++task_id) + (void)submit_task(task_state, core_type, task_id, roots, 32); +} + +void build_multi_root(const simpler::hbg::Tensor &task_state, int64_t core_type) { + for (int64_t task_id = 0; task_id < kTaskCapacity; ++task_id) + (void)submit_task(task_state, core_type, task_id, nullptr, 0); +} + +void build_random(const simpler::hbg::Tensor &task_state, int64_t core_type) { + TaskId ids[kTaskCapacity]; + ids[0] = submit_task(task_state, core_type, 0, nullptr, 0); + uint64_t random = UINT64_C(0x9e3779b97f4a7c15); + for (int64_t task_id = 1; task_id < kTaskCapacity; ++task_id) { + random = random * UINT64_C(6364136223846793005) + 1; + int32_t requested = static_cast(random % 5); + TaskId deps[4]; + int32_t count = 0; + for (int32_t edge = 0; edge < requested; ++edge) { + random = random * UINT64_C(6364136223846793005) + 1; + int64_t producer = static_cast(random % static_cast(task_id)); + bool duplicate = false; + for (int32_t i = 0; i < count; ++i) + duplicate = duplicate || deps[i] == ids[producer]; + if (!duplicate) deps[count++] = ids[producer]; + } + ids[task_id] = submit_task(task_state, core_type, task_id, deps, count); + } +} + +} // namespace + +extern "C" { + +__attribute__((visibility("default"))) OrchestrationConfig aicpu_orchestration_config(const ChipTaskArgs &args) { + (void)args; + return OrchestrationConfig{.expected_arg_count = 3}; +} + +__attribute__((visibility("default"))) void aicpu_orchestration_entry(const ChipTaskArgs &args) { + const simpler::hbg::Tensor &task_state = args.tensor(0).ref(); + int64_t graph_case = static_cast(args.scalar(0)); + int64_t core_type = static_cast(args.scalar(1)); + switch (graph_case) { + case 0: + build_chain(task_state, core_type); + break; + case 1: + build_diamonds(task_state, core_type); + break; + case 2: + build_fanout(task_state, core_type); + break; + case 3: + build_dense_fanin(task_state, core_type); + break; + case 4: + build_multi_root(task_state, core_type); + break; + case 5: + build_random(task_state, core_type); + break; + default: + rt_report_fatal(SIMPLER_ERROR_INVALID_ARGS, "unsupported graph case %ld", graph_case); + break; + } +} + +} // extern "C" diff --git a/tests/st/a5/host_build_graph/single_core_dag/test_single_core_dag.py b/tests/st/a5/host_build_graph/single_core_dag/test_single_core_dag.py new file mode 100644 index 0000000000..4c96921d3b --- /dev/null +++ b/tests/st/a5/host_build_graph/single_core_dag/test_single_core_dag.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Correctness gates for homogeneous and mixed AIC/AIV dependency graphs.""" + +import ctypes + +import torch +from simpler.task_interface import ArgDirection as D + +from simpler_setup import Scalar, SceneTestCase, TaskArgsBuilder, TensorArg, scene_test + +GRAPH_CASES = { + "chain_64": (0, 64), + "diamond_8": (1, 25), + "fanout_63": (2, 64), + "fanin_32x32": (3, 64), + "multi_root_64": (4, 64), + "random_64": (5, 64), +} + +AUTOMATIC_CASES = {"aic_chain_64", "aiv_chain_64", "mixed_diamond_8"} + + +@scene_test(level=2, runtime="host_build_graph") +class TestHbgSingleCoreDag(SceneTestCase): + RTOL = 0 + ATOL = 0 + + CALLABLE = { + "orchestration": { + "source": "kernels/orchestration/single_core_dag_orch.cpp", + "function_name": "aicpu_orchestration_entry", + "signature": [D.INOUT], + }, + "incores": [ + { + "func_id": 0, + "source": "kernels/check_dag.cpp", + "core_type": "aic", + "signature": [D.INOUT], + }, + { + "func_id": 1, + "source": "kernels/check_dag.cpp", + "core_type": "aiv", + "signature": [D.INOUT], + }, + ], + } + + CASES = [ + { + "name": f"{core_type}_{name}", + "platforms": ["a5sim", "a5"], + "params": { + "graph_case": graph_case, + "task_count": task_count, + "core_type": core_type, + }, + **({} if f"{core_type}_{name}" in AUTOMATIC_CASES else {"manual": True}), + } + for core_type, names in { + "aic": ("chain_64",), + "aiv": ("chain_64",), + "mixed": tuple(GRAPH_CASES), + }.items() + for name in names + for graph_case, task_count in (GRAPH_CASES[name],) + ] + + def generate_args(self, params): + return TaskArgsBuilder( + TensorArg("task_state", torch.zeros(64 * 8, dtype=torch.int64)), + Scalar("graph_case", ctypes.c_int64(params["graph_case"])), + Scalar("core_type", ctypes.c_int64({"aic": 0, "aiv": 1, "mixed": 2}[params["core_type"]])), + ) + + def compute_golden(self, args, params): + count = params["task_count"] + args.task_state[: count * 8 : 8] = torch.arange(1, count + 1, dtype=torch.int64) + + +if __name__ == "__main__": + SceneTestCase.run_module(__name__) diff --git a/tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aic_root_orch.cpp b/tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aic_root_orch.cpp new file mode 100644 index 0000000000..7a3c40ff51 --- /dev/null +++ b/tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aic_root_orch.cpp @@ -0,0 +1,27 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "orchestration_api.h" // NOLINT(build/include_subdir) + +extern "C" { + +__attribute__((visibility("default"))) OrchestrationConfig aicpu_orchestration_config(const ChipTaskArgs &args) { + (void)args; + return OrchestrationConfig{.expected_arg_count = 1}; +} + +__attribute__((visibility("default"))) void aicpu_orchestration_entry(const ChipTaskArgs &args) { + CoreTaskArgs root; + root.add_inout(args.tensor(0).ref()); + rt_submit_aic_task(0, root); +} + +} // extern "C" diff --git a/tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aic_spmd_fallback_orch.cpp b/tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aic_spmd_fallback_orch.cpp new file mode 100644 index 0000000000..7268205444 --- /dev/null +++ b/tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aic_spmd_fallback_orch.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "orchestration_api.h" // NOLINT(build/include_subdir) + +extern "C" { + +__attribute__((visibility("default"))) OrchestrationConfig aicpu_orchestration_config(const ChipTaskArgs &args) { + (void)args; + return OrchestrationConfig{.expected_arg_count = 1}; +} + +__attribute__((visibility("default"))) void aicpu_orchestration_entry(const ChipTaskArgs &args) { + CoreTaskArgs root; + root.add_inout(args.tensor(0).ref()); + root.launch_spec.set_block_num(2); + rt_submit_aic_task(0, root); +} + +} // extern "C" diff --git a/tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aiv_root_orch.cpp b/tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aiv_root_orch.cpp new file mode 100644 index 0000000000..326d604d35 --- /dev/null +++ b/tests/st/a5/host_build_graph/single_root/kernels/orchestration/single_aiv_root_orch.cpp @@ -0,0 +1,29 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "orchestration_api.h" // NOLINT(build/include_subdir) + +extern "C" { + +__attribute__((visibility("default"))) OrchestrationConfig aicpu_orchestration_config(const ChipTaskArgs &args) { + (void)args; + return OrchestrationConfig{.expected_arg_count = 3}; +} + +__attribute__((visibility("default"))) void aicpu_orchestration_entry(const ChipTaskArgs &args) { + CoreTaskArgs root; + root.add_input(args.tensor(0).ref()); + root.add_input(args.tensor(1).ref()); + root.add_inout(args.tensor(2).ref()); + rt_submit_aiv_task(0, root); +} + +} // extern "C" diff --git a/tests/st/a5/host_build_graph/single_root/test_single_root.py b/tests/st/a5/host_build_graph/single_root/test_single_root.py new file mode 100644 index 0000000000..ac82a397e9 --- /dev/null +++ b/tests/st/a5/host_build_graph/single_root/test_single_root.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Gates for one dependency-free AIC or AIV root.""" + +import torch +from simpler.task_interface import ArgDirection as D + +from simpler_setup import SceneTestCase, TaskArgsBuilder, TensorArg, scene_test + + +@scene_test(level=2, runtime="host_build_graph") +class TestHbgSingleAicRoot(SceneTestCase): + RTOL = 0 + ATOL = 0 + + CALLABLE = { + "orchestration": { + "source": "kernels/orchestration/single_aic_root_orch.cpp", + "function_name": "aicpu_orchestration_entry", + "signature": [D.INOUT], + }, + "incores": [ + { + "func_id": 0, + "source": "../../tensormap_and_ringbuffer/dummy_task/kernels/aic/kernel_write_const.cpp", + "core_type": "aic", + "signature": [D.INOUT], + } + ], + } + + CASES = [ + { + "name": "single_aic_root", + "platforms": ["a5sim", "a5"], + "params": {}, + }, + ] + + def generate_args(self, params): + return TaskArgsBuilder(TensorArg("out", torch.zeros(16, dtype=torch.float32))) + + def compute_golden(self, args, params): + args.out[0] = 42.0 + + +@scene_test(level=2, runtime="host_build_graph") +class TestHbgSingleAivRoot(SceneTestCase): + RTOL = 0 + ATOL = 0 + + CALLABLE = { + "orchestration": { + "source": "kernels/orchestration/single_aiv_root_orch.cpp", + "function_name": "aicpu_orchestration_entry", + "signature": [D.IN, D.IN, D.INOUT], + }, + "incores": [ + { + "func_id": 0, + "source": "../vector_example/kernels/aiv/kernel_add.cpp", + "core_type": "aiv", + "signature": [D.IN, D.IN, D.INOUT], + } + ], + } + + CASES = [ + { + "name": "single_aiv_root", + "platforms": ["a5sim", "a5"], + "params": {}, + }, + ] + + def generate_args(self, params): + size = 128 * 128 + return TaskArgsBuilder( + TensorArg("a", torch.full((size,), 2.0, dtype=torch.float32)), + TensorArg("b", torch.full((size,), 3.0, dtype=torch.float32)), + TensorArg("out", torch.zeros(size, dtype=torch.float32)), + ) + + def compute_golden(self, args, params): + args.out[:] = args.a + args.b + + +@scene_test(level=2, runtime="host_build_graph") +class TestHbgSingleAicSpmdLegacyFallback(TestHbgSingleAicRoot): + CALLABLE = { + **TestHbgSingleAicRoot.CALLABLE, + "orchestration": { + "source": "kernels/orchestration/single_aic_spmd_fallback_orch.cpp", + "function_name": "aicpu_orchestration_entry", + "signature": [D.INOUT], + }, + } + CASES = [ + { + "name": "single_aic_spmd_legacy_fallback", + "platforms": ["a5sim", "a5"], + "params": {}, + }, + ] + + +if __name__ == "__main__": + SceneTestCase.run_module(__name__) diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index fade2092d9..a8aeb0ae5e 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -943,6 +943,9 @@ target_sources(test_hbg_submit_poison PRIVATE ${CMAKE_SOURCE_DIR}/../../../src/common/platform/shared/aicpu/args_dump_aicpu.cpp ) add_a5_hbg_runtime_test(test_a5_hbg_scheduler_contracts a5/test_hbg_scheduler_contracts.cpp) +target_include_directories(test_a5_hbg_scheduler_contracts PRIVATE + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/host_build_graph/aicpu +) add_a5_hbg_runtime_test(test_a5_hbg_scheduler_ready a5/test_hbg_scheduler_ready.cpp) add_a5_hbg_runtime_test(test_a5_hbg_scheduler_dispatch a5/test_hbg_scheduler_dispatch.cpp) add_a2a3_hbg_runtime_test(test_hbg_graph_submit_failure common/test_hbg_graph_submit_failure.cpp) diff --git a/tests/ut/cpp/a5/test_hbg_scheduler_contracts.cpp b/tests/ut/cpp/a5/test_hbg_scheduler_contracts.cpp index 13bf70973d..49e4b2d96d 100644 --- a/tests/ut/cpp/a5/test_hbg_scheduler_contracts.cpp +++ b/tests/ut/cpp/a5/test_hbg_scheduler_contracts.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -20,12 +21,66 @@ #include #include "host_build_graph/runtime_types.h" +#include "aicore_scheduler_error.h" +#include "aicore_scheduler_state.h" #include "scheduler/scheduler_graph.h" #include "scheduler/scheduler_topology.h" #include "scheduler/scheduler_types.h" +#include "scheduler/scheduler_watchdog.h" namespace { +TEST(AicoreSchedulerError, MapsInternalFailureToExistingHostStatus) { + EXPECT_EQ(aicore_scheduler_runtime_error_code(0), SIMPLER_ERROR_NONE); + EXPECT_EQ(aicore_scheduler_runtime_error_code(1), SIMPLER_ERROR_INVALID_ARGS); + EXPECT_EQ( + aicore_scheduler_runtime_error_code(static_cast(SchedulerGraphResult::TIMEOUT)), + SIMPLER_ERROR_SCHEDULER_TIMEOUT + ); + EXPECT_EQ(aicore_scheduler_runtime_error_code(UINT64_MAX), SIMPLER_ERROR_INVALID_ARGS); +} + +TEST(AicoreSchedulerError, LatchesFirstHostVisibleFailure) { + std::atomic status{SIMPLER_ERROR_NONE}; + EXPECT_FALSE(latch_aicore_scheduler_runtime_error(&status, 0)); + EXPECT_EQ(status.load(std::memory_order_relaxed), SIMPLER_ERROR_NONE); + + EXPECT_TRUE(latch_aicore_scheduler_runtime_error(&status, static_cast(SchedulerGraphResult::TIMEOUT))); + EXPECT_EQ(status.load(std::memory_order_relaxed), SIMPLER_ERROR_SCHEDULER_TIMEOUT); + + status.store(SIMPLER_ERROR_READY_QUEUE_OVERFLOW, std::memory_order_relaxed); + EXPECT_FALSE(latch_aicore_scheduler_runtime_error(&status, 2)); + EXPECT_EQ(status.load(std::memory_order_relaxed), SIMPLER_ERROR_READY_QUEUE_OVERFLOW); +} + +TEST(AicoreSchedulerState, DistinguishesResidentAndExplicitLegacyModes) { + EXPECT_FALSE(aicore_scheduler_runtime_mode_is_resident(0)); + EXPECT_TRUE(aicore_scheduler_runtime_mode_is_resident(SCHEDULER_RUNTIME_MODE_RESIDENT_PENDING)); + EXPECT_TRUE(aicore_scheduler_runtime_mode_is_resident(SCHEDULER_RUNTIME_MODE_RESIDENT_READY)); + EXPECT_FALSE(aicore_scheduler_runtime_mode_is_resident(SCHEDULER_RUNTIME_MODE_LEGACY_GRAPH)); + EXPECT_FALSE(aicore_scheduler_runtime_mode_is_resident(SCHEDULER_RUNTIME_MODE_LEGACY_UNSUPPORTED_SHAPE)); + + EXPECT_FALSE(aicore_scheduler_runtime_mode_is_explicit_legacy(0)); + EXPECT_FALSE(aicore_scheduler_runtime_mode_is_explicit_legacy(SCHEDULER_RUNTIME_MODE_RESIDENT_PENDING)); + EXPECT_FALSE(aicore_scheduler_runtime_mode_is_explicit_legacy(SCHEDULER_RUNTIME_MODE_RESIDENT_READY)); + EXPECT_TRUE(aicore_scheduler_runtime_mode_is_explicit_legacy(SCHEDULER_RUNTIME_MODE_LEGACY_GRAPH)); + EXPECT_TRUE(aicore_scheduler_runtime_mode_is_explicit_legacy(SCHEDULER_RUNTIME_MODE_LEGACY_UNSUPPORTED_SHAPE)); +} + +TEST(AicoreSchedulerState, ResidentV0AcceptsOnlySingleLaneSingleBlockTasks) { + EXPECT_TRUE(scheduler_resident_v0_task_shape_supported(1, 1, false)); + EXPECT_FALSE(scheduler_resident_v0_task_shape_supported(2, 1, false)); + EXPECT_FALSE(scheduler_resident_v0_task_shape_supported(1, 2, false)); + EXPECT_FALSE(scheduler_resident_v0_task_shape_supported(1, 1, true)); +} + +TEST(AicoreSchedulerWatchdog, UsesElapsedWallClockBudget) { + EXPECT_FALSE(scheduler_watchdog_expired(100, UINT64_MAX, 0)); + EXPECT_FALSE(scheduler_watchdog_expired(100, 199, 100)); + EXPECT_TRUE(scheduler_watchdog_expired(100, 200, 100)); + EXPECT_TRUE(scheduler_watchdog_expired(UINT64_MAX - 10, 4, 15)); +} + class SchedulerStateBuffer { public: explicit SchedulerStateBuffer(const AicoreSchedulerLayout &layout) :