Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pto_isa.pin
Original file line number Diff line number Diff line change
@@ -1 +1 @@
be5ccb765a4ce5d14ca5da8b0e2f182d7f003369
fe2f68dac50a608c3bf11ac8d3438cd78fef0d98
133 changes: 107 additions & 26 deletions simpler_setup/pto_isa.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,19 @@
"""PTO-ISA dependency management: resolve the pinned managed checkout.

``pto_isa.pin`` is the single source of truth for the PTO-ISA revision.
``ensure_pto_isa_root()`` always manages ``PROJECT_ROOT/build/pto-isa``:
``ensure_pto_isa_root()`` resolves ``PROJECT_ROOT/build/pto-isa``:

1. Read the required commit from ``pto_isa.pin``.
2. If a managed checkout already exists **clean and at exactly the pin**, use it
2. If a sibling ``../pto-isa`` checkout exists at exactly the pin, link
``build/pto-isa`` to it and use that tree. This keeps paired simpler +
pto-isa workspaces on the same local revision during cross-repo migrations.
3. If a managed checkout already exists **clean and at exactly the pin**, use it
as-is — it already *is* the pinned ISA, so no checkout and no network.
3. Otherwise (missing, wrong revision, or dirty) obtain the pin fresh: clone
4. Otherwise (missing, wrong revision, or dirty) obtain the pin fresh: clone
over HTTPS (``--no-checkout`` so the default branch is never materialized)
and force-check-out the pin. GitHub is attempted three times; if it cannot
provide the complete pinned checkout, fall back to the GitCode mirror.
and force-check-out the pin. The configured source is attempted three times;
if it cannot provide the complete pinned checkout, fall back to the other
known PTO-ISA source.
4. Verify HEAD exactly matches the pin before returning.

Two deliberate choices:
Expand All @@ -41,6 +45,7 @@
import fcntl
import json
import logging
import os
import re
import shutil
import subprocess
Expand All @@ -59,9 +64,10 @@

logger = logging.getLogger(__name__)

_PTO_ISA_GITCODE_HTTPS = "https://gitcode.com/wxwnnzdyd/pto-isa.git"
_PTO_ISA_GITHUB_HTTPS = "https://github.com/hw-native-sys/pto-isa.git"
_PTO_ISA_GITCODE_HTTPS = "https://gitcode.com/luohuan40/pto-isa.git"
_PTO_ISA_PIN_RE = re.compile(r"^[0-9a-fA-F]{40}$")
PTO_ISA_CLONE_URL_ENV = "PTO_ISA_CLONE_URL"
PTO_ISA_PIN_FILE = "pto_isa.pin"
PTO_ISA_BUILD_METADATA = "pto_isa_build.json"

Expand Down Expand Up @@ -234,6 +240,23 @@ def get_pto_isa_clone_path() -> Path:
return PROJECT_ROOT / "build" / "pto-isa"


def get_pto_isa_clone_url() -> str:
"""Clone URL for managed PTO-ISA checkouts."""
return os.environ.get(PTO_ISA_CLONE_URL_ENV, _PTO_ISA_GITCODE_HTTPS).strip() or _PTO_ISA_GITCODE_HTTPS


def _clone_sources() -> list[str]:
"""Return distinct PTO-ISA clone sources in preference order."""
sources = [get_pto_isa_clone_url(), _PTO_ISA_GITCODE_HTTPS, _PTO_ISA_GITHUB_HTTPS]
return list(dict.fromkeys(sources))


def get_sibling_pto_isa_checkout_path(clone_path: Optional[Path] = None) -> Path:
"""Sibling PTO-ISA checkout used by paired local simpler/pto-isa worktrees."""
managed = clone_path or get_pto_isa_clone_path()
return managed.parent.parent.parent / "pto-isa"


def _is_cloned(path: Path) -> bool:
"""Return True if `path` looks like a valid PTO-ISA clone (has include/)."""
return (path / "include").is_dir()
Expand Down Expand Up @@ -312,6 +335,64 @@ def _remove_clone(target: Path, verbose: bool) -> None:
target.unlink(missing_ok=True)


def _can_replace_with_sibling_link(target: Path, verbose: bool) -> bool:
"""Return true when replacing target cannot discard local changes."""
if not (target.exists() or target.is_symlink()):
return True
if target.is_symlink():
return True
if not _is_cloned(target):
return True

current_head = get_pto_isa_head(str(target))
if current_head and _is_pristine_at_commit(target, current_head, verbose=verbose):
return True

if verbose:
logger.warning(
f"Refusing to replace non-pristine pto-isa checkout at {target}; "
"move it aside before linking the sibling checkout."
)
return False


def _link_sibling_checkout_if_pinned(clone_path: Path, required_commit: str, verbose: bool) -> Optional[str]:
"""Point build/pto-isa at sibling ../pto-isa when it exactly matches the pin."""
sibling = get_sibling_pto_isa_checkout_path(clone_path)
if sibling.resolve() == clone_path.resolve():
return None
if not _is_cloned(sibling):
return None

actual_commit = get_pto_isa_head(str(sibling))
if actual_commit != required_commit:
if verbose:
logger.warning(
f"Ignoring sibling pto-isa checkout at {sibling}: "
f"expected {required_commit}, got {actual_commit or '<unknown>'}"
)
return None

if clone_path.exists() or clone_path.is_symlink():
if clone_path.resolve() == sibling.resolve():
return str(clone_path.resolve())
if not _can_replace_with_sibling_link(clone_path, verbose):
return None
_remove_clone(clone_path, verbose)

try:
clone_path.parent.mkdir(parents=True, exist_ok=True)
clone_path.symlink_to(sibling.resolve(), target_is_directory=True)
except OSError as e:
if verbose:
logger.warning(f"Failed to link sibling pto-isa checkout {sibling} -> {clone_path}: {e}")
return None

if verbose:
logger.info(f"Using sibling pto-isa checkout at {sibling} via {clone_path}")
return str(clone_path.resolve())


def _land_on_commit(clone_path: Path, commit: str, verbose: bool) -> bool:
"""Force-detach-checkout a freshly cloned tree onto `commit`. False on failure.

Expand Down Expand Up @@ -427,9 +508,9 @@ def _clone(target: Path, commit: str, verbose: bool) -> bool:

Any existing checkout at `target` is removed first, so this always yields a
clean tree at exactly `commit` — the sync can never be blocked by local
modifications in a preexisting (cached/preset) managed checkout. GitHub is
tried three times for the complete clone-and-pin operation before GitCode is
used as a fallback source.
modifications in a preexisting (cached/preset) managed checkout. The
configured source is tried three times for the complete clone-and-pin
operation before the other known PTO-ISA source is used as a fallback.
"""
if not _is_git_available():
if verbose:
Expand All @@ -444,20 +525,16 @@ def _clone(target: Path, commit: str, verbose: bool) -> bool:
return False

try:
if _clone_from_remote(
target,
commit,
_PTO_ISA_GITHUB_HTTPS,
attempts=_CLONE_ATTEMPTS,
verbose=verbose,
):
return True

logger.warning(
f"GitHub could not provide PTO-ISA commit {commit} after {_CLONE_ATTEMPTS} attempts; "
f"falling back to {_PTO_ISA_GITCODE_HTTPS}"
)
return _clone_from_remote(target, commit, _PTO_ISA_GITCODE_HTTPS, attempts=1, verbose=verbose)
sources = _clone_sources()
for index, remote in enumerate(sources):
if _clone_from_remote(target, commit, remote, attempts=_CLONE_ATTEMPTS, verbose=verbose):
return True
if index + 1 < len(sources):
logger.warning(
f"PTO-ISA source {remote} could not provide commit {commit} "
f"after {_CLONE_ATTEMPTS} attempts; trying {sources[index + 1]}"
)
return False
except Exception as e: # noqa: BLE001
if verbose:
logger.warning(f"Failed to clone pto-isa: {e}")
Expand All @@ -481,15 +558,19 @@ def ensure_pto_isa_root(verbose: bool = False) -> str:
f"PTO-ISA not available.\n"
f" The managed checkout must live at {clone_path} and match {PROJECT_ROOT / PTO_ISA_PIN_FILE}.\n"
f" If auto-clone failed, manually run:\n"
f" git clone {_PTO_ISA_GITHUB_HTTPS} {clone_path}\n"
f" Or use the fallback mirror:\n"
f" git clone {_PTO_ISA_GITCODE_HTTPS} {clone_path}"
f" git clone {get_pto_isa_clone_url()} {clone_path}\n"
f" Or use another known source:\n"
f" git clone {_PTO_ISA_GITHUB_HTTPS} {clone_path}"
)
return resolved


def _ensure_locked(clone_path: Path, required_commit: str, verbose: bool) -> Optional[str]:
"""Inner logic executed while holding the file lock."""
sibling_checkout = _link_sibling_checkout_if_pinned(clone_path, required_commit, verbose=verbose)
if sibling_checkout is not None:
return sibling_checkout

# Reuse an existing checkout ONLY when it is already exactly the pin: a clean
# working tree at HEAD == pin already *is* the pinned ISA (git objects are
# content-addressed), so use it as-is — no checkout, no network. This is the
Expand Down
7 changes: 5 additions & 2 deletions simpler_setup/runtime_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,9 +245,12 @@ def _build_cache_stamp(self, pto_isa_commit: Optional[str] = None) -> str:
return ""
if pto_isa_commit is None:
pto_isa_commit = self._resolve_build_pto_isa_commit()
parts = [runtime_commit]
if pto_isa_commit:
return f"{runtime_commit}:pto-isa={pto_isa_commit}"
return runtime_commit
parts.append(f"pto-isa={pto_isa_commit}")
if self._variant == "onboard":
parts.append(f"ascend-home={os.environ.get('ASCEND_HOME_PATH', '').strip()}")
return ":".join(parts)

def _lookup_binaries(self, name: str, output_dir: Path) -> RuntimeBinaries:
"""Look up pre-built binaries from output_dir.
Expand Down
16 changes: 13 additions & 3 deletions src/a2a3/platform/onboard/host/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -121,14 +121,19 @@ target_compile_options(host_runtime
target_compile_definitions(host_runtime PRIVATE SIMPLER_PLATFORM_NAME="a2a3")

if(SIMPLER_ENABLE_PTO_SDMA_WORKSPACE)
target_compile_definitions(host_runtime PRIVATE SIMPLER_ENABLE_PTO_SDMA_WORKSPACE=1)
target_compile_definitions(host_runtime PRIVATE
SIMPLER_ENABLE_PTO_SDMA_WORKSPACE=1
PTO_COMM_WORKSPACE_SDMA_SUPPORTED=1
PTO_COMM_WORKSPACE_URMA_SUPPORTED=0
PTO_COMM_WORKSPACE_RDMA_SUPPORTED=0
)
endif()

set(SIMPLER_PTO_ISA_BUILD_COMMIT "" CACHE STRING
"PTO-ISA commit used for host_runtime cache key")

# Bake the resolved pto-isa commit into the compile command. pto-isa headers
# (e.g. kSdmaMaxChan in sdma_workspace_manager.hpp) are compiled into this .so,
# Bake the resolved pto-isa commit into the compile command. pto-isa workspace
# headers (e.g. kSdmaWorkspaceBytes) are compiled into this .so,
# but a pto-isa update leaves the runtime repo HEAD — and the pto-isa header
# mtimes — untouched, so a plain reinstall serves a stale object from ccache.
# This define perturbs the ccache key so a pto-isa bump forces a real recompile.
Expand Down Expand Up @@ -172,6 +177,7 @@ endif()
# CANN 9.x exposes the working non-V2 HCCL entry points through libhcomm.
# Link it explicitly so comm_hccl.cpp can follow the same initialization path
# as the pto-isa communication tests.
unset(HCOMM_LIB CACHE)
find_library(
HCOMM_LIB
NAMES hcomm
Expand Down Expand Up @@ -214,6 +220,10 @@ target_link_directories(host_runtime
${ASCEND_HOME_PATH}/runtime/lib64
)

set_target_properties(host_runtime PROPERTIES
BUILD_RPATH "${ASCEND_HOME_PATH}/lib64;${ASCEND_HOME_PATH}/runtime/lib64;${ASCEND_HOME_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/lib64"
)

set_target_properties(host_runtime PROPERTIES OUTPUT_NAME "host_runtime")

# Apply compiler sanitizers to this host-compiled target. No-op unless
Expand Down
49 changes: 13 additions & 36 deletions src/a2a3/platform/onboard/host/comm_hccl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
#include "hccl/hccl_comm.h"
#include "hccl/hccl_types.h"
#ifdef SIMPLER_ENABLE_PTO_SDMA_WORKSPACE
#include "pto/comm/async/sdma/sdma_workspace_manager.hpp"
#include "pto/comm/workspace.hpp"
#endif

// Thin wrappers around the HCCL public APIs we use. Kept as a translation
Expand Down Expand Up @@ -1076,14 +1076,6 @@ static int domain_alloc_via_ipc(
return 0;
}

// Host wrapper owning one SDMA-enabled Worker's provisioned resources. The
// opaque handle returned to the runner IS this manager; dma_workspace_release()
// destroys it. There is no per-device generation gate: an SDMA-enabled Worker
// owns its provider for its whole life and releases it at finalize.
#ifdef SIMPLER_ENABLE_PTO_SDMA_WORKSPACE
using SdmaManager = pto::comm::sdma::SdmaWorkspaceManager;
#endif

extern "C" uint32_t dma_workspace_supported_mask(void) {
#ifdef SIMPLER_ENABLE_PTO_SDMA_WORKSPACE
return uint32_t{1} << DMA_WORKSPACE_SDMA;
Expand All @@ -1095,9 +1087,8 @@ extern "C" uint32_t dma_workspace_supported_mask(void) {
extern "C" uint32_t dma_workspace_channel_count(void) {
#ifdef SIMPLER_ENABLE_PTO_SDMA_WORKSPACE
// kSdmaMaxChannelGroups, not the device-side kSdmaMaxChannel: the two are the
// same 48 (PTO static_asserts kPostMaxQueues == kSdmaMaxChannelGroups) but
// only this one comes in through the host-safe workspace-manager header, and
// it is what SdmaWorkspaceManager::Init actually creates streams for.
// same 48 (PTO static_asserts kPostMaxQueues == kSdmaMaxChannelGroups) and
// this is the host-visible count used by the unified SDMA workspace provider.
return pto::comm::sdma::kSdmaMaxChannelGroups;
#else
return 0;
Expand All @@ -1115,29 +1106,16 @@ extern "C" int dma_workspace_provision(uint32_t required_mask, uint64_t *addr_ou
if ((required_mask & (uint32_t{1} << DMA_WORKSPACE_SDMA)) == 0) return 0;
if (count <= DMA_WORKSPACE_SDMA) return -1;
try {
auto manager = std::make_unique<SdmaManager>();
bool init_ok = false;
try {
// Init creates the 48 STARS streams + 16KB workspace. It may fail
// after creating only a subset; destructing that partial manager on
// an error-state card can itself stall, so leak it on failure rather
// than risk the stall.
init_ok = manager->Init();
} catch (...) {
LOG_ERROR("SdmaWorkspaceManager::Init threw; abandoning its partial resources");
}
if (!init_ok) {
(void)manager.release();
auto workspace = std::make_unique<pto::comm::Workspace>();
pto::comm::WorkspaceRequest req{};
const auto status = pto::comm::CreateWorkspace(pto::comm::DmaEngine::SDMA, req, workspace.get());
if (status != pto::comm::WorkspaceStatus::Ok || workspace->addr == nullptr) {
pto::comm::AbandonWorkspace(workspace.get());
LOG_ERROR("dma_workspace_provision: SDMA workspace creation failed (status=%d)", static_cast<int>(status));
return -1;
}
const uint64_t addr = reinterpret_cast<uint64_t>(manager->GetWorkspaceAddr());
if (addr == 0) {
(void)manager.release();
LOG_ERROR("dma_workspace_provision: manager returned a null workspace address");
return -1;
}
addr_out[DMA_WORKSPACE_SDMA] = addr;
*handle_out = manager.release();
addr_out[DMA_WORKSPACE_SDMA] = reinterpret_cast<uint64_t>(workspace->addr);
*handle_out = workspace.release();
return 0;
} catch (...) {
LOG_ERROR("dma_workspace_provision: exception while provisioning SDMA");
Expand All @@ -1152,8 +1130,8 @@ extern "C" void dma_workspace_release(void *handle) {
#ifdef SIMPLER_ENABLE_PTO_SDMA_WORKSPACE
if (!handle) return;
try {
std::unique_ptr<SdmaManager> manager(static_cast<SdmaManager *>(handle));
manager.reset();
std::unique_ptr<pto::comm::Workspace> workspace(static_cast<pto::comm::Workspace *>(handle));
pto::comm::DestroyWorkspace(workspace.get());
} catch (...) {
LOG_ERROR("dma_workspace_release: exception while releasing SDMA resources");
}
Expand Down Expand Up @@ -1780,7 +1758,6 @@ extern "C" int comm_destroy(CommHandle h) try {
if (rc == 0) rc = -1;
}
}

// NOTE: we do NOT destroy h->stream — it is caller-owned.
// We also do NOT call aclrtResetDevice / aclFinalize here. Device/ACL
// lifecycle belongs to DeviceRunner, whose finalize() releases all
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,14 @@ register_pto_async_event(AsyncCtx &ctx, const PtoAsyncEvent &event, const PtoAsy
defer_error(ctx, SIMPLER_ERROR_ASYNC_COMPLETION_INVALID);
return;
}

::pto::comm::sdma::SdmaSession sdma_session;
::pto::comm::sdma::detail::LoadSdmaSession(session, sdma_session);
if (!sdma_session.valid || sdma_session.runtimeCtx.postDoneBase == nullptr) {
defer_error(ctx, SIMPLER_ERROR_ASYNC_COMPLETION_INVALID);
return;
}

for (uint32_t queue_id = 0; queue_id < queue_num; ++queue_id) {
register_sdma_post_done_record(
ctx, ::pto::comm::sdma::detail::GetPostDoneRecordAddr(post_done_base, queue_id), post_id
Expand Down
9 changes: 8 additions & 1 deletion src/a5/platform/onboard/host/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,12 @@ target_compile_options(host_runtime
target_compile_definitions(host_runtime PRIVATE SIMPLER_PLATFORM_NAME="a5")

if(SIMPLER_ENABLE_PTO_SDMA_WORKSPACE)
target_compile_definitions(host_runtime PRIVATE SIMPLER_ENABLE_PTO_SDMA_WORKSPACE=1)
target_compile_definitions(host_runtime PRIVATE
SIMPLER_ENABLE_PTO_SDMA_WORKSPACE=1
PTO_COMM_WORKSPACE_SDMA_SUPPORTED=1
PTO_COMM_WORKSPACE_URMA_SUPPORTED=0
PTO_COMM_WORKSPACE_RDMA_SUPPORTED=0
)
endif()
if(SIMPLER_ENABLE_PTO_URMA_WORKSPACE)
target_compile_definitions(host_runtime PRIVATE SIMPLER_ENABLE_PTO_URMA_WORKSPACE=1)
Expand Down Expand Up @@ -181,13 +186,15 @@ target_include_directories(host_runtime

# Prefer libhccl when the CANN package ships it. Newer CANN packages may only
# ship libhcomm, matching pto-isa's own A5 comm tests, so fall back to hcomm.
unset(HCCL_LIB CACHE)
find_library(HCCL_LIB
NAMES hccl
PATHS
${ASCEND_HOME_PATH}/lib64
${ASCEND_ARCH_HOME}/lib64
NO_DEFAULT_PATH
)
unset(HCOMM_LIB CACHE)
find_library(HCOMM_LIB
NAMES hcomm
PATHS
Expand Down
Loading
Loading