diff --git a/.github/workflows/debian-portable.yml b/.github/workflows/debian-portable.yml index 081c0b6b..38b138f6 100644 --- a/.github/workflows/debian-portable.yml +++ b/.github/workflows/debian-portable.yml @@ -47,7 +47,7 @@ jobs: - name: Upgrade Rust toolchain run: | apt-get update - apt-get install -y uuid-dev nasm + apt-get install -y uuid-dev nasm patchelf apt-get install -y --no-install-recommends curl ca-certificates curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ sh -s -- -y --profile minimal --default-toolchain stable @@ -86,6 +86,7 @@ jobs: model_list.json \ model_info.json echo "Created tarball with symlinks preserved" + # The default backend is XRT, so confirm libxrt is bundled. tar tzf $GITHUB_WORKSPACE/fastflowlm_${VERSION}_linux.tar.gz | grep libxrt - name: Upload portable build artifact diff --git a/.gitignore b/.gitignore index 3f010c3b..8bf22b3e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build +build_*/ .vscode __pycache__ _site diff --git a/hrx-integration/.gitignore b/hrx-integration/.gitignore new file mode 100644 index 00000000..993f7275 --- /dev/null +++ b/hrx-integration/.gitignore @@ -0,0 +1,2 @@ +# Fetched HRX release artifact (populated by fetch-hrx-release.sh). +.hrx-release/ diff --git a/hrx-integration/fetch-hrx-release.ps1 b/hrx-integration/fetch-hrx-release.ps1 new file mode 100644 index 00000000..cc64e509 --- /dev/null +++ b/hrx-integration/fetch-hrx-release.ps1 @@ -0,0 +1,157 @@ +<# +.SYNOPSIS + Fetch and verify the pinned HRX amdxdna *Windows public package* release. + +.DESCRIPTION + Windows counterpart of fetch-hrx-release.sh. Downloads the Windows .zip pinned + in hrx-release.env (HRX_RELEASE_ASSET_WINDOWS / HRX_RELEASE_SHA256_WINDOWS), + verifies its checksum, extracts it, locates the HRX CMake package config, and + prints the package prefix to feed find_package(hrx) via CMAKE_PREFIX_PATH. + + After XADX removal, FLM consumes HRX through find_package(hrx CONFIG REQUIRED) + from the public package (CMake package config + hrx.dll/import lib + public + headers), not the former HRX_DIR/HRX_BUILD source+build tree. So configure the + FLM build with: + + -DCMAKE_PREFIX_PATH= + + The archive is extracted into \.hrx-release\. + + When running in GitHub Actions, also writes `prefix=` to + $env:GITHUB_OUTPUT. + +.PARAMETER OutDir + Optional output directory (defaults to \.hrx-release). +#> +[CmdletBinding()] +param( + [string]$OutDir +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$here = $PSScriptRoot +$releaseEnv = Join-Path $here 'hrx-release.env' +if (-not (Test-Path -LiteralPath $releaseEnv)) { + throw "missing $releaseEnv" +} + +# Parse the shell-style .env (KEY=VALUE or KEY="VALUE"; ignore comments/blanks). +$env_vars = @{} +foreach ($line in Get-Content -LiteralPath $releaseEnv) { + $trimmed = $line.Trim() + if ($trimmed -eq '' -or $trimmed.StartsWith('#')) { continue } + if ($trimmed -match '^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$') { + $key = $Matches[1] + $val = $Matches[2].Trim().Trim('"').Trim("'") + $env_vars[$key] = $val + } +} + +function Require-Var([string]$name) { + if (-not $env_vars.ContainsKey($name) -or [string]::IsNullOrWhiteSpace($env_vars[$name])) { + throw "missing $name in $releaseEnv" + } + return $env_vars[$name] +} + +$repo = Require-Var 'HRX_RELEASE_REPO' +$tag = Require-Var 'HRX_RELEASE_TAG' +$asset = Require-Var 'HRX_RELEASE_ASSET_WINDOWS' +$sha256 = (Require-Var 'HRX_RELEASE_SHA256_WINDOWS').ToLower() + +if ($tag -like 'TODO*' -or $asset -like '*TODO*' -or $sha256 -like 'todo*') { + throw "hrx-release.env still has TODO placeholders. The Linux agent must " + + "publish the HRX public package and fill in REPO/TAG/ASSET/SHA256 first." +} + +if (-not $OutDir) { $OutDir = Join-Path $here '.hrx-release' } +New-Item -ItemType Directory -Force -Path $OutDir | Out-Null + +$baseUrl = "https://github.com/$repo/releases/download/$tag" +$zip = Join-Path $OutDir $asset +$shaFile = "$zip.sha256" + +function Download-File([string]$url, [string]$path) { + Write-Host "download: $url" + for ($attempt = 1; $attempt -le 3; $attempt++) { + try { + Invoke-WebRequest -Uri $url -OutFile $path -UseBasicParsing + return + } catch { + if ($attempt -eq 3) { throw } + Start-Sleep -Seconds 2 + } + } +} + +Download-File "$baseUrl/$asset" $zip +Download-File "$baseUrl/$asset.sha256" $shaFile + +$remoteSha = ((Get-Content -LiteralPath $shaFile | Select-Object -First 1) -split '\s+')[0].ToLower() +if ([string]::IsNullOrWhiteSpace($remoteSha)) { throw "empty remote sha256 file: $shaFile" } +if ($remoteSha -ne $sha256) { + throw "remote sha256 mismatch: expected $sha256, got $remoteSha" +} + +$actualSha = (Get-FileHash -LiteralPath $zip -Algorithm SHA256).Hash.ToLower() +if ($actualSha -ne $sha256) { + throw "zip sha256 mismatch: expected $sha256, got $actualSha" +} + +# The archive root directory mirrors the asset name without the .zip suffix. +$artifactRootName = [System.IO.Path]::GetFileNameWithoutExtension($asset) +if ([string]::IsNullOrWhiteSpace($artifactRootName)) { throw "could not determine zip root" } + +$artifactRoot = Join-Path $OutDir $artifactRootName +if (Test-Path -LiteralPath $artifactRoot) { + Remove-Item -LiteralPath $artifactRoot -Recurse -Force +} + +Expand-Archive -LiteralPath $zip -DestinationPath $OutDir -Force + +# Some archives contain the package files at the top level rather than under a +# directory named after the asset; fall back to the extraction dir in that case. +if (-not (Test-Path -LiteralPath $artifactRoot)) { + $artifactRoot = $OutDir +} + +# Locate the HRX CMake package config (hrx-config.cmake / hrxConfig.cmake). Its +# containing dir is what find_package(hrx CONFIG) needs on CMAKE_PREFIX_PATH; we +# report the package prefix (two levels up from lib/cmake/hrx when present). +$configFile = Get-ChildItem -LiteralPath $artifactRoot -Recurse -File ` + -Include 'hrx-config.cmake', 'hrxConfig.cmake' -ErrorAction SilentlyContinue | + Select-Object -First 1 +if (-not $configFile) { + throw "no HRX CMake package config (hrx-config.cmake/hrxConfig.cmake) found " + + "under $artifactRoot -- is this a public HRX package built with the " + + "CMake packaging flow and IREE_HAL_DRIVER_AMDXDNA=ON?" +} + +$configDir = Split-Path -Parent $configFile.FullName +# Prefer the install prefix (…//lib/cmake/hrx -> ); else use the +# config dir directly. Both are valid on CMAKE_PREFIX_PATH. +$prefix = $configDir +if ((Split-Path -Leaf $configDir) -ieq 'hrx') { + $cmakeDir = Split-Path -Parent $configDir + if ((Split-Path -Leaf $cmakeDir) -ieq 'cmake') { + $libDir = Split-Path -Parent $cmakeDir + $prefix = Split-Path -Parent $libDir + } +} + +Write-Host "HRX_ARTIFACT_ROOT=$artifactRoot" +Write-Host "HRX_CMAKE_CONFIG=$($configFile.FullName)" +Write-Host "HRX_CMAKE_PREFIX=$prefix" +Write-Host "" +Write-Host "Configure FLM with: -DCMAKE_PREFIX_PATH=`"$prefix`"" +if ($env:GITHUB_OUTPUT) { + # Append without a BOM: Windows PowerShell 5.1's `Out-File -Encoding utf8` + # emits a UTF-8 BOM, which would corrupt the output key name and make + # steps..outputs.prefix resolve to empty. + [System.IO.File]::AppendAllText( + $env:GITHUB_OUTPUT, + "prefix=$prefix`n", + (New-Object System.Text.UTF8Encoding($false))) +} diff --git a/hrx-integration/fetch-hrx-release.sh b/hrx-integration/fetch-hrx-release.sh new file mode 100755 index 00000000..8b5f238f --- /dev/null +++ b/hrx-integration/fetch-hrx-release.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Fetch and verify the pinned HRX amdxdna *public package* release artifact. +# +# After XADX removal, FLM consumes HRX via find_package(hrx CONFIG REQUIRED) from +# the public package (CMake package config + libhrx.so + public headers), not the +# former HRX_DIR/HRX_BUILD source+build tree. This script downloads + verifies + +# extracts the package and prints its CMake package prefix (feed it to the FLM +# build via -DCMAKE_PREFIX_PATH). The Linux agent owns finalizing this path. +set -euo pipefail + +usage() { + cat <<'USAGE' +usage: fetch-hrx-release.sh [out-dir] + +Downloads the HRX public package pinned in ./hrx-release.env, verifies its +checksum, extracts it, locates the HRX CMake package config, and prints the +package prefix for find_package(hrx). + +When running in GitHub Actions, the script also writes: + prefix= +to $GITHUB_OUTPUT. +USAGE +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$here" +release_env="$here/hrx-release.env" +[[ -f "$release_env" ]] || die "missing $release_env" + +# shellcheck disable=SC1090 +source "$release_env" + +: "${HRX_RELEASE_REPO:?missing HRX_RELEASE_REPO in $release_env}" +: "${HRX_RELEASE_TAG:?missing HRX_RELEASE_TAG in $release_env}" +: "${HRX_RELEASE_ASSET:?missing HRX_RELEASE_ASSET in $release_env}" +: "${HRX_RELEASE_SHA256:?missing HRX_RELEASE_SHA256 in $release_env}" + +case "$HRX_RELEASE_TAG$HRX_RELEASE_ASSET$HRX_RELEASE_SHA256" in + *TODO*) + die "hrx-release.env still has TODO placeholders. The Linux agent must publish the HRX public package and fill in REPO/TAG/ASSET/SHA256 first." + ;; +esac + +out_dir="${1:-$root/.hrx-release}" +mkdir -p "$out_dir" + +base_url="https://github.com/${HRX_RELEASE_REPO}/releases/download/${HRX_RELEASE_TAG}" +tarball="$out_dir/$HRX_RELEASE_ASSET" +sha_file="$tarball.sha256" + +download() { + local url="$1" + local path="$2" + echo "download: $url" + curl -fsSL --retry 3 --retry-delay 2 -o "$path" "$url" +} + +download "$base_url/$HRX_RELEASE_ASSET" "$tarball" +download "$base_url/$HRX_RELEASE_ASSET.sha256" "$sha_file" + +remote_sha="$(awk '{print $1; exit}' "$sha_file")" +[[ -n "$remote_sha" ]] || die "empty remote sha256 file: $sha_file" +[[ "$remote_sha" == "$HRX_RELEASE_SHA256" ]] || + die "remote sha256 mismatch: expected $HRX_RELEASE_SHA256, got $remote_sha" + +actual_sha="$(sha256sum "$tarball" | awk '{print $1}')" +[[ "$actual_sha" == "$HRX_RELEASE_SHA256" ]] || + die "tarball sha256 mismatch: expected $HRX_RELEASE_SHA256, got $actual_sha" + +# Public package ships as .tar.zst (fall back to gzip for legacy assets). +# Decompress zstd via the standalone binary piped into tar, so extraction does +# not depend on tar being built with its zstd plugin, and fail with a clear +# message when the zstd package is not installed. +case "$HRX_RELEASE_ASSET" in + *.tar.zst) + command -v zstd >/dev/null 2>&1 || + die "zstd is required to extract $HRX_RELEASE_ASSET but was not found on PATH (install the 'zstd' package)" + zstd -dc "$tarball" | tar -C "$out_dir" -xf - + ;; + *.tar.gz|*.tgz) tar -C "$out_dir" -xzf "$tarball" ;; + *) tar -C "$out_dir" -xf "$tarball" ;; +esac + +# Locate the HRX CMake package config; its dir feeds find_package(hrx CONFIG). +config_file="$(find "$out_dir" -type f \( -name 'hrx-config.cmake' -o -name 'hrxConfig.cmake' \) 2>/dev/null | head -n1)" +[[ -n "$config_file" ]] || + die "no HRX CMake package config found under $out_dir -- is this a public HRX package built with the CMake packaging flow and IREE_HAL_DRIVER_AMDXDNA=ON?" + +config_dir="$(dirname "$config_file")" +# Prefer the install prefix (...//lib/cmake/hrx -> ); else the config dir. +prefix="$config_dir" +if [[ "$(basename "$config_dir")" == "hrx" ]]; then + cmake_dir="$(dirname "$config_dir")" + if [[ "$(basename "$cmake_dir")" == "cmake" ]]; then + prefix="$(dirname "$(dirname "$cmake_dir")")" + fi +fi + +echo "HRX_CMAKE_CONFIG=$config_file" +echo "HRX_CMAKE_PREFIX=$prefix" +echo "" +echo "Configure FLM with: -DCMAKE_PREFIX_PATH=\"$prefix\"" +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "prefix=$prefix" >> "$GITHUB_OUTPUT" +fi diff --git a/hrx-integration/hrx-release.env b/hrx-integration/hrx-release.env new file mode 100644 index 00000000..bfe65999 --- /dev/null +++ b/hrx-integration/hrx-release.env @@ -0,0 +1,12 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# Pinned HRX (amdxdna) release consumed by the HRX runtime path. +HRX_RELEASE_REPO="jtuyls/hrx" +HRX_RELEASE_TAG="flm-hrx-amdxdna-v2026.07.30" +# For Linux. +HRX_RELEASE_ASSET="hrx-amdxdna-2026.07.30-amdxdna-hal-native-rel-eb0b39f-linux-x86_64.tar.zst" +HRX_RELEASE_SHA256="661ed94051cc6ad04f53739b2df7a791aecb658bc435bd5a6ff3c46716696345" +# For Windows. +HRX_RELEASE_ASSET_WINDOWS="hrx-amdxdna-2026.07.30-amdxdna-hal-native-rel-eb0b39f-windows-x86_64.zip" +HRX_RELEASE_SHA256_WINDOWS="b3af74c9b393ce49ef71a3cbec2cbe92be6d40601858b41009d2bda57eadeb43" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6773768d..dd4d33fe 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,10 @@ cmake_minimum_required(VERSION 3.22) project(flm LANGUAGES CXX VERSION 0.1.0) +if(POLICY CMP0207) + cmake_policy(SET CMP0207 NEW) +endif() + if(NOT DEFINED FLM_VERSION) message(FATAL_ERROR "FLM_VERSION must be specified externally. Use -DFLM_VERSION= when running cmake.") endif() @@ -29,232 +33,190 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/build/) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) # ——————————————————————————————————————————————— -# User-tweakable static build settings +# NPU runtime backend selection. +# +# FLM ships two interchangeable NPU dispatch backends: +# * XRT — the Xilinx Run Time (the historical default), and +# * HRX — the HRX amdxdna runtime. +# +# The reviewer-requested build flag selects between them (0 = XRT, 1 = HRX) with +# XRT as the default, so the XRT build is unchanged and HRX is strictly opt-in. +# FLM_USE_HRX=OFF selects XRT; FLM_USE_HRX=ON selects HRX. The choice drives: +# - which runtime headers/namespace alias the sources see (FLM_USE_HRX define), +# - which prebuilt engine libs are consumed (lib/xrt vs lib/hrx), and +# - which runtime is discovered, linked and bundled below. # ——————————————————————————————————————————————— -option(FLM_PORTABLE_BUILD "Build portable distribution with bundled libraries (XRT dynamic, FFmpeg static)" OFF) +option(FLM_USE_HRX "Use the HRX amdxdna NPU runtime instead of XRT (0=XRT default, 1=HRX)" OFF) +option(FLM_PORTABLE_BUILD "Build portable distribution with bundled runtime libraries" OFF) -if(WIN32) - set(XRT_INCLUDE_DIR C:/dev/XRT/src/runtime_src/core/include CACHE PATH "Where XRT headers live") - set(XRT_LIB_DIR C:/dev/xrtNPUfromDLL CACHE PATH "Where XRT libs live") +if(FLM_USE_HRX) + set(FLM_RUNTIME_NAME "hrx") + message(STATUS "FLM NPU runtime backend: HRX (FLM_USE_HRX=ON)") else() - # On Linux, find pkg-config first, then use it to find XRT - find_package(PkgConfig) - if(PkgConfig_FOUND) - pkg_check_modules(XRT xrt) - if(XRT_FOUND) - message(STATUS "Found XRT via pkg-config") - message(STATUS " XRT include dirs: ${XRT_INCLUDE_DIRS}") - message(STATUS " XRT library dirs: ${XRT_LIBRARY_DIRS}") - message(STATUS " XRT libraries: ${XRT_LIBRARIES}") - endif() - endif() - - # Check if we need to fetch dependencies from source for portable build - # For portable build: always build FFmpeg statically, only fetch XRT if not found - set(NEED_FETCH_XRT FALSE) - set(NEED_BUILD_FFMPEG_STATIC FALSE) - set(NEED_BUILD_ZLIB_STATIC FALSE) - - if(FLM_PORTABLE_BUILD) - # Always build FFmpeg statically for portable builds - set(NEED_BUILD_FFMPEG_STATIC TRUE) - set(NEED_BUILD_ZLIB_STATIC TRUE) - message(STATUS "Portable build: FFmpeg and zlib will be built statically") - - if(NOT XRT_FOUND) - set(NEED_FETCH_XRT TRUE) - message(STATUS "XRT not found on system, will fetch from source for portable build") - else() - message(STATUS "XRT found on system, will use it for portable build") - endif() - endif() - - # Include FetchContent if needed for either XRT, FFmpeg, or zlib - if(NEED_FETCH_XRT OR NEED_BUILD_FFMPEG_STATIC OR NEED_BUILD_ZLIB_STATIC) - include(FetchContent) - endif() + set(FLM_RUNTIME_NAME "xrt") + message(STATUS "FLM NPU runtime backend: XRT (FLM_USE_HRX=OFF, default)") +endif() - if(NEED_FETCH_XRT) - message(STATUS "Fetching XRT from source...") - - # ——————————————————————————————————————————————— - # 1. Fetch XRT first (XDNA driver depends on it) - # ——————————————————————————————————————————————— - set(XRT_GIT_REPO "https://github.com/Xilinx/XRT.git" CACHE STRING "XRT Git repository URL") - set(XRT_GIT_TAG "2.21.75" CACHE STRING "XRT Git tag/branch to fetch") - - FetchContent_Declare( - xrt_source - GIT_REPOSITORY ${XRT_GIT_REPO} - GIT_TAG ${XRT_GIT_TAG} - GIT_SHALLOW TRUE - ) - - # Configure XRT build options - set(XRT_NATIVE_BUILD ON CACHE BOOL "Native XRT build" FORCE) - set(BUILD_DOCS OFF CACHE BOOL "Build documentation" FORCE) - - # Don't compile Alveo support - only base XRT needed - set(XRT_BASE 1 CACHE STRING "Build base XRT only" FORCE) - set(XRT_ALVEO 0 CACHE STRING "Disable Alveo support" FORCE) - - # Disable building tools/utilities that have errors or aren't needed - set(XRT_BUILD_XBMGMT OFF CACHE BOOL "Build xbmgmt" FORCE) - set(XRT_BUILD_XBUTIL OFF CACHE BOOL "Build xbutil" FORCE) - set(XRT_BUILD_XBT OFF CACHE BOOL "Build xbt" FORCE) - - message(STATUS "Fetching XRT sources...") - FetchContent_MakeAvailable(xrt_source) - - # Set XRT paths for XDNA driver - set(XRT_REPO_DIR ${xrt_source_SOURCE_DIR}) # Root of XRT repo - set(XRT_BUILD_DIR ${xrt_source_BINARY_DIR}) - set(XRT_INCLUDE_DIRS ${XRT_REPO_DIR}/src/runtime_src/core/include) - set(XRT_FOUND TRUE) # Mark XRT as found for include path logic - - message(STATUS "XRT fetched successfully") - message(STATUS " XRT repo dir: ${XRT_REPO_DIR}") - message(STATUS " XRT build dir: ${XRT_BUILD_DIR}") - - # Set variables for later use - set(XRT_BUILT_FROM_SOURCE TRUE) - set(XRT_LIBRARY_DIRS ${XRT_BUILD_DIR}/src/runtime_src) - - # Note: XDNA userspace plugin is NOT built here as it requires exact version - # matching with XRT and has compatibility issues. The plugin should be: - # 1. Installed from system packages (libxrt-npu2) - # 2. Or placed in /usr/lib/ or lib/ directory manually - message(STATUS "Note: XDNA plugin (libxrt_driver_xdna.so.2) must be provided separately") - endif() +# Prebuilt engine libraries live in a per-backend subdirectory so the XRT-built +# and HRX-built .so/.dll/.lib never collide (see src/lib/xrt and src/lib/hrx). +set(FLM_ENGINE_LIB_DIR "${CMAKE_SOURCE_DIR}/lib/${FLM_RUNTIME_NAME}") - # ——————————————————————————————————————————————— - # Build FFmpeg as static libraries for portable builds - # ——————————————————————————————————————————————— - if(NEED_BUILD_FFMPEG_STATIC) - set(FFMPEG_GIT_REPO "https://github.com/FFmpeg/FFmpeg.git" CACHE STRING "FFmpeg Git repository URL") - set(FFMPEG_GIT_TAG "n7.1" CACHE STRING "FFmpeg Git tag/branch to fetch") - - FetchContent_Declare( - ffmpeg_source - GIT_REPOSITORY ${FFMPEG_GIT_REPO} - GIT_TAG ${FFMPEG_GIT_TAG} - GIT_SHALLOW TRUE - ) - - message(STATUS "Fetching FFmpeg sources...") - FetchContent_Populate(ffmpeg_source) - - set(FFMPEG_SOURCE_DIR ${ffmpeg_source_SOURCE_DIR}) - set(FFMPEG_BUILD_DIR ${CMAKE_BINARY_DIR}/ffmpeg-build) - set(FFMPEG_INSTALL_DIR ${CMAKE_BINARY_DIR}/ffmpeg-install) - - # Configure and build FFmpeg as static libraries - message(STATUS "Configuring FFmpeg for static build...") - file(MAKE_DIRECTORY ${FFMPEG_BUILD_DIR}) - - # Run FFmpeg configure - execute_process( - COMMAND ${FFMPEG_SOURCE_DIR}/configure - --prefix=${FFMPEG_INSTALL_DIR} - --enable-static - --disable-shared - --disable-programs - --disable-doc - --disable-htmlpages - --disable-manpages - --disable-podpages - --disable-txtpages - --enable-pic - --enable-zlib - --enable-avcodec - --enable-avformat - --enable-avutil - --enable-swscale - --enable-swresample - --disable-vaapi - --disable-libdrm - --disable-bzlib - --disable-lzma - WORKING_DIRECTORY ${FFMPEG_BUILD_DIR} - RESULT_VARIABLE FFMPEG_CONFIGURE_RESULT - OUTPUT_FILE ${FFMPEG_BUILD_DIR}/configure.log - ERROR_FILE ${FFMPEG_BUILD_DIR}/configure.log - ) - - if(NOT FFMPEG_CONFIGURE_RESULT EQUAL 0) - message(FATAL_ERROR "FFmpeg configure failed. Check ${FFMPEG_BUILD_DIR}/configure.log") +# ——————————————————————————————————————————————— +# NPU runtime discovery. +# HRX: consumed from its public CMake package via find_package(hrx). +# XRT: discovered via pkg-config (system install), a manual /opt/xilinx/xrt +# fallback, or fetched from source for a portable build. +# ——————————————————————————————————————————————— +if(FLM_USE_HRX) + find_package(hrx CONFIG REQUIRED) +else() + if(WIN32) + set(XRT_INCLUDE_DIR C:/dev/XRT/src/runtime_src/core/include CACHE PATH "Where XRT headers live") + set(XRT_LIB_DIR C:/dev/xrtNPUfromDLL CACHE PATH "Where XRT libs live") + else() + find_package(PkgConfig) + if(PkgConfig_FOUND) + pkg_check_modules(XRT xrt) + if(XRT_FOUND) + message(STATUS "Found XRT via pkg-config") + message(STATUS " XRT include dirs: ${XRT_INCLUDE_DIRS}") + message(STATUS " XRT library dirs: ${XRT_LIBRARY_DIRS}") + message(STATUS " XRT libraries: ${XRT_LIBRARIES}") + endif() endif() - message(STATUS "Building FFmpeg static libraries...") - # Limit parallelism to avoid resource exhaustion during FFmpeg build - # FFmpeg compilation is memory-intensive, so use at most 4 jobs - if(DEFINED CMAKE_BUILD_PARALLEL_LEVEL AND CMAKE_BUILD_PARALLEL_LEVEL GREATER 4) - set(FFMPEG_BUILD_JOBS 4) - elseif(DEFINED CMAKE_BUILD_PARALLEL_LEVEL) - set(FFMPEG_BUILD_JOBS ${CMAKE_BUILD_PARALLEL_LEVEL}) - else() - set(FFMPEG_BUILD_JOBS 2) - endif() - execute_process( - COMMAND make -j${FFMPEG_BUILD_JOBS} - WORKING_DIRECTORY ${FFMPEG_BUILD_DIR} - RESULT_VARIABLE FFMPEG_BUILD_RESULT - OUTPUT_FILE ${FFMPEG_BUILD_DIR}/build.log - ERROR_FILE ${FFMPEG_BUILD_DIR}/build.log - ) - - if(NOT FFMPEG_BUILD_RESULT EQUAL 0) - message(FATAL_ERROR "FFmpeg build failed. Check ${FFMPEG_BUILD_DIR}/build.log") + # Portable build: fetch XRT from source if it is not installed on the host. + if(FLM_PORTABLE_BUILD AND NOT XRT_FOUND) + message(STATUS "XRT not found on system; fetching from source for portable build") + include(FetchContent) + set(XRT_GIT_REPO "https://github.com/Xilinx/XRT.git" CACHE STRING "XRT Git repository URL") + set(XRT_GIT_TAG "2.21.75" CACHE STRING "XRT Git tag/branch to fetch") + FetchContent_Declare(xrt_source + GIT_REPOSITORY ${XRT_GIT_REPO} + GIT_TAG ${XRT_GIT_TAG} + GIT_SHALLOW TRUE) + set(XRT_NATIVE_BUILD ON CACHE BOOL "Native XRT build" FORCE) + set(BUILD_DOCS OFF CACHE BOOL "Build documentation" FORCE) + set(XRT_BASE 1 CACHE STRING "Build base XRT only" FORCE) + set(XRT_ALVEO 0 CACHE STRING "Disable Alveo support" FORCE) + set(XRT_BUILD_XBMGMT OFF CACHE BOOL "Build xbmgmt" FORCE) + set(XRT_BUILD_XBUTIL OFF CACHE BOOL "Build xbutil" FORCE) + set(XRT_BUILD_XBT OFF CACHE BOOL "Build xbt" FORCE) + FetchContent_MakeAvailable(xrt_source) + set(XRT_REPO_DIR ${xrt_source_SOURCE_DIR}) + set(XRT_BUILD_DIR ${xrt_source_BINARY_DIR}) + set(XRT_INCLUDE_DIRS ${XRT_REPO_DIR}/src/runtime_src/core/include) + set(XRT_FOUND TRUE) + set(XRT_BUILT_FROM_SOURCE TRUE) + set(XRT_LIBRARY_DIRS ${XRT_BUILD_DIR}/src/runtime_src) + message(STATUS "Note: XDNA plugin (libxrt_driver_xdna.so.2) must be provided separately") endif() - message(STATUS "Installing FFmpeg static libraries...") - execute_process( - COMMAND make install - WORKING_DIRECTORY ${FFMPEG_BUILD_DIR} - RESULT_VARIABLE FFMPEG_INSTALL_RESULT - ) - - if(NOT FFMPEG_INSTALL_RESULT EQUAL 0) - message(FATAL_ERROR "FFmpeg install failed") + # Manual fallback if pkg-config didn't find XRT and it wasn't fetched. + if(NOT XRT_FOUND AND NOT XRT_BUILT_FROM_SOURCE) + set(XRT_INCLUDE_DIR /opt/xilinx/xrt/include CACHE PATH "Where XRT headers live") + set(XRT_LIB_DIR /opt/xilinx/xrt/lib CACHE PATH "Where XRT libs live") endif() - set(FFMPEG_BUILT_FROM_SOURCE TRUE) - set(FFMPEG_INCLUDE_DIRS ${FFMPEG_INSTALL_DIR}/include) - set(FFMPEG_LIBRARY_DIRS ${FFMPEG_INSTALL_DIR}/lib) - - message(STATUS "FFmpeg built successfully") - message(STATUS " FFmpeg include dir: ${FFMPEG_INCLUDE_DIRS}") - message(STATUS " FFmpeg library dir: ${FFMPEG_LIBRARY_DIRS}") - endif() - - # ——————————————————————————————————————————————— - # Build zlib as static library for portable builds - # ——————————————————————————————————————————————— - if(NEED_BUILD_ZLIB_STATIC) - message(STATUS "Fetching zlib source...") - - FetchContent_Declare( - zlib_source - GIT_REPOSITORY "https://github.com/madler/zlib.git" - GIT_TAG "v1.3.1" - GIT_SHALLOW TRUE - ) - - FetchContent_MakeAvailable(zlib_source) + # Portable XRT build: statically link FFmpeg (and zlib) so the self- + # contained tarball does not depend on the host having libavformat/ + # libavcodec/... installed. FFmpeg and zlib are fetched and built from + # source; the resulting .a archives are linked into flm below. Non- + # portable builds keep using the system FFmpeg via pkg-config. + if(FLM_PORTABLE_BUILD) + include(FetchContent) + message(STATUS "Portable build: FFmpeg and zlib will be built statically") + + # ——— FFmpeg (static) ——— + set(FFMPEG_GIT_REPO "https://github.com/FFmpeg/FFmpeg.git" CACHE STRING "FFmpeg Git repository URL") + set(FFMPEG_GIT_TAG "n7.1" CACHE STRING "FFmpeg Git tag/branch to fetch") + FetchContent_Declare(ffmpeg_source + GIT_REPOSITORY ${FFMPEG_GIT_REPO} + GIT_TAG ${FFMPEG_GIT_TAG} + GIT_SHALLOW TRUE) + message(STATUS "Fetching FFmpeg sources...") + FetchContent_Populate(ffmpeg_source) + + set(FFMPEG_SOURCE_DIR ${ffmpeg_source_SOURCE_DIR}) + set(FFMPEG_BUILD_DIR ${CMAKE_BINARY_DIR}/ffmpeg-build) + set(FFMPEG_INSTALL_DIR ${CMAKE_BINARY_DIR}/ffmpeg-install) + file(MAKE_DIRECTORY ${FFMPEG_BUILD_DIR}) + + message(STATUS "Configuring FFmpeg for static build...") + execute_process( + COMMAND ${FFMPEG_SOURCE_DIR}/configure + --prefix=${FFMPEG_INSTALL_DIR} + --enable-static + --disable-shared + --disable-programs + --disable-doc + --disable-htmlpages + --disable-manpages + --disable-podpages + --disable-txtpages + --enable-pic + --enable-zlib + --enable-avcodec + --enable-avformat + --enable-avutil + --enable-swscale + --enable-swresample + --disable-vaapi + --disable-libdrm + --disable-bzlib + --disable-lzma + WORKING_DIRECTORY ${FFMPEG_BUILD_DIR} + RESULT_VARIABLE FFMPEG_CONFIGURE_RESULT + OUTPUT_FILE ${FFMPEG_BUILD_DIR}/configure.log + ERROR_FILE ${FFMPEG_BUILD_DIR}/configure.log) + if(NOT FFMPEG_CONFIGURE_RESULT EQUAL 0) + message(FATAL_ERROR "FFmpeg configure failed. Check ${FFMPEG_BUILD_DIR}/configure.log") + endif() - set(ZLIB_BUILT_FROM_SOURCE TRUE) - set(ZLIB_INCLUDE_DIRS ${zlib_source_SOURCE_DIR}) - set(ZLIB_LIBRARY_DIRS ${zlib_source_BINARY_DIR}) - set(ZLIB_STATIC_LIB ${zlib_source_BINARY_DIR}/libz.a) + message(STATUS "Building FFmpeg static libraries...") + # FFmpeg compilation is memory-intensive, so cap parallelism at 4. + if(DEFINED CMAKE_BUILD_PARALLEL_LEVEL AND CMAKE_BUILD_PARALLEL_LEVEL GREATER 4) + set(FFMPEG_BUILD_JOBS 4) + elseif(DEFINED CMAKE_BUILD_PARALLEL_LEVEL) + set(FFMPEG_BUILD_JOBS ${CMAKE_BUILD_PARALLEL_LEVEL}) + else() + set(FFMPEG_BUILD_JOBS 2) + endif() + execute_process( + COMMAND make -j${FFMPEG_BUILD_JOBS} + WORKING_DIRECTORY ${FFMPEG_BUILD_DIR} + RESULT_VARIABLE FFMPEG_BUILD_RESULT + OUTPUT_FILE ${FFMPEG_BUILD_DIR}/build.log + ERROR_FILE ${FFMPEG_BUILD_DIR}/build.log) + if(NOT FFMPEG_BUILD_RESULT EQUAL 0) + message(FATAL_ERROR "FFmpeg build failed. Check ${FFMPEG_BUILD_DIR}/build.log") + endif() - message(STATUS "zlib built successfully") - message(STATUS " zlib static lib: ${ZLIB_STATIC_LIB}") - endif() + message(STATUS "Installing FFmpeg static libraries...") + execute_process( + COMMAND make install + WORKING_DIRECTORY ${FFMPEG_BUILD_DIR} + RESULT_VARIABLE FFMPEG_INSTALL_RESULT) + if(NOT FFMPEG_INSTALL_RESULT EQUAL 0) + message(FATAL_ERROR "FFmpeg install failed") + endif() - # Fallback to manual paths if pkg-config didn't find XRT and FLM_FETCH_XDNA is not enabled - if(NOT XRT_FOUND AND NOT XRT_BUILT_FROM_SOURCE) - set(XRT_INCLUDE_DIR /opt/xilinx/xrt/include CACHE PATH "Where XRT headers live") - set(XRT_LIB_DIR /opt/xilinx/xrt/lib CACHE PATH "Where XRT libs live") + set(FFMPEG_BUILT_FROM_SOURCE TRUE) + set(FFMPEG_INCLUDE_DIRS ${FFMPEG_INSTALL_DIR}/include) + set(FFMPEG_LIBRARY_DIRS ${FFMPEG_INSTALL_DIR}/lib) + message(STATUS "FFmpeg built successfully (static): ${FFMPEG_LIBRARY_DIRS}") + + # ——— zlib (static) ——— + message(STATUS "Fetching zlib source...") + FetchContent_Declare(zlib_source + GIT_REPOSITORY "https://github.com/madler/zlib.git" + GIT_TAG "v1.3.1" + GIT_SHALLOW TRUE) + FetchContent_MakeAvailable(zlib_source) + set(ZLIB_BUILT_FROM_SOURCE TRUE) + set(ZLIB_STATIC_LIB ${zlib_source_BINARY_DIR}/libz.a) + message(STATUS "zlib built successfully (static): ${ZLIB_STATIC_LIB}") + endif() endif() endif() @@ -307,27 +269,48 @@ endif() add_executable(flm ${SOURCES} ${HEADERS}) -if(NOT WIN32) - find_package(Threads REQUIRED) - find_package(Boost REQUIRED COMPONENTS program_options) +if(WIN32) + if(VCPKG_TOOLCHAIN) + # A vcpkg toolchain is active (e.g. the rocm-npu-staging dev.py build or + # a local CMakePresets build): the vcpkg tree ships CMake package configs, + # so resolve the native deps in CONFIG mode. find_package auto-detects the + # correct versioned import-lib names, so this works regardless of the + # installed Boost/FFTW versions. + find_package(Boost CONFIG REQUIRED COMPONENTS program_options) + find_package(CURL CONFIG REQUIRED) + find_package(FFMPEG REQUIRED) + find_package(FFTW3 CONFIG REQUIRED) + find_package(FFTW3f CONFIG REQUIRED) + find_package(FFTW3l CONFIG REQUIRED) + endif() + # Otherwise (the bare self-hosted CI runner) there is no vcpkg toolchain and + # no Boost in vcpkg: Boost is a standalone b2 build under C:/dev/boost_1_88_0 + # and curl/ffmpeg/fftw are linked by raw name from the vcpkg lib dir. That + # path is wired via the include/link dirs and raw library names guarded by + # NOT VCPKG_TOOLCHAIN below. +else() + find_package(Boost CONFIG REQUIRED COMPONENTS program_options) + # The Linux build environment installs these via apt, which provides the + # CMake FindCURL module and pkg-config files rather than CMake package + # configs. Resolve CURL via the module and FFmpeg/FFTW via pkg-config. find_package(CURL REQUIRED) - # PkgConfig is already found earlier for XRT detection - if(PkgConfig_FOUND) - pkg_check_modules(FFTW3_PKG fftw3) - pkg_check_modules(FFTW3F_PKG fftw3f) - pkg_check_modules(FFTW3L_PKG fftw3l) - pkg_check_modules(AVFORMAT_PKG libavformat) - pkg_check_modules(AVCODEC_PKG libavcodec) - pkg_check_modules(AVUTIL_PKG libavutil) - pkg_check_modules(SWSCALE_PKG libswscale) - pkg_check_modules(SWRESAMPLE_PKG libswresample) + find_package(Threads REQUIRED) + find_package(PkgConfig REQUIRED) + pkg_check_modules(FFTW3 REQUIRED IMPORTED_TARGET fftw3) + pkg_check_modules(FFTW3F REQUIRED IMPORTED_TARGET fftw3f) + pkg_check_modules(FFTW3L REQUIRED IMPORTED_TARGET fftw3l) + # Portable builds link FFmpeg statically (built from source above), so the + # system FFmpeg is only resolved via pkg-config for non-portable builds. + if(NOT FFMPEG_BUILT_FROM_SOURCE) + pkg_check_modules(FFMPEG REQUIRED IMPORTED_TARGET + libavformat libavcodec libavutil libswscale libswresample) endif() - # Skip readline for portable builds to avoid GPL dependency + # readline is GPL; skip it for portable builds (the CLI falls back to a + # plain line reader via the FASTFLOWLM_USE_READLINE guard) so the portable + # tarball carries no readline/ncurses runtime dependency. if(NOT FLM_PORTABLE_BUILD) - find_path(READLINE_INCLUDE_DIR readline/readline.h) - find_library(READLINE_LIBRARY readline) - find_library(READLINE_TINFO_LIBRARY tinfo) - find_library(READLINE_NCURSES_LIBRARY ncurses) + pkg_check_modules(readline REQUIRED IMPORTED_TARGET readline) + pkg_check_modules(ncurses REQUIRED IMPORTED_TARGET ncursesw) endif() endif() @@ -339,21 +322,20 @@ target_include_directories(flm PUBLIC ${CMAKE_SOURCE_DIR}/runner ${CMAKE_SOURCE_DIR}/server ${CMAKE_SOURCE_DIR}/pull + ${FFMPEG_INCLUDE_DIRS} ) -# Add XRT include directories -if(NOT WIN32 AND XRT_FOUND) - target_include_directories(flm PUBLIC ${XRT_INCLUDE_DIRS}) -else() - target_include_directories(flm PUBLIC ${XRT_INCLUDE_DIR}) -endif() - -# Add FFmpeg include directories if built from source -if(FFMPEG_BUILT_FROM_SOURCE) - target_include_directories(flm PUBLIC ${FFMPEG_INCLUDE_DIRS}) +# XRT backend: add the XRT headers (HRX carries its own via the hrx::hrx target). +if(NOT FLM_USE_HRX) + if(NOT WIN32 AND XRT_FOUND) + target_include_directories(flm PUBLIC ${XRT_INCLUDE_DIRS}) + else() + target_include_directories(flm PUBLIC ${XRT_INCLUDE_DIR}) + endif() endif() -if(WIN32) +if(WIN32 AND NOT VCPKG_TOOLCHAIN) + # Bare CI runner: standalone Boost + vcpkg headers by absolute path. target_include_directories(flm PUBLIC C:/dev/boost_1_88_0 C:/dev/vcpkg/installed/x64-windows/include/ @@ -368,14 +350,15 @@ target_compile_definitions(flm PUBLIC __NPU_VERSION__=\"${NPU_VERSION}\" ) +# Select the NPU runtime backend seen by the sources (device_runtime.hpp et al.). +if(FLM_USE_HRX) + target_compile_definitions(flm PUBLIC FLM_USE_HRX=1) +endif() + if(WIN32) target_compile_definitions(flm PUBLIC WIN32_LEAN_AND_MEAN NOMINMAX - # Additional definitions for static linking - CURL_STATICLIB - BOOST_ALL_NO_LIB - BOOST_ALL_STATIC_LINK # Handle legacy stdio functions _CRT_SECURE_NO_WARNINGS _CRT_NONSTDC_NO_DEPRECATE @@ -383,18 +366,32 @@ if(WIN32) ) endif() +if(WIN32 AND NOT VCPKG_TOOLCHAIN) + # Bare CI runner: statically link the standalone Boost + curl. + target_compile_definitions(flm PUBLIC + CURL_STATICLIB + BOOST_ALL_NO_LIB + BOOST_ALL_STATIC_LINK + ) +endif() + target_link_directories(flm PUBLIC - ${CMAKE_SOURCE_DIR}/lib + ${FLM_ENGINE_LIB_DIR} # per-backend prebuilt engine libs (lib/xrt or lib/hrx) + ${CMAKE_SOURCE_DIR}/lib # shared third-party runtime libs (ffmpeg/curl/fftw/... on Windows) + ${FFMPEG_LIBRARY_DIRS} ) -# Add XRT library directories -if(NOT WIN32 AND XRT_FOUND) - target_link_directories(flm PUBLIC ${XRT_LIBRARY_DIRS}) -else() - target_link_directories(flm PUBLIC ${XRT_LIB_DIR}) +# XRT backend: add the XRT runtime library dir (aiebu ships in lib/). +if(NOT FLM_USE_HRX) + if(NOT WIN32 AND XRT_FOUND) + target_link_directories(flm PUBLIC ${XRT_LIBRARY_DIRS}) + else() + target_link_directories(flm PUBLIC ${XRT_LIB_DIR}) + endif() endif() -if(WIN32) +if(WIN32 AND NOT VCPKG_TOOLCHAIN) + # Bare CI runner: standalone Boost stage libs + vcpkg import libs. target_link_directories(flm PUBLIC C:/dev/boost_1_88_0/stage/lib C:/dev/vcpkg/installed/x64-windows/lib @@ -406,23 +403,6 @@ if(MSVC) target_link_libraries(flm PUBLIC ${STATIC_LIBS}) endif() -# ——————————————————————————————————————————————— -# Link XRT libraries (always dynamic) -# ——————————————————————————————————————————————— -if(XRT_BUILT_FROM_SOURCE) - # If built from source, link to the shared library targets - if(TARGET xrt_coreutil) - target_link_libraries(flm PUBLIC xrt_coreutil) - message(STATUS " Linked: xrt_coreutil (shared)") - endif() -elseif(NOT WIN32 AND XRT_FOUND) - # Use pkg-config provided libraries - target_link_libraries(flm PUBLIC ${XRT_LIBRARIES}) -else() - # Fallback to manual linking - target_link_libraries(flm PUBLIC xrt_coreutil) -endif() - # Link your custom libraries (these may still be DLLs if no static versions available) target_link_libraries(flm PUBLIC q4_npu_eXpress @@ -449,117 +429,101 @@ target_link_libraries(flm PUBLIC mha ) -if(WIN32) - target_link_libraries(flm PUBLIC - aiebu_static - ) +# Link the selected NPU runtime after the detail model libraries above so their +# runtime references (libhrx / libxrt_coreutil) resolve. +if(FLM_USE_HRX) + # (Windows: hrx.lib import lib for hrx.dll; Linux: libhrx.so.) + target_link_libraries(flm PUBLIC hrx::hrx) + if(NOT WIN32) + set_target_properties(flm PROPERTIES + BUILD_RPATH "$;${FLM_ENGINE_LIB_DIR}") + endif() else() - target_link_libraries(flm PUBLIC - aiebu - ) -endif() - -# Link FFmpeg libraries (static if built from source, dynamic otherwise) -if(FFMPEG_BUILT_FROM_SOURCE) - # Link static FFmpeg libraries - target_link_directories(flm PUBLIC ${FFMPEG_LIBRARY_DIRS}) - if(ZLIB_BUILT_FROM_SOURCE) - set(ZLIB_LIB ${ZLIB_STATIC_LIB}) + # XRT: link xrt_coreutil (+ aiebu for the ELF assembler used by npu_utils_xrt). + if(XRT_BUILT_FROM_SOURCE AND TARGET xrt_coreutil) + target_link_libraries(flm PUBLIC xrt_coreutil) + elseif(NOT WIN32 AND XRT_FOUND) + target_link_libraries(flm PUBLIC ${XRT_LIBRARIES}) + else() + target_link_libraries(flm PUBLIC xrt_coreutil) + endif() + if(WIN32) + target_link_libraries(flm PUBLIC aiebu_static) else() - set(ZLIB_LIB z) + target_link_libraries(flm PUBLIC aiebu) + endif() + if(NOT WIN32) + set_target_properties(flm PROPERTIES + BUILD_RPATH "${FLM_ENGINE_LIB_DIR}") endif() +endif() + +if(WIN32 AND VCPKG_TOOLCHAIN) + # Local/managed vcpkg: link the imported targets from the CONFIG packages + # found above (versioned import-lib names resolved automatically). target_link_libraries(flm PUBLIC - ${FFMPEG_LIBRARY_DIRS}/libavformat.a - ${FFMPEG_LIBRARY_DIRS}/libavcodec.a - ${FFMPEG_LIBRARY_DIRS}/libavutil.a - ${FFMPEG_LIBRARY_DIRS}/libswscale.a - ${FFMPEG_LIBRARY_DIRS}/libswresample.a - ${ZLIB_LIB} + ${FFMPEG_LIBRARIES} + Boost::program_options + CURL::libcurl + FFTW3::fftw3 + FFTW3::fftw3f + FFTW3::fftw3l ) - message(STATUS "Linking static FFmpeg libraries") -else() - # Link dynamic FFmpeg libraries (default) +elseif(WIN32) + # Bare CI runner: link the native deps by raw name from the standalone Boost + # + vcpkg lib dirs wired above (no CMake package configs there). target_link_libraries(flm PUBLIC avformat avcodec avutil swscale swresample - ) -endif() - -if(WIN32) - target_link_libraries(flm PUBLIC libcurl libboost_program_options-vc143-mt-x64-1_88 libfftw3-3 libfftw3f-3 libfftw3l-3 ) -else() +elseif(FFMPEG_BUILT_FROM_SOURCE) + # Portable Linux: link the static FFmpeg archives built above (plus zlib) so + # flm carries no libav*/libsw* runtime dependency. + if(ZLIB_BUILT_FROM_SOURCE) + set(_flm_zlib_lib ${ZLIB_STATIC_LIB}) + else() + set(_flm_zlib_lib z) + endif() target_link_libraries(flm PUBLIC + ${FFMPEG_LIBRARY_DIRS}/libavformat.a + ${FFMPEG_LIBRARY_DIRS}/libavcodec.a + ${FFMPEG_LIBRARY_DIRS}/libavutil.a + ${FFMPEG_LIBRARY_DIRS}/libswscale.a + ${FFMPEG_LIBRARY_DIRS}/libswresample.a + ${_flm_zlib_lib} + Boost::program_options CURL::libcurl + PkgConfig::FFTW3 + PkgConfig::FFTW3F + PkgConfig::FFTW3L + ) +else() + target_link_libraries(flm PUBLIC + ${FFMPEG_LIBRARIES} Boost::program_options + CURL::libcurl + PkgConfig::FFTW3 + PkgConfig::FFTW3F + PkgConfig::FFTW3L ) +endif() +if(NOT WIN32) target_compile_options(flm PUBLIC -mavx -mavx2) - # For portable builds, link FFTW dynamically (bundled at install time) - # For non-portable, use pkg-config if available - if(FFTW3_PKG_FOUND) - target_include_directories(flm PUBLIC ${FFTW3_PKG_INCLUDE_DIRS}) - target_link_libraries(flm PUBLIC ${FFTW3_PKG_LIBRARIES}) - else() - target_link_libraries(flm PUBLIC fftw3) - endif() - if(FFTW3F_PKG_FOUND) - target_include_directories(flm PUBLIC ${FFTW3F_PKG_INCLUDE_DIRS}) - target_link_libraries(flm PUBLIC ${FFTW3F_PKG_LIBRARIES}) - else() - target_link_libraries(flm PUBLIC fftw3f) - endif() - if(FFTW3L_PKG_FOUND) - target_include_directories(flm PUBLIC ${FFTW3L_PKG_INCLUDE_DIRS}) - target_link_libraries(flm PUBLIC ${FFTW3L_PKG_LIBRARIES}) - else() - target_link_libraries(flm PUBLIC fftw3l) - endif() - - # Only use pkg-config FFmpeg if not building static FFmpeg from source - if(NOT FFMPEG_BUILT_FROM_SOURCE) - if(AVFORMAT_PKG_FOUND) - target_include_directories(flm PUBLIC ${AVFORMAT_PKG_INCLUDE_DIRS}) - target_link_libraries(flm PUBLIC ${AVFORMAT_PKG_LIBRARIES}) - endif() - if(AVCODEC_PKG_FOUND) - target_include_directories(flm PUBLIC ${AVCODEC_PKG_INCLUDE_DIRS}) - target_link_libraries(flm PUBLIC ${AVCODEC_PKG_LIBRARIES}) - endif() - if(AVUTIL_PKG_FOUND) - target_include_directories(flm PUBLIC ${AVUTIL_PKG_INCLUDE_DIRS}) - target_link_libraries(flm PUBLIC ${AVUTIL_PKG_LIBRARIES}) - endif() - if(SWSCALE_PKG_FOUND) - target_include_directories(flm PUBLIC ${SWSCALE_PKG_INCLUDE_DIRS}) - target_link_libraries(flm PUBLIC ${SWSCALE_PKG_LIBRARIES}) - endif() - if(SWRESAMPLE_PKG_FOUND) - target_include_directories(flm PUBLIC ${SWRESAMPLE_PKG_INCLUDE_DIRS}) - target_link_libraries(flm PUBLIC ${SWRESAMPLE_PKG_LIBRARIES}) - endif() - endif() - - if(READLINE_LIBRARY) + # readline is only linked for non-portable builds (see the discovery block); + # the CLI compiles a plain line-reader fallback when the macro is undefined. + if(NOT FLM_PORTABLE_BUILD) target_compile_definitions(flm PUBLIC FASTFLOWLM_USE_READLINE=1) - if(READLINE_INCLUDE_DIR) - target_include_directories(flm PUBLIC ${READLINE_INCLUDE_DIR}) - endif() - target_link_libraries(flm PUBLIC ${READLINE_LIBRARY}) - if(READLINE_TINFO_LIBRARY) - target_link_libraries(flm PUBLIC ${READLINE_TINFO_LIBRARY}) - elseif(READLINE_NCURSES_LIBRARY) - target_link_libraries(flm PUBLIC ${READLINE_NCURSES_LIBRARY}) - endif() + target_link_libraries(flm PUBLIC PkgConfig::readline PkgConfig::ncurses) endif() # Link with dl library for dlopen() support (used for preloading bundled libraries) @@ -581,6 +545,7 @@ if(WIN32) target_link_libraries(flm PRIVATE ntdll wsock32 ws2_32 Bcrypt iphlpapi userenv psapi + crypt32 secur32 advapi32 normaliz wldap32 ) else() target_link_libraries(flm PRIVATE Threads::Threads) @@ -598,6 +563,11 @@ if(WIN32) ) endif() +# Default install location for model_list.json / xclbins (matches the app's +# relocatable "/../share/flm" lookup). Overridden to the prefix root for +# the flat portable layout below. +set(FLM_SHARE_DESTINATION "share/flm") + # Add a custom target to check for remaining DLL dependencies if(WIN32) add_custom_target(check_dependencies ALL @@ -611,201 +581,301 @@ else() set(CMAKE_INSTALL_LIBDIR "lib") endif() - file(GLOB so_libs "${CMAKE_SOURCE_DIR}/lib/*.so") - install(FILES ${so_libs} DESTINATION lib) - - # Bundle system libraries for portable distribution + # ———————————————————————————————————————————————————————————— + # Install layout. Portable = flat, self-contained tree rooted at the install + # prefix (consumed by the portable tarball workflow); otherwise an FHS layout + # (bin/, lib/, share/) consumed by the .deb packaging. + # ———————————————————————————————————————————————————————————— if(FLM_PORTABLE_BUILD) - # Copy XRT and XDNA libraries while preserving symlinks - install(CODE " - # Find source directory for XRT libraries - set(XRT_SOURCE_DIR \"\") - if(EXISTS \"/usr/lib/libxrt_coreutil.so.2\") - set(XRT_SOURCE_DIR \"/usr/lib\") - elseif(EXISTS \"/usr/lib/x86_64-linux-gnu/libxrt_coreutil.so.2\") - set(XRT_SOURCE_DIR \"/usr/lib/x86_64-linux-gnu\") - endif() - - if(XRT_SOURCE_DIR) - # Find all XRT and Boost library files (including symlinks) - file(GLOB XRT_FILES \"\${XRT_SOURCE_DIR}/libxrt*.so*\") - file(GLOB BOOST_FILES \"\${XRT_SOURCE_DIR}/libboost_program_options.so*\") - - foreach(LIB_FILE \${XRT_FILES} \${BOOST_FILES}) - get_filename_component(LIB_NAME \"\${LIB_FILE}\" NAME) - set(DEST_FILE \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib/\${LIB_NAME}\") - - # Check if it's a symlink - if(IS_SYMLINK \"\${LIB_FILE}\") - # Get symlink target (relative path) - file(READ_SYMLINK \"\${LIB_FILE}\" LINK_TARGET) - message(STATUS \"Creating symlink: \${LIB_NAME} -> \${LINK_TARGET}\") - execute_process( - COMMAND \${CMAKE_COMMAND} -E create_symlink \"\${LINK_TARGET}\" \"\${DEST_FILE}\" - ) - else() - # Copy regular file - message(STATUS \"Copying: \${LIB_NAME}\") - file(COPY \"\${LIB_FILE}\" DESTINATION \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib/\" - FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE - GROUP_READ GROUP_EXECUTE - WORLD_READ WORLD_EXECUTE) - endif() - endforeach() - - # Patch XRT libraries to use \$ORIGIN as RPATH so they find - # dependencies in the same lib/ directory instead of system paths - find_program(PATCHELF_EXECUTABLE patchelf) - if(PATCHELF_EXECUTABLE) - file(GLOB XRT_LIBS \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib/libxrt_*.so.*\") - foreach(XRT_LIB \${XRT_LIBS}) - get_filename_component(XRT_LIB_NAME \"\${XRT_LIB}\" NAME) - message(STATUS \"Setting RPATH on \${XRT_LIB_NAME}\") - execute_process( - COMMAND \${PATCHELF_EXECUTABLE} --set-rpath \"\$ORIGIN\" \"\${XRT_LIB}\" - RESULT_VARIABLE RET - ) - if(NOT RET EQUAL 0) - message(WARNING \"Failed to set RPATH on \${XRT_LIB_NAME}\") - endif() - endforeach() - else() - message(STATUS \"patchelf not found, skipping RPATH patching for XRT libraries\") - endif() + set(FLM_BIN_DESTINATION ".") + set(FLM_ENGINE_LIB_DESTINATION "${CMAKE_INSTALL_LIBDIR}") + set(FLM_SHARE_DESTINATION ".") + set(FLM_FLM_INSTALL_RPATH "$ORIGIN/${CMAKE_INSTALL_LIBDIR}") + # Engine libs sit next to libhrx in /lib. + set(FLM_ENGINE_INSTALL_RPATH "$ORIGIN") + elseif(FLM_USE_HRX) + # HRX FHS layout: engine libs sit in /lib/flm and the bundled NPU + # runtime (libhrx) one level up in /lib, so engines resolve it via + # $ORIGIN/.. — this separation is what lets the HRX .deb ship libhrx. + set(FLM_BIN_DESTINATION "bin") + set(FLM_ENGINE_LIB_DESTINATION "${CMAKE_INSTALL_LIBDIR}/flm") + set(FLM_SHARE_DESTINATION "share/flm") + set(FLM_FLM_INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/flm;$ORIGIN/../${CMAKE_INSTALL_LIBDIR}") + set(FLM_ENGINE_INSTALL_RPATH "$ORIGIN:$ORIGIN/..") + else() + # XRT FHS layout: identical to upstream main — engine libs install directly + # into /lib and flm resolves them via $ORIGIN/../lib. XRT itself is + # an external dependency (libxrt-npu2) resolved from the system. + set(FLM_BIN_DESTINATION "bin") + set(FLM_ENGINE_LIB_DESTINATION "${CMAKE_INSTALL_LIBDIR}") + set(FLM_SHARE_DESTINATION "share/flm") + set(FLM_FLM_INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}") + set(FLM_ENGINE_INSTALL_RPATH "$ORIGIN") + endif() - # XRT internally constructs library paths using \$XILINX_XRT prefix - # combined with lib/x86_64-linux-gnu. Create the expected structure - # with symlinks so XRT can find libxrt_core.so.2 from its bundled location. - set(XRT_MULTIARCH_DIR \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib/x86_64-linux-gnu\") - file(MAKE_DIRECTORY \${XRT_MULTIARCH_DIR}) - - # Create symlinks for XRT libraries that XRT coreutil looks up - foreach(LIB_NAME libxrt_core.so libxrt_core.so.2 libxrt_core.so.2.21.75 - libxrt_coreutil.so libxrt_coreutil.so.2 libxrt_coreutil.so.2.21.75 - libxrt_driver_xdna.so.2 libxrt_driver_xdna.so.2.21.75 - libxrt_hip.so libxrt_hip.so.2 libxrt_hip.so.2.21.75 - libxrt_hwemu.so libxrt_hwemu.so.2 libxrt_hwemu.so.2.21.75 - libxrt_noop.so libxrt_noop.so.2 libxrt_noop.so.2.21.75 - libxrt_swemu.so libxrt_swemu.so.2 libxrt_swemu.so.2.21.75 - libxrt++.so libxrt++.so.2 libxrt++.so.2.21.75) - if(EXISTS \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib/\${LIB_NAME}\") - set(LINK_TARGET \"../\${LIB_NAME}\") - set(LINK_DEST \"\${XRT_MULTIARCH_DIR}/\${LIB_NAME}\") - if(IS_SYMLINK \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib/\${LIB_NAME}\") - # Read the original symlink target and make it relative to multiarch dir - file(READ_SYMLINK \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib/\${LIB_NAME}\" ORIG_TARGET) - set(LINK_TARGET \"../\${ORIG_TARGET}\") - endif() - message(STATUS \"Creating symlink in multiarch dir: \${LIB_NAME} -> \${LINK_TARGET}\") - execute_process( - COMMAND \${CMAKE_COMMAND} -E create_symlink \"\${LINK_TARGET}\" \"\${LINK_DEST}\" - ) - endif() - endforeach() - else() - message(WARNING \"XRT libraries not found in /usr/lib or /usr/lib/x86_64-linux-gnu\") - endif() - ") - # Copy FFTW libraries while preserving symlinks so the portable - # distribution does not depend on libfftw3 being installed on the host. + file(GLOB so_libs "${FLM_ENGINE_LIB_DIR}/*.so*") + install(FILES ${so_libs} DESTINATION "${FLM_ENGINE_LIB_DESTINATION}") + set_target_properties(flm PROPERTIES INSTALL_RPATH "${FLM_FLM_INSTALL_RPATH}") + + # Engine .so file names, used below to keep them out of the flm dependency + # closure (they are installed explicitly, above). + set(_flm_engine_names "") + set(_flm_engine_post_exclude "") + foreach(_flm_src ${so_libs}) + get_filename_component(_flm_name "${_flm_src}" NAME) + list(APPEND _flm_engine_names "${_flm_name}") + string(REPLACE "." "[.]" _flm_name_re "${_flm_name}") + list(APPEND _flm_engine_post_exclude ".*/${_flm_name_re}$") + endforeach() + + # HRX only: the prebuilt HRX engine .so ship with an absolute build-machine + # RUNPATH and a libhrx.so.0 NEEDED entry. Rewrite the RUNPATH to a + # relocatable $ORIGIN-based one so libhrx resolves at runtime and, crucially, + # so dpkg-shlibdeps can locate it during .deb packaging. The XRT engine .so + # are shipped unmodified: the portable wrapper makes ./lib discoverable via + # LD_LIBRARY_PATH, and for the .deb XRT is an external dependency resolved + # from the system. + if(FLM_USE_HRX) + find_program(PATCHELF_EXECUTABLE patchelf REQUIRED) install(CODE " - # Collect FFTW shared objects from the standard library locations - set(FFTW_FILES \"\") - foreach(SEARCH_DIR \"/usr/lib\" \"/usr/lib/x86_64-linux-gnu\") - file(GLOB DIR_FFTW \"\${SEARCH_DIR}/libfftw3*.so*\") - list(APPEND FFTW_FILES \${DIR_FFTW}) - endforeach() - - if(FFTW_FILES) - foreach(LIB_FILE \${FFTW_FILES}) - get_filename_component(LIB_NAME \"\${LIB_FILE}\" NAME) - set(DEST_FILE \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib/\${LIB_NAME}\") - - if(IS_SYMLINK \"\${LIB_FILE}\") - file(READ_SYMLINK \"\${LIB_FILE}\" LINK_TARGET) - message(STATUS \"Creating symlink: \${LIB_NAME} -> \${LINK_TARGET}\") - execute_process( - COMMAND \${CMAKE_COMMAND} -E create_symlink \"\${LINK_TARGET}\" \"\${DEST_FILE}\" - ) - else() - message(STATUS \"Copying: \${LIB_NAME}\") - file(COPY \"\${LIB_FILE}\" DESTINATION \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib/\" - FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE - GROUP_READ GROUP_EXECUTE - WORLD_READ WORLD_EXECUTE) + set(_flm_engine_names \"${_flm_engine_names}\") + foreach(_flm_name \${_flm_engine_names}) + set(_flm_lib \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${FLM_ENGINE_LIB_DESTINATION}/\${_flm_name}\") + if(EXISTS \"\${_flm_lib}\" AND NOT IS_SYMLINK \"\${_flm_lib}\") + execute_process( + COMMAND \"${PATCHELF_EXECUTABLE}\" --set-rpath \"${FLM_ENGINE_INSTALL_RPATH}\" \"\${_flm_lib}\" + RESULT_VARIABLE _flm_patchelf_rc) + if(NOT _flm_patchelf_rc EQUAL 0) + message(FATAL_ERROR \"patchelf --set-rpath failed for \${_flm_lib}\") endif() - endforeach() - - # Patch FFTW libraries to use \$ORIGIN as RPATH so any transitive - # dependencies resolve from the bundled lib/ directory. - find_program(PATCHELF_EXECUTABLE patchelf) - if(PATCHELF_EXECUTABLE) - file(GLOB FFTW_LIBS \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib/libfftw3*.so.*\") - foreach(FFTW_LIB \${FFTW_LIBS}) - if(NOT IS_SYMLINK \"\${FFTW_LIB}\") - get_filename_component(FFTW_LIB_NAME \"\${FFTW_LIB}\" NAME) - message(STATUS \"Setting RPATH on \${FFTW_LIB_NAME}\") - execute_process( - COMMAND \${PATCHELF_EXECUTABLE} --set-rpath \"\$ORIGIN\" \"\${FFTW_LIB}\" - RESULT_VARIABLE RET - ) - if(NOT RET EQUAL 0) - message(WARNING \"Failed to set RPATH on \${FFTW_LIB_NAME}\") - endif() - endif() - endforeach() - else() - message(STATUS \"patchelf not found, skipping RPATH patching for FFTW libraries\") endif() - else() - message(WARNING \"FFTW libraries not found in /usr/lib or /usr/lib/x86_64-linux-gnu\") - endif() + endforeach() ") endif() + # Portable build ships a small wrapper as `flm`; the packaging step renames the + # real binary to `flm-real` and this wrapper to `flm`. if(FLM_PORTABLE_BUILD) - # Portable build: flat structure with ./flm, ./lib/, ./xclbins/ - set_target_properties(flm PROPERTIES INSTALL_RPATH "$ORIGIN/lib") - else() - # Standard FHS layout - set_target_properties(flm PROPERTIES INSTALL_RPATH "$ORIGIN/../lib") + configure_file( + "${CMAKE_SOURCE_DIR}/flm-wrapper.sh.in" + "${CMAKE_BINARY_DIR}/flm-wrapper.sh" + @ONLY) + install(PROGRAMS "${CMAKE_BINARY_DIR}/flm-wrapper.sh" DESTINATION "${FLM_SHARE_DESTINATION}") endif() endif() -if(FLM_PORTABLE_BUILD) - # Portable build: flat layout - set_target_properties(flm PROPERTIES INSTALL_RPATH "$ORIGIN/lib") - - # Create wrapper script - configure_file( - "${CMAKE_SOURCE_DIR}/flm-wrapper.sh.in" - "${CMAKE_BINARY_DIR}/flm-wrapper.sh" - @ONLY - ) - - # Install everything flatly - renaming happens during packaging - install(TARGETS flm RUNTIME DESTINATION .) - install(FILES model_list.json DESTINATION .) - install(FILES model_info.json DESTINATION .) - install(DIRECTORY xclbins DESTINATION .) - install(FILES "${CMAKE_BINARY_DIR}/flm-wrapper.sh" DESTINATION . RENAME flm-wrapper.sh) +# Directory holding the selected NPU runtime shared library, used by the Windows +# dependency set, the portable XRT bundling, and the HRX dependency closure. HRX +# ships libhrx via its imported target; XRT is a system/library-dir dependency. +if(FLM_USE_HRX) + set(_flm_rt_dep_dir "$") + # Sentinel that never matches a real library name (an empty PRE_EXCLUDE regex + # would match — and thus exclude — everything). + set(_flm_rt_pre_exclude "__flm_no_such_lib__") +elseif(NOT WIN32 AND XRT_FOUND) + set(_flm_rt_dep_dir "${XRT_LIBRARY_DIRS}") else() - # Standard FHS layout - install(TARGETS flm - RUNTIME DESTINATION bin - BUNDLE DESTINATION . - ) - install(FILES model_list.json DESTINATION share/flm) - install(FILES model_info.json DESTINATION share/flm) - install(DIRECTORY xclbins DESTINATION share/flm) + set(_flm_rt_dep_dir "${XRT_LIB_DIR}") endif() -if(NOT WIN32 AND NOT FLM_PORTABLE_BUILD AND NOT CMAKE_INSTALL_PREFIX STREQUAL "/usr" AND NOT CMAKE_INSTALL_PREFIX STREQUAL "/usr/local") +if(WIN32) + install(TARGETS flm + RUNTIME_DEPENDENCY_SET fastflowlm_runtime_dependencies + RUNTIME DESTINATION bin) + install(RUNTIME_DEPENDENCY_SET fastflowlm_runtime_dependencies + DESTINATION bin + DIRECTORIES + "${FLM_ENGINE_LIB_DIR}" + "${CMAKE_SOURCE_DIR}/lib" + "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/bin" + "${_flm_rt_dep_dir}" + PRE_EXCLUDE_REGEXES "api-ms-.*" "ext-ms-.*" + # Optional Windows feature-on-demand / attestation / lockdown DLLs + # that are not present as files on all hosts, so the resolver would + # report them unresolved. They are OS components, never bundled. + # Patterns are case-insensitive because PE import names are lowercase. + "[Aa][Zz][Uu][Rr][Ee][Aa][Tt][Tt][Ee][Ss][Tt].*" + "[Hh][Vv][Ss][Ii][Ff][Ii][Ll][Ee][Tt][Rr][Uu][Ss][Tt].*" + "[Pp][Dd][Mm][Uu][Tt][Ii][Ll][Ii][Tt][Ii][Ee][Ss].*" + "[Ww][Pp][Aa][Xx][Hh][Oo][Ll][Dd][Ee][Rr].*" + POST_EXCLUDE_REGEXES ".*[Ww]indows[/\\\\][Ss]ystem32[/\\\\].*") +elseif(NOT FLM_USE_HRX AND FLM_PORTABLE_BUILD) + # Portable XRT: bundle the XRT runtime explicitly. XRT is deliberately + # handled here instead of via the dependency closure below: flm only lists + # libxrt_coreutil as a link-time NEEDED entry, while XRT dlopens libxrt_core, + # the libxrt_driver_xdna plugin and others at run time via a path it builds + # as $XILINX_XRT/lib/x86_64-linux-gnu/. Those are invisible to + # file(GET_RUNTIME_DEPENDENCIES), so copy the whole libxrt*.so* set (plus + # boost_program_options), re-root each real lib at $ORIGIN, and mirror them + # under lib/x86_64-linux-gnu for XRT's internal lookup. FFTW is bundled the + # same way just below. Other shared deps (libcurl, libgomp, ...) resolve from + # the host. + install(TARGETS flm RUNTIME DESTINATION "${FLM_BIN_DESTINATION}") + find_program(PATCHELF_EXECUTABLE patchelf REQUIRED) install(CODE " - message(STATUS \"Creating symlink for flm\") - file(MAKE_DIRECTORY \"$ENV{DESTDIR}/usr/local/bin\") - execute_process(COMMAND \"${CMAKE_COMMAND}\" -E create_symlink - \"${CMAKE_INSTALL_PREFIX}/bin/flm\" - \"$ENV{DESTDIR}/usr/local/bin/flm\")" - ) + # Locate a directory that actually holds the XRT runtime libraries. + set(_flm_xrt_src \"\") + foreach(_cand \"${_flm_rt_dep_dir}\" \"/usr/lib/x86_64-linux-gnu\" \"/usr/lib\") + if(_cand AND EXISTS \"\${_cand}/libxrt_coreutil.so.2\") + set(_flm_xrt_src \"\${_cand}\") + break() + endif() + endforeach() + if(NOT _flm_xrt_src) + message(FATAL_ERROR + \"Portable XRT build: could not locate libxrt_coreutil.so.2 to bundle\") + endif() + message(STATUS \"Bundling XRT runtime libraries from \${_flm_xrt_src}\") + + set(_flm_libdir \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}\") + file(MAKE_DIRECTORY \"\${_flm_libdir}\") + + # Copy every libxrt*.so* (from the XRT dir) plus libboost_program_options.so* + # (searched across the standard system dirs, since Boost may live apart + # from XRT — e.g. XRT under /opt/xilinx but Boost in the multiarch dir), + # preserving the symlink chain. + file(GLOB _flm_xrt_files \"\${_flm_xrt_src}/libxrt*.so*\") + foreach(_bdir \"\${_flm_xrt_src}\" \"/usr/lib/x86_64-linux-gnu\" \"/usr/lib\") + file(GLOB _flm_boost_files \"\${_bdir}/libboost_program_options.so*\") + if(_flm_boost_files) + list(APPEND _flm_xrt_files \${_flm_boost_files}) + break() + endif() + endforeach() + foreach(_f \${_flm_xrt_files}) + get_filename_component(_n \"\${_f}\" NAME) + if(IS_SYMLINK \"\${_f}\") + file(READ_SYMLINK \"\${_f}\" _t) + execute_process(COMMAND \"\${CMAKE_COMMAND}\" -E create_symlink + \"\${_t}\" \"\${_flm_libdir}/\${_n}\") + else() + file(COPY \"\${_f}\" DESTINATION \"\${_flm_libdir}\" + FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + endif() + endforeach() + + # Re-root the real XRT libs at \$ORIGIN so they find each other in lib/. + file(GLOB _flm_xrt_real \"\${_flm_libdir}/libxrt*.so.*\") + foreach(_l \${_flm_xrt_real}) + if(NOT IS_SYMLINK \"\${_l}\") + execute_process( + COMMAND \"${PATCHELF_EXECUTABLE}\" --set-rpath \"\$ORIGIN\" \"\${_l}\" + RESULT_VARIABLE _rc) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR \"patchelf --set-rpath failed for \${_l}\") + endif() + endif() + endforeach() + + # Mirror the bundled XRT libs under lib/x86_64-linux-gnu for XRT's lookup. + set(_flm_multiarch \"\${_flm_libdir}/x86_64-linux-gnu\") + file(MAKE_DIRECTORY \"\${_flm_multiarch}\") + file(GLOB _flm_xrt_all \"\${_flm_libdir}/libxrt*.so*\") + foreach(_l \${_flm_xrt_all}) + get_filename_component(_n \"\${_l}\" NAME) + if(IS_SYMLINK \"\${_l}\") + file(READ_SYMLINK \"\${_l}\" _t) + execute_process(COMMAND \"\${CMAKE_COMMAND}\" -E create_symlink + \"../\${_t}\" \"\${_flm_multiarch}/\${_n}\") + else() + execute_process(COMMAND \"\${CMAKE_COMMAND}\" -E create_symlink + \"../\${_n}\" \"\${_flm_multiarch}/\${_n}\") + endif() + endforeach() + ") + # Bundle FFTW so the portable tree does not depend on libfftw3 on the host. + install(CODE " + set(_flm_libdir \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}\") + set(_flm_fftw_files \"\") + foreach(_dir \"${_flm_rt_dep_dir}\" \"/usr/lib/x86_64-linux-gnu\" \"/usr/lib\") + if(_dir) + file(GLOB _dir_fftw \"\${_dir}/libfftw3*.so*\") + list(APPEND _flm_fftw_files \${_dir_fftw}) + endif() + endforeach() + foreach(_f \${_flm_fftw_files}) + get_filename_component(_n \"\${_f}\" NAME) + if(IS_SYMLINK \"\${_f}\") + file(READ_SYMLINK \"\${_f}\" _t) + execute_process(COMMAND \"\${CMAKE_COMMAND}\" -E create_symlink + \"\${_t}\" \"\${_flm_libdir}/\${_n}\") + else() + file(COPY \"\${_f}\" DESTINATION \"\${_flm_libdir}\" + FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + endif() + endforeach() + file(GLOB _flm_fftw_real \"\${_flm_libdir}/libfftw3*.so.*\") + foreach(_l \${_flm_fftw_real}) + if(NOT IS_SYMLINK \"\${_l}\") + execute_process( + COMMAND \"${PATCHELF_EXECUTABLE}\" --set-rpath \"\$ORIGIN\" \"\${_l}\" + RESULT_VARIABLE _rc) + endif() + endforeach() + ") +elseif(NOT FLM_USE_HRX) + # System XRT (.deb): XRT is an external dependency (the libxrt-npu2 package) + # and flm's other shared libraries are resolved from the system via + # ${shlibs:Depends}. Nothing is bundled here. + install(TARGETS flm RUNTIME DESTINATION "${FLM_BIN_DESTINATION}") +else() + # HRX: bundle flm's third-party runtime dependencies into + # ${CMAKE_INSTALL_LIBDIR} (libhrx plus the engine .so private deps). + # + # Subtlety: some private dependencies (libgomp, libmvec, ...) are pulled in + # ONLY by the prebuilt engine .so, not by flm itself. The engine .so are + # installed and RUNPATH-patched explicitly into FLM_ENGINE_LIB_DESTINATION + # above, so the resolver must NOT emit a second, unpatched copy of them + # (which would retain the absolute build-machine RUNPATH) — but their private + # deps DO need to be in the closure. + # + # A RUNTIME_DEPENDENCY_SET rooted only at flm cannot express this: listing + # the engine .so in POST_EXCLUDE_REGEXES also stops traversal INTO them, so + # their private deps silently drop out of the bundle (that is exactly how + # libgomp/libmvec went missing). Instead, scan flm together with the engine + # .so as ROOTS via file(GET_RUNTIME_DEPENDENCIES): roots are traversed for + # their dependencies but are never themselves emitted, so libgomp/libmvec are + # captured while the engine .so stay solely under the patched install above. + # The engine names are still POST_EXCLUDEd as a belt-and-suspenders guard for + # the case of one engine lib depending on another. + install(TARGETS flm RUNTIME DESTINATION "${FLM_BIN_DESTINATION}") + install(CODE " + set(_flm_engine_libs \"${so_libs}\") + set(_flm_engine_post_exclude \"${_flm_engine_post_exclude}\") + file(GET_RUNTIME_DEPENDENCIES + EXECUTABLES \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${FLM_BIN_DESTINATION}/flm\" + LIBRARIES \${_flm_engine_libs} + RESOLVED_DEPENDENCIES_VAR _flm_resolved + UNRESOLVED_DEPENDENCIES_VAR _flm_unresolved + DIRECTORIES + \"${FLM_ENGINE_LIB_DIR}\" + \"${CMAKE_SOURCE_DIR}/lib\" + \"${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib\" + \"${_flm_rt_dep_dir}\" + PRE_EXCLUDE_REGEXES + \"linux-vdso\\\\.so.*\" + \"ld-linux.*\" + \"lib(c|dl|m|pthread|rt)\\\\.so.*\" + \"${_flm_rt_pre_exclude}\" + POST_EXCLUDE_REGEXES \${_flm_engine_post_exclude}) + if(_flm_unresolved) + message(FATAL_ERROR + \"FastFlowLM runtime dependency closure is incomplete; unresolved: \${_flm_unresolved}\") + endif() + # file(INSTALL) prepends \$ENV{DESTDIR} itself, so DESTINATION must not. + file(INSTALL + DESTINATION \"\${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}\" + TYPE SHARED_LIBRARY + FOLLOW_SYMLINK_CHAIN + FILES \${_flm_resolved}) + ") endif() + +install(FILES model_list.json DESTINATION "${FLM_SHARE_DESTINATION}") +install(FILES model_info.json DESTINATION "${FLM_SHARE_DESTINATION}") + +# xclbins, which are loaded by shared libraries need to be in location +# relative to the executable, so we install them relative to the binary. +install(DIRECTORY xclbins DESTINATION "${FLM_SHARE_DESTINATION}") diff --git a/src/common/AutoEmbeddingModel/auto_embedding_model.cpp b/src/common/AutoEmbeddingModel/auto_embedding_model.cpp index d51f65af..4dee9ee1 100644 --- a/src/common/AutoEmbeddingModel/auto_embedding_model.cpp +++ b/src/common/AutoEmbeddingModel/auto_embedding_model.cpp @@ -11,7 +11,7 @@ std::unordered_set embeddingModelTags = { "embed-gemma", "embed-gemma:300m" }; -AutoEmbeddingModel::AutoEmbeddingModel(xrt::device* npu_device_inst, std::string current_model) { +AutoEmbeddingModel::AutoEmbeddingModel(flm_rt::device* npu_device_inst, std::string current_model) { this->npu_device_inst = npu_device_inst; this->current_model = current_model; } diff --git a/src/common/AutoEmbeddingModel/modeling_gemma_embedding.cpp b/src/common/AutoEmbeddingModel/modeling_gemma_embedding.cpp index ba10efb8..375c0b0b 100644 --- a/src/common/AutoEmbeddingModel/modeling_gemma_embedding.cpp +++ b/src/common/AutoEmbeddingModel/modeling_gemma_embedding.cpp @@ -8,7 +8,7 @@ #include "AutoEmbeddingModel/modeling_gemma_embedding.hpp" #include "models/gemma_embedding/gemma_embedding.hpp" -Gemma_Embedding::Gemma_Embedding(xrt::device* npu_device_inst) : AutoEmbeddingModel(npu_device_inst, "embed-gemma:300m") { +Gemma_Embedding::Gemma_Embedding(flm_rt::device* npu_device_inst) : AutoEmbeddingModel(npu_device_inst, "embed-gemma:300m") { } Gemma_Embedding::~Gemma_Embedding() = default; diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index ab4ee4b5..b81df50a 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -8,7 +8,7 @@ #include "AutoModel/automodel.hpp" -AutoModel::AutoModel(xrt::device* npu_device_inst, std::string current_model) { +AutoModel::AutoModel(flm_rt::device* npu_device_inst, std::string current_model) { this->npu_device_inst = npu_device_inst; this->current_model = current_model; this->total_tokens = 0; diff --git a/src/common/AutoModel/modeling_gemma3.cpp b/src/common/AutoModel/modeling_gemma3.cpp index 8334bb89..90409216 100644 --- a/src/common/AutoModel/modeling_gemma3.cpp +++ b/src/common/AutoModel/modeling_gemma3.cpp @@ -8,7 +8,7 @@ #include "AutoModel/modeling_gemma3.hpp" /************ Gemma3 family **************/ -Gemma3::Gemma3(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma3") {} +Gemma3::Gemma3(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma3") {} void Gemma3::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/AutoModel/modeling_gemma3_text.cpp b/src/common/AutoModel/modeling_gemma3_text.cpp index ea3384bb..bb4826be 100644 --- a/src/common/AutoModel/modeling_gemma3_text.cpp +++ b/src/common/AutoModel/modeling_gemma3_text.cpp @@ -9,7 +9,7 @@ /************ Gemma3_Text_Only family **************/ -Gemma3_Text_Only::Gemma3_Text_Only(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma3_Text_Only") {} +Gemma3_Text_Only::Gemma3_Text_Only(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma3_Text_Only") {} void Gemma3_Text_Only::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { diff --git a/src/common/AutoModel/modeling_gemma4e.cpp b/src/common/AutoModel/modeling_gemma4e.cpp index f46148cc..2d705447 100644 --- a/src/common/AutoModel/modeling_gemma4e.cpp +++ b/src/common/AutoModel/modeling_gemma4e.cpp @@ -410,7 +410,7 @@ std::pair parse_gemma4e_tool_content(std::string tool_content /************ Gemma4e family **************/ -Gemma4e::Gemma4e(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma4e") {} +Gemma4e::Gemma4e(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Gemma4e") {} void Gemma4e::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { diff --git a/src/common/AutoModel/modeling_gpt_oss.cpp b/src/common/AutoModel/modeling_gpt_oss.cpp index 82b32418..51228bdb 100644 --- a/src/common/AutoModel/modeling_gpt_oss.cpp +++ b/src/common/AutoModel/modeling_gpt_oss.cpp @@ -7,7 +7,7 @@ #include "AutoModel/modeling_gpt_oss.hpp" -GPT_OSS::GPT_OSS(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "gpt-oss") {} +GPT_OSS::GPT_OSS(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "gpt-oss") {} void GPT_OSS::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->model_path = model_path; diff --git a/src/common/AutoModel/modeling_lfm2.cpp b/src/common/AutoModel/modeling_lfm2.cpp index f788e063..82384c7a 100644 --- a/src/common/AutoModel/modeling_lfm2.cpp +++ b/src/common/AutoModel/modeling_lfm2.cpp @@ -10,7 +10,7 @@ #include "metrices.hpp" /************ LFM2 family **************/ -LFM2::LFM2(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "LFM2") {} +LFM2::LFM2(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "LFM2") {} void LFM2::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); @@ -265,7 +265,7 @@ StreamResult LFM2::parse_stream_content(const std::string content) { /*********** LFM2_5_TK family ***********/ -LFM2_5_TK::LFM2_5_TK(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "LFM2_5_TK") {} +LFM2_5_TK::LFM2_5_TK(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "LFM2_5_TK") {} void LFM2_5_TK::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/AutoModel/modeling_llama3.cpp b/src/common/AutoModel/modeling_llama3.cpp index d2f3d5f1..5ebcbd1d 100644 --- a/src/common/AutoModel/modeling_llama3.cpp +++ b/src/common/AutoModel/modeling_llama3.cpp @@ -8,7 +8,7 @@ #include "AutoModel/modeling_llama3.hpp" /************ Llama3 family **************/ -Llama3::Llama3(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Llama3") {} +Llama3::Llama3(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Llama3") {} void Llama3::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); @@ -90,7 +90,7 @@ std::string Llama3::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform } /************ DeepSeek_r1_8b family **************/ -DeepSeek_r1_8b::DeepSeek_r1_8b(xrt::device* npu_device_inst) : AutoModel(npu_device_inst) {} +DeepSeek_r1_8b::DeepSeek_r1_8b(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst) {} void DeepSeek_r1_8b::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/AutoModel/modeling_nanbeige.cpp b/src/common/AutoModel/modeling_nanbeige.cpp index 6c18f04a..4138d1b3 100644 --- a/src/common/AutoModel/modeling_nanbeige.cpp +++ b/src/common/AutoModel/modeling_nanbeige.cpp @@ -8,7 +8,7 @@ #include "AutoModel/modeling_nanbeige.hpp" /************ Nanbeige family **************/ -Nanbeige::Nanbeige(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Nanbeige") {} +Nanbeige::Nanbeige(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Nanbeige") {} void Nanbeige::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/AutoModel/modeling_phi4.cpp b/src/common/AutoModel/modeling_phi4.cpp index b02628e8..06c164f5 100644 --- a/src/common/AutoModel/modeling_phi4.cpp +++ b/src/common/AutoModel/modeling_phi4.cpp @@ -8,7 +8,7 @@ #include "AutoModel/modeling_phi4.hpp" /************ Phi4 family **************/ -Phi4::Phi4(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Phi4") {} +Phi4::Phi4(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Phi4") {} void Phi4::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/AutoModel/modeling_qwen2.cpp b/src/common/AutoModel/modeling_qwen2.cpp index 8f8513b5..3cd62b44 100644 --- a/src/common/AutoModel/modeling_qwen2.cpp +++ b/src/common/AutoModel/modeling_qwen2.cpp @@ -9,7 +9,7 @@ #include "AutoModel/modeling_qwen2.hpp" /************ Qwen2 family **************/ -Qwen2::Qwen2(xrt::device* npu_device_inst) : AutoModel(npu_device_inst) {} +Qwen2::Qwen2(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst) {} void Qwen2::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/AutoModel/modeling_qwen2vl.cpp b/src/common/AutoModel/modeling_qwen2vl.cpp index bc6ceba7..8701538f 100644 --- a/src/common/AutoModel/modeling_qwen2vl.cpp +++ b/src/common/AutoModel/modeling_qwen2vl.cpp @@ -12,7 +12,7 @@ /************ Qwen2VL family **************/ -Qwen2VL::Qwen2VL(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen2VL") {} +Qwen2VL::Qwen2VL(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen2VL") {} void Qwen2VL::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/AutoModel/modeling_qwen3.cpp b/src/common/AutoModel/modeling_qwen3.cpp index 299c4101..0be89dc5 100644 --- a/src/common/AutoModel/modeling_qwen3.cpp +++ b/src/common/AutoModel/modeling_qwen3.cpp @@ -10,7 +10,7 @@ /************ Qwen3 family **************/ -Qwen3::Qwen3(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen3") {} +Qwen3::Qwen3(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen3") {} void Qwen3::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); @@ -307,7 +307,7 @@ StreamResult Qwen3::parse_stream_content(const std::string content) { } /************ Qwen3_IT family **************/ -Qwen3_IT::Qwen3_IT(xrt::device* npu_device_inst) : AutoModel(npu_device_inst) {} +Qwen3_IT::Qwen3_IT(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst) {} void Qwen3_IT::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); @@ -458,7 +458,7 @@ StreamResult Qwen3_IT::parse_stream_content(const std::string content) { } /************ Qwen3_TK family **************/ -Qwen3_TK::Qwen3_TK(xrt::device* npu_device_inst) : AutoModel(npu_device_inst) {} +Qwen3_TK::Qwen3_TK(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst) {} void Qwen3_TK::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); @@ -680,7 +680,7 @@ StreamResult Qwen3_TK::parse_stream_content(const std::string content) { } /************ DeepSeek_r1_0528_8b family **************/ -DeepSeek_r1_0528_8b::DeepSeek_r1_0528_8b(xrt::device* npu_device_inst) : AutoModel(npu_device_inst) {} +DeepSeek_r1_0528_8b::DeepSeek_r1_0528_8b(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst) {} void DeepSeek_r1_0528_8b::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/AutoModel/modeling_qwen3_5_omni.cpp b/src/common/AutoModel/modeling_qwen3_5_omni.cpp index 43649e45..1fdab50f 100644 --- a/src/common/AutoModel/modeling_qwen3_5_omni.cpp +++ b/src/common/AutoModel/modeling_qwen3_5_omni.cpp @@ -4,7 +4,7 @@ #include "AutoModel/modeling_qwen3_5_omni.hpp" -Qwen3_5_Omni::Qwen3_5_Omni(xrt::device* npu_device_inst) +Qwen3_5_Omni::Qwen3_5_Omni(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst) {} /// \brief Pull the thinker-scope vision config (with VL fallbacks). diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index 37900818..a3b2d2a1 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -11,7 +11,7 @@ /************ Qwen3_5VL family **************/ -Qwen3_5VL::Qwen3_5VL(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen3_5VL") {} +Qwen3_5VL::Qwen3_5VL(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen3_5VL") {} void Qwen3_5VL::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/AutoModel/modeling_qwen3_6_moe.cpp b/src/common/AutoModel/modeling_qwen3_6_moe.cpp index 11af490c..5fb0f168 100644 --- a/src/common/AutoModel/modeling_qwen3_6_moe.cpp +++ b/src/common/AutoModel/modeling_qwen3_6_moe.cpp @@ -11,7 +11,7 @@ /************ Qwen3_6_MOE family **************/ -Qwen3_6_MOE::Qwen3_6_MOE(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen3_6_MOE") {} +Qwen3_6_MOE::Qwen3_6_MOE(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen3_6_MOE") {} void Qwen3_6_MOE::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/AutoModel/modeling_qwen3vl.cpp b/src/common/AutoModel/modeling_qwen3vl.cpp index 1b3b81dc..0884517e 100644 --- a/src/common/AutoModel/modeling_qwen3vl.cpp +++ b/src/common/AutoModel/modeling_qwen3vl.cpp @@ -11,7 +11,7 @@ /************ Qwen3VL family **************/ -Qwen3VL::Qwen3VL(xrt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen3VL") {} +Qwen3VL::Qwen3VL(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen3VL") {} void Qwen3VL::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); diff --git a/src/common/utils.cpp b/src/common/utils.cpp index a3576bfa..d4fd9aea 100644 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -31,30 +31,27 @@ std::string find_model_list() { } } -#ifndef _WIN32 - // Linux: Portable - // if (std::filesystem::exists("model_list.json")) { - // return "model_list.json"; - // } + // Portable development-tree location (next to the executable, then CWD). std::string exe_dir = get_executable_directory(); std::string exe_relative_path = exe_dir + "/model_list.json"; if (std::filesystem::exists(exe_relative_path)) { return exe_relative_path; } + if (std::filesystem::exists("model_list.json")) { + return "model_list.json"; + } - // Linux: install + // Relocatable installed bundle, independent of its original prefix. + std::string bundle_path = exe_dir + "/../share/flm/model_list.json"; + if (std::filesystem::exists(bundle_path)) { + return bundle_path; + } + + // Legacy configured prefix. std::string installed_path = install_prefix + "/share/flm/model_list.json"; if (std::filesystem::exists(installed_path)) { return installed_path; } -#else - // Windows: Check relative to executable - std::string exe_dir = get_executable_directory(); - std::string exe_relative_path = exe_dir + "\\model_list.json"; - if (std::filesystem::exists(exe_relative_path)) { - return exe_relative_path; - } -#endif // If not found, throw an error throw std::runtime_error("model_list.json not found. Please set FLM_CONFIG_PATH or place it next to the executable."); @@ -124,29 +121,26 @@ std::string find_xclbin_path() { } } -#ifndef _WIN32 - // Linux: Portable - // if (std::filesystem::exists("xclbins")) { - // return "."; - // } + // Portable development-tree location (next to the executable, then CWD). std::string exe_dir = get_executable_directory(); - std::string exe_relative_path = exe_dir; - if (std::filesystem::exists(exe_relative_path + "/xclbins")) { - return exe_relative_path; + if (std::filesystem::exists(exe_dir + "/xclbins")) { + return exe_dir; } - // Linux: install + if (std::filesystem::exists("xclbins")) { + return "."; + } + + // Relocatable installed bundle. The caller appends /xclbins. + std::string bundle_path = exe_dir + "/../share/flm"; + if (std::filesystem::exists(bundle_path + "/xclbins")) { + return bundle_path; + } + + // Legacy configured prefix. std::string installed_path = xclbin_prefix; - if (std::filesystem::exists(installed_path)) { + if (std::filesystem::exists(installed_path + "/xclbins")) { return installed_path; } -#else - // Windows: Check relative to executable - std::string exe_dir = get_executable_directory(); - std::string exe_relative_path = exe_dir; - if (std::filesystem::exists(exe_relative_path)) { - return exe_relative_path; - } -#endif // If not found, throw an error throw std::runtime_error("xclbins not found. Please set FLM_XCLBIN_PATH or place it next to the executable."); diff --git a/src/common/whisper/modeling_whisper.cpp b/src/common/whisper/modeling_whisper.cpp index afac93c7..513ab0cb 100644 --- a/src/common/whisper/modeling_whisper.cpp +++ b/src/common/whisper/modeling_whisper.cpp @@ -7,7 +7,7 @@ #include "whisper/modeling_whisper.hpp" -Whisper::Whisper(xrt::device* npu_device_inst){ +Whisper::Whisper(flm_rt::device* npu_device_inst){ this->device = npu_device_inst; time_stamp = 0; diff --git a/src/flm-wrapper.sh.in b/src/flm-wrapper.sh.in index 201a5667..c9c509fe 100644 --- a/src/flm-wrapper.sh.in +++ b/src/flm-wrapper.sh.in @@ -1,36 +1,45 @@ #!/bin/bash -# Wrapper script for the FLM portable build -# Sets XILINX_XRT so XRT can find its bundled libraries at $XILINX_XRT/lib/x86_64-linux-gnu/ +# Wrapper for the FLM portable build. +# +# The portable tree is self-contained: the real binary (flm-real) and all of its +# bundled shared libraries (the NPU runtime + the NPU engine .so) live under +# ./lib next to this script. The engine libs carry an $ORIGIN-relative RUNPATH, +# so this wrapper mainly makes ./lib discoverable and then execs the real binary. +# +# This template is shared by both NPU runtime backends; the active one is +# substituted in by CMake (see FLM_RUNTIME_NAME): 0 = XRT, 1 = HRX. +FLM_RUNTIME="@FLM_RUNTIME_NAME@" -# Determine the directory where this script is located +# Directory containing this script (and the bundled tree). SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# XRT constructs library paths as $XILINX_XRT/lib/x86_64-linux-gnu/ -# Point XILINX_XRT to the portable root so XRT finds ./lib/x86_64-linux-gnu/ -export XILINX_XRT="$SCRIPT_DIR" - -# Add bundled libraries to LD_LIBRARY_PATH for runtime resolution +# Make the bundled libraries resolvable at runtime. export LD_LIBRARY_PATH="$SCRIPT_DIR/lib:$LD_LIBRARY_PATH" -# Ensure the multiarch directory structure exists with symlinks to bundled libs -MULTIARCH_DIR="$SCRIPT_DIR/lib/x86_64-linux-gnu" -if [ ! -d "$MULTIARCH_DIR" ]; then - mkdir -p "$MULTIARCH_DIR" - cd "$SCRIPT_DIR/lib" - for lib in libxrt*.so* libxrt++.so*; do - if [ -f "$lib" ] || [ -L "$lib" ]; then - # Handle symlinks by following them to get the actual target - if [ -L "$lib" ]; then - target=$(readlink "$lib") - ln -sf "../$target" "$MULTIARCH_DIR/$lib" 2>/dev/null - else - ln -sf "../$lib" "$MULTIARCH_DIR/$lib" 2>/dev/null +if [ "$FLM_RUNTIME" = "xrt" ]; then + # XRT constructs library paths as $XILINX_XRT/lib/x86_64-linux-gnu/. + # Point XILINX_XRT at the portable root so XRT finds ./lib/x86_64-linux-gnu/. + export XILINX_XRT="$SCRIPT_DIR" + + # Ensure the multiarch directory structure exists with symlinks to the + # bundled XRT libs, so XRT's internal path construction resolves. + MULTIARCH_DIR="$SCRIPT_DIR/lib/x86_64-linux-gnu" + if [ ! -d "$MULTIARCH_DIR" ]; then + mkdir -p "$MULTIARCH_DIR" + cd "$SCRIPT_DIR/lib" + for lib in libxrt*.so* libxrt++.so*; do + if [ -f "$lib" ] || [ -L "$lib" ]; then + if [ -L "$lib" ]; then + target=$(readlink "$lib") + ln -sf "../$target" "$MULTIARCH_DIR/$lib" 2>/dev/null + else + ln -sf "../$lib" "$MULTIARCH_DIR/$lib" 2>/dev/null + fi fi - fi - done - cd - > /dev/null + done + cd - > /dev/null + fi fi -# Execute the real binary +# Execute the real binary, forwarding all arguments. exec "$SCRIPT_DIR/flm-real" "$@" - diff --git a/src/include/AutoEmbeddingModel/all_embedding_model.hpp b/src/include/AutoEmbeddingModel/all_embedding_model.hpp index 066f36a2..4e0c5da9 100644 --- a/src/include/AutoEmbeddingModel/all_embedding_model.hpp +++ b/src/include/AutoEmbeddingModel/all_embedding_model.hpp @@ -18,7 +18,7 @@ inline std::string complete_simple_embedding_tag(std::string model_tag) { } -inline std::pair> get_auto_embedding_model(const std::string& model_tag, xrt::device* npu_device_inst) { +inline std::pair> get_auto_embedding_model(const std::string& model_tag, flm_rt::device* npu_device_inst) { #ifndef FASTFLOWLM_LINUX_LIMITED_MODELS static std::unordered_set gemma_embed_tags = { diff --git a/src/include/AutoEmbeddingModel/auto_embedding_model.hpp b/src/include/AutoEmbeddingModel/auto_embedding_model.hpp index 73e2c049..64b79fa4 100644 --- a/src/include/AutoEmbeddingModel/auto_embedding_model.hpp +++ b/src/include/AutoEmbeddingModel/auto_embedding_model.hpp @@ -53,7 +53,7 @@ class AutoEmbeddingModel { std::unique_ptr q4nx = nullptr; bool is_model_loaded = false; std::string current_model = ""; - xrt::device* npu_device_inst = nullptr; + flm_rt::device* npu_device_inst = nullptr; std::unique_ptr npu = nullptr; bool enable_preemption = false; @@ -69,7 +69,7 @@ class AutoEmbeddingModel { //************ Shared by all models *************/ virtual ~AutoEmbeddingModel() = default; - AutoEmbeddingModel(xrt::device* npu_device_inst, std::string current_model = ""); + AutoEmbeddingModel(flm_rt::device* npu_device_inst, std::string current_model = ""); /// \brief Get the current model /// \return the current model std::string get_current_model(); diff --git a/src/include/AutoEmbeddingModel/modeling_gemma_embedding.hpp b/src/include/AutoEmbeddingModel/modeling_gemma_embedding.hpp index 53369492..d0ef9d72 100644 --- a/src/include/AutoEmbeddingModel/modeling_gemma_embedding.hpp +++ b/src/include/AutoEmbeddingModel/modeling_gemma_embedding.hpp @@ -52,7 +52,7 @@ class Gemma_Embedding : public AutoEmbeddingModel{ } } public: - Gemma_Embedding(xrt::device* npu_device_inst); + Gemma_Embedding(flm_rt::device* npu_device_inst); ~Gemma_Embedding(); void load_model(std::string model_path, json model_info, bool enable_preemption = false) override; diff --git a/src/include/AutoModel/all_models.hpp b/src/include/AutoModel/all_models.hpp index 0cf20ef8..2eed7733 100644 --- a/src/include/AutoModel/all_models.hpp +++ b/src/include/AutoModel/all_models.hpp @@ -50,7 +50,7 @@ typedef enum { error_embedding } SupportedModelFamily; -inline std::pair> get_auto_model(const std::string& model_tag, model_list& available_models, xrt::device* npu_device_inst) { +inline std::pair> get_auto_model(const std::string& model_tag, model_list& available_models, flm_rt::device* npu_device_inst) { static const std::map modelFamilyMap = { diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index 0d7f877f..ed3a6726 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -142,7 +142,7 @@ class AutoModel { bool is_model_loaded = false; std::string current_model = ""; std::vector token_history; - xrt::device* npu_device_inst = nullptr; + flm_rt::device* npu_device_inst = nullptr; std::unique_ptr npu = nullptr; bool enable_preemption = false; std::vector checkpoint_his; @@ -205,7 +205,7 @@ class AutoModel { //************ Shared by all models *************/ virtual ~AutoModel() = default; - AutoModel(xrt::device* npu_device_inst, std::string current_model = ""); + AutoModel(flm_rt::device* npu_device_inst, std::string current_model = ""); void reset_parser() { buffer_.clear(); diff --git a/src/include/AutoModel/modeling_gemma3.hpp b/src/include/AutoModel/modeling_gemma3.hpp index b8f647b4..1f24c399 100644 --- a/src/include/AutoModel/modeling_gemma3.hpp +++ b/src/include/AutoModel/modeling_gemma3.hpp @@ -27,7 +27,7 @@ class Gemma3 : public AutoModel { buffer preprocess_image(bytes& image); public: - Gemma3(xrt::device* npu_device_inst); + Gemma3(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_gemma3_text.hpp b/src/include/AutoModel/modeling_gemma3_text.hpp index b2090c3f..d7e4e10c 100644 --- a/src/include/AutoModel/modeling_gemma3_text.hpp +++ b/src/include/AutoModel/modeling_gemma3_text.hpp @@ -29,7 +29,7 @@ class Gemma3_Text_Only : public AutoModel { void setup_tokenizer(std::string model_path); public: - Gemma3_Text_Only(xrt::device* npu_device_inst); + Gemma3_Text_Only(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_gemma4e.hpp b/src/include/AutoModel/modeling_gemma4e.hpp index 13952e88..25977edb 100644 --- a/src/include/AutoModel/modeling_gemma4e.hpp +++ b/src/include/AutoModel/modeling_gemma4e.hpp @@ -80,7 +80,7 @@ class Gemma4e : public AutoModel { public: - Gemma4e(xrt::device* npu_device_inst); + Gemma4e(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; diff --git a/src/include/AutoModel/modeling_gpt_oss.hpp b/src/include/AutoModel/modeling_gpt_oss.hpp index 95202497..f0451199 100644 --- a/src/include/AutoModel/modeling_gpt_oss.hpp +++ b/src/include/AutoModel/modeling_gpt_oss.hpp @@ -33,7 +33,7 @@ class GPT_OSS : public AutoModel { accumulated_text.clear(); } public: - GPT_OSS(xrt::device* npu_device_inst); + GPT_OSS(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_info, int default_context_length = -1, bool enable_preemption = false) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; diff --git a/src/include/AutoModel/modeling_lfm2.hpp b/src/include/AutoModel/modeling_lfm2.hpp index bf6c2699..2c0a947b 100644 --- a/src/include/AutoModel/modeling_lfm2.hpp +++ b/src/include/AutoModel/modeling_lfm2.hpp @@ -24,7 +24,7 @@ class LFM2 : public AutoModel { } public: - LFM2(xrt::device* npu_device_inst); + LFM2(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; @@ -53,7 +53,7 @@ class LFM2_5_TK : public AutoModel { } public: - LFM2_5_TK(xrt::device* npu_device_inst); + LFM2_5_TK(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_llama3.hpp b/src/include/AutoModel/modeling_llama3.hpp index 261e7d44..d8ce46e8 100644 --- a/src/include/AutoModel/modeling_llama3.hpp +++ b/src/include/AutoModel/modeling_llama3.hpp @@ -14,7 +14,7 @@ class Llama3 : public AutoModel { void setup_tokenizer(std::string model_path); public: - Llama3(xrt::device* npu_device_inst); + Llama3(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; @@ -36,7 +36,7 @@ class DeepSeek_r1_8b : public AutoModel { void setup_tokenizer(std::string model_path); public: - DeepSeek_r1_8b(xrt::device* npu_device_inst); + DeepSeek_r1_8b(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_nanbeige.hpp b/src/include/AutoModel/modeling_nanbeige.hpp index b224529a..69bad87c 100644 --- a/src/include/AutoModel/modeling_nanbeige.hpp +++ b/src/include/AutoModel/modeling_nanbeige.hpp @@ -16,7 +16,7 @@ class Nanbeige : public AutoModel { std::string nanbeige_filter(int token); public: - Nanbeige(xrt::device* npu_device_inst); + Nanbeige(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_phi4.hpp b/src/include/AutoModel/modeling_phi4.hpp index 528ebde9..66937d6d 100644 --- a/src/include/AutoModel/modeling_phi4.hpp +++ b/src/include/AutoModel/modeling_phi4.hpp @@ -13,7 +13,7 @@ class Phi4 : public AutoModel { void setup_tokenizer(std::string model_path); public: - Phi4(xrt::device* npu_device_inst); + Phi4(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_qwen2.hpp b/src/include/AutoModel/modeling_qwen2.hpp index afa1712b..467c6493 100644 --- a/src/include/AutoModel/modeling_qwen2.hpp +++ b/src/include/AutoModel/modeling_qwen2.hpp @@ -17,7 +17,7 @@ class Qwen2 : public AutoModel { void setup_tokenizer(std::string model_path); public: - Qwen2(xrt::device* npu_device_inst); + Qwen2(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_qwen2vl.hpp b/src/include/AutoModel/modeling_qwen2vl.hpp index 66d673dd..d59f9e49 100644 --- a/src/include/AutoModel/modeling_qwen2vl.hpp +++ b/src/include/AutoModel/modeling_qwen2vl.hpp @@ -43,7 +43,7 @@ class Qwen2VL : public AutoModel { void preprocess_image(qwen2vl_image_t& image, std::vector &pixel_values); public: - Qwen2VL(xrt::device* npu_device_inst); + Qwen2VL(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_qwen3.hpp b/src/include/AutoModel/modeling_qwen3.hpp index c30f171e..fafd1705 100644 --- a/src/include/AutoModel/modeling_qwen3.hpp +++ b/src/include/AutoModel/modeling_qwen3.hpp @@ -24,7 +24,7 @@ class Qwen3 : public AutoModel { void setup_tokenizer(std::string model_path); public: - Qwen3(xrt::device* npu_device_inst); + Qwen3(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; @@ -85,7 +85,7 @@ class Qwen3_IT : public AutoModel { void setup_tokenizer(std::string model_path); public: - Qwen3_IT(xrt::device* npu_device_inst); + Qwen3_IT(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; @@ -108,7 +108,7 @@ class Qwen3_TK : public AutoModel { void setup_tokenizer(std::string model_path); public: - Qwen3_TK(xrt::device* npu_device_inst); + Qwen3_TK(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; @@ -133,7 +133,7 @@ class DeepSeek_r1_0528_8b : public AutoModel { void setup_tokenizer(std::string model_path); public: - DeepSeek_r1_0528_8b(xrt::device* npu_device_inst); + DeepSeek_r1_0528_8b(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_qwen3_5_omni.hpp b/src/include/AutoModel/modeling_qwen3_5_omni.hpp index 38507ee4..b2549125 100644 --- a/src/include/AutoModel/modeling_qwen3_5_omni.hpp +++ b/src/include/AutoModel/modeling_qwen3_5_omni.hpp @@ -28,7 +28,7 @@ /************ Qwen3_5_Omni **************/ class Qwen3_5_Omni : public AutoModel { public: - explicit Qwen3_5_Omni(xrt::device* npu_device_inst); + explicit Qwen3_5_Omni(flm_rt::device* npu_device_inst); ~Qwen3_5_Omni() override = default; /// \brief Load config + weights and set up tokenizer / sampler. diff --git a/src/include/AutoModel/modeling_qwen3_5vl.hpp b/src/include/AutoModel/modeling_qwen3_5vl.hpp index 569ccc67..7a3fd4c2 100644 --- a/src/include/AutoModel/modeling_qwen3_5vl.hpp +++ b/src/include/AutoModel/modeling_qwen3_5vl.hpp @@ -48,7 +48,7 @@ class Qwen3_5VL : public AutoModel { void preprocess_image(qwen3_5vl_image_t& image, std::vector &pixel_values); public: - Qwen3_5VL(xrt::device* npu_device_inst); + Qwen3_5VL(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_qwen3_6_moe.hpp b/src/include/AutoModel/modeling_qwen3_6_moe.hpp index 0df99697..b7de7a7e 100644 --- a/src/include/AutoModel/modeling_qwen3_6_moe.hpp +++ b/src/include/AutoModel/modeling_qwen3_6_moe.hpp @@ -48,7 +48,7 @@ class Qwen3_6_MOE : public AutoModel { void preprocess_image(qwen3_6_moe_image_t& image, std::vector &pixel_values); public: - Qwen3_6_MOE(xrt::device* npu_device_inst); + Qwen3_6_MOE(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; diff --git a/src/include/AutoModel/modeling_qwen3vl.hpp b/src/include/AutoModel/modeling_qwen3vl.hpp index a3b37c97..7879aa49 100644 --- a/src/include/AutoModel/modeling_qwen3vl.hpp +++ b/src/include/AutoModel/modeling_qwen3vl.hpp @@ -44,7 +44,7 @@ class Qwen3VL : public AutoModel { void preprocess_image(qwen3vl_image_t& image, std::vector &pixel_values); public: - Qwen3VL(xrt::device* npu_device_inst); + Qwen3VL(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; //void toggle_enable_think() override; @@ -114,7 +114,7 @@ class Qwen3VL_Thinking : public Qwen3VL { int think_marker_id; public: - Qwen3VL_Thinking(xrt::device* npu_device_inst) : Qwen3VL(npu_device_inst) { + Qwen3VL_Thinking(flm_rt::device* npu_device_inst) : Qwen3VL(npu_device_inst) { } std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; diff --git a/src/include/buffer.hpp b/src/include/buffer.hpp index dde591fa..53c14c61 100644 --- a/src/include/buffer.hpp +++ b/src/include/buffer.hpp @@ -15,13 +15,12 @@ #include #include -#define __XRT__ +#define FLM_DEVICE_BUFFER -#ifdef __XRT__ -#include "xrt/xrt_bo.h" -#include "xrt/xrt_kernel.h" -#include "xrt/xrt_device.h" -#include "xrt/experimental/xrt_ext.h" +#ifdef FLM_DEVICE_BUFFER +// Pulls in the selected NPU runtime backend (XRT or HRX) behind the neutral +// `flm_rt` alias. All device buffer types below are referenced as flm_rt::bo. +#include "device_runtime.hpp" #endif #include "utils/debug_utils.hpp" @@ -35,10 +34,10 @@ class bytes { uint8_t* data_; size_t size_; bool is_owner_; -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER bool is_bo_owner_; - xrt::bo* bo_; - std::unique_ptr owned_bo_; + flm_rt::bo* bo_; + std::unique_ptr owned_bo_; #endif public: @@ -46,7 +45,7 @@ class bytes { /// \note This is a buffer wrapper that maps to a bo_buffer or other memory without performing a deep copy. /// \note A copy (or mapping) does not duplicate the underlying memory; it only maps the pointer. bytes() : data_(nullptr), size_(0), is_owner_(false) -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER , is_bo_owner_(false), bo_(nullptr), owned_bo_(nullptr) #endif {} @@ -54,7 +53,7 @@ class bytes { /// \brief copy constructor /// \param other the other bytes bytes(const bytes& other) : owned_data_(nullptr), data_(other.data_), size_(other.size_), is_owner_(false) -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER , is_bo_owner_(false), bo_(other.bo_), owned_bo_(nullptr) #endif {} @@ -63,14 +62,14 @@ class bytes { /// \param other the other bytes bytes(bytes&& other) noexcept : owned_data_(std::move(other.owned_data_)), data_(other.data_), size_(other.size_), is_owner_(other.is_owner_) -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER , is_bo_owner_(other.is_bo_owner_), bo_(other.bo_), owned_bo_(std::move(other.owned_bo_)) #endif { other.data_ = nullptr; other.size_ = 0; other.is_owner_ = false; -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER other.is_bo_owner_ = false; other.bo_ = nullptr; other.owned_bo_ = nullptr; @@ -81,7 +80,7 @@ class bytes { /// \param size the size bytes(size_t size) : size_(size), is_owner_(true) -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER , is_bo_owner_(false), bo_(nullptr), owned_bo_(nullptr) #endif { @@ -104,15 +103,15 @@ class bytes { /// \param size the size bytes(uint8_t* data, size_t size) : owned_data_(nullptr), data_(data), size_(size), is_owner_(false) -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER , is_bo_owner_(false), bo_(nullptr), owned_bo_(nullptr) #endif {} -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER /// \brief constructor /// \param bo the bo - bytes(xrt::bo& bo) + bytes(flm_rt::bo& bo) : owned_data_(nullptr), data_(bo.map()), size_(bo.size()), is_owner_(false), is_bo_owner_(false), bo_(&bo), owned_bo_(nullptr) {} @@ -122,27 +121,27 @@ class bytes { /// \param kernel the kernel /// \param group_id the group id /// \param flags the flags - bytes(xrt::device& device, size_t size) + bytes(flm_rt::device& device, size_t size) : owned_data_(nullptr), size_(size), is_owner_(false), is_bo_owner_(true) { if (size > 3ull * 1024 * 1024 * 1024 || size == 0){ throw std::runtime_error("Invalid size for bytes allocation"); } size_t alignment = 1024 * 1024; - int padded_size = (size + alignment - 1) / alignment * alignment; // 4KB alignment, , (xrt::ext::bo::access_mode)(xrt::ext::bo::access_mode::read_write | xrt::ext::bo::access_mode::process) + int padded_size = (size + alignment - 1) / alignment * alignment; // 1MB alignment try { - owned_bo_ = std::make_unique(device, padded_size); + owned_bo_ = std::make_unique(device, padded_size); } catch (const std::exception& e) { - throw std::runtime_error(std::string("Failed to allocate xrt::ext::bo: ") + e.what()); + throw std::runtime_error(std::string("Failed to allocate flm_rt::ext::bo: ") + e.what()); } // uint64_t bo_address = reinterpret_cast(owned_bo_->map()); // while ( ((bo_address & 0xF0000000) == 0x60000000) || // ((bo_address & 0xF0000000) == 0x70000000) ) { - // owned_bo_ = std::make_unique(device, padded_size); + // owned_bo_ = std::make_unique(device, padded_size); // //header_print("info", "Re-allocating proj_weights for layer " + std::to_string(i) + " to avoid address in 0x60000000 - 0x7FFFFFFF, new address: " + std::to_string(reinterpret_cast(proj_weights[i].data()))); // bo_address = reinterpret_cast(owned_bo_->map()); // } @@ -158,7 +157,7 @@ class bytes { owned_data_.reset(); } data_ = nullptr; -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER if (is_bo_owner_) { owned_bo_.reset(); } @@ -176,7 +175,7 @@ class bytes { data_ = other.data_; size_ = other.size_; is_owner_ = false; -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER if (is_bo_owner_){ owned_bo_.reset(); } @@ -198,7 +197,7 @@ class bytes { data_ = other.data_; size_ = other.size_; is_owner_ = other.is_owner_; -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER if (is_bo_owner_){ owned_bo_.reset(); } @@ -248,7 +247,7 @@ class bytes { /// \brief resize /// \param new_size the new size void resize(size_t new_size) { -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER assert(!is_bo_owner_); #endif if (data_ != nullptr && !is_owner_) { @@ -270,7 +269,7 @@ class bytes { /// \brief free, release the memory or the bo void free() { -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER assert(!is_bo_owner_); #endif if (is_owner_){ @@ -279,7 +278,7 @@ class bytes { data_ = nullptr; size_ = 0; is_owner_ = false; -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER if (is_bo_owner_){ owned_bo_.reset(); } @@ -298,20 +297,28 @@ class bytes { /// \brief is owner /// \return the is owner bool is_owner() const { return is_owner_; } -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER /// \brief is bo owner /// \return the is bo owner bool is_bo_owner() const { return is_bo_owner_; } - /// \brief sync to device + /// \brief sync to device (host writes -> device) +#if defined(FLM_USE_HRX) + void sync_to_device() { assert(bo_); bo_->flush(); } +#else void sync_to_device() { assert(bo_); bo_->sync(XCL_BO_SYNC_BO_TO_DEVICE); } +#endif - /// \brief sync from device + /// \brief sync from device (device writes -> host) +#if defined(FLM_USE_HRX) + void sync_from_device() { assert(bo_); bo_->invalidate(); } +#else void sync_from_device() { assert(bo_); bo_->sync(XCL_BO_SYNC_BO_FROM_DEVICE); } +#endif /// \brief bo /// \return the bo - xrt::bo& bo() { assert(bo_); return *bo_; } + flm_rt::bo& bo() { assert(bo_); return *bo_; } #endif /// \brief from file @@ -361,10 +368,10 @@ class buffer : public bytes { /// \note Transfers ownership (owned_data_/owned_bo_) so a returned buffer does not dangle. buffer(buffer&& other) noexcept : bytes(std::move(other)) {} -#ifdef __XRT__ +#ifdef FLM_DEVICE_BUFFER /// \brief constructor /// \param bo the bo - buffer(xrt::bo& bo) : bytes(bo) {} + buffer(flm_rt::bo& bo) : bytes(bo) {} /// \brief constructor /// \param count the count @@ -372,7 +379,7 @@ class buffer : public bytes { /// \param kernel the kernel /// \param group_id the group id /// \param flags the flags - buffer(xrt::device& device, size_t count) + buffer(flm_rt::device& device, size_t count) : bytes(device, count * sizeof(T)) {} #endif diff --git a/src/include/device_runtime.hpp b/src/include/device_runtime.hpp new file mode 100644 index 00000000..403691bc --- /dev/null +++ b/src/include/device_runtime.hpp @@ -0,0 +1,24 @@ +/// \file device_runtime.hpp +/// \brief Selects the NPU runtime backend (XRT or HRX) at build time and exposes +/// it to the rest of the codebase under a single neutral alias namespace +/// `flm_rt`, so device-facing code stays backend-agnostic. +/// \note The backend is chosen via the FLM_USE_HRX build flag wired in +/// src/CMakeLists.txt (0 = XRT default, 1 = HRX): +/// - FLM_USE_HRX defined -> HRX amdxdna runtime (namespace hrx) +/// - FLM_USE_HRX undefined -> Xilinx Run Time (XRT) (namespace xrt) +/// Both backends expose the same device-facing surface used across FLM +/// (device, bo, ext::bo, kernel, hw_context, run, runlist, xclbin, +/// info::device), so referencing them through `flm_rt::` keeps a single +/// source tree building against either runtime. +#pragma once + +#if defined(FLM_USE_HRX) +#include "hrx_cpp/hrx_cpp.hpp" +namespace flm_rt = hrx; +#else +#include "xrt/xrt_bo.h" +#include "xrt/xrt_kernel.h" +#include "xrt/xrt_device.h" +#include "xrt/experimental/xrt_ext.h" +namespace flm_rt = xrt; +#endif diff --git a/src/include/hrx_cpp/hrx_cpp.hpp b/src/include/hrx_cpp/hrx_cpp.hpp new file mode 100644 index 00000000..d34bf6bf --- /dev/null +++ b/src/include/hrx_cpp/hrx_cpp.hpp @@ -0,0 +1,403 @@ +/// \file hrx_cpp.hpp +/// \brief Minimal C++ `namespace hrx` providing the device/buffer/kernel/run +/// API that FastFlowLM uses, implemented directly on top of libhrx +/// (hrx_runtime.h). +/// +/// NPU control code goes straight from npu_sequence::dump() into an HRX XADX +/// "direct executable"; there is no separate assembler step. +/// +/// Coherence model: buffers are device-visible, host-coherent, mapped once +/// (persistent). FastFlowLM already brackets device work with explicit +/// sync_to_device()/sync_from_device() calls, so we map those directly to +/// hrx_buffer_flush_range()/hrx_buffer_invalidate_range(). Dispatch is +/// hrx_stream_dispatch() + hrx_stream_synchronize(). +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hrx_amdxdna.h" +#include "hrx_runtime.h" + +// ---- ert_cmd_state: command states that FLM's npu_utils returns/maps. +#ifndef FLM_ERT_CMD_STATE_DEFINED +#define FLM_ERT_CMD_STATE_DEFINED +enum ert_cmd_state { + ERT_CMD_STATE_NEW = 1, + ERT_CMD_STATE_QUEUED = 2, + ERT_CMD_STATE_RUNNING = 3, + ERT_CMD_STATE_COMPLETED = 4, + ERT_CMD_STATE_ERROR = 5, + ERT_CMD_STATE_ABORT = 6, + ERT_CMD_STATE_SUBMITTED = 7, + ERT_CMD_STATE_TIMEOUT = 8, + ERT_CMD_STATE_NORESPONSE = 9, + ERT_CMD_STATE_SKERROR = 10, + ERT_CMD_STATE_SKCRASHED = 11, + ERT_CMD_STATE_MAX = 12, +}; +#endif + +namespace hrx { + +// --------------------------------------------------------------------------- +// Process-wide HRX runtime (one device + one stream), lazily initialized. +// --------------------------------------------------------------------------- +class Runtime { +public: + static Runtime& get() { + static Runtime r; + return r; + } + hrx_device_t dev = nullptr; + hrx_stream_t stream = nullptr; + bool ok = false; + + void ensure() { + if (dev) return; + hrx_status_t init_status = hrx_gpu_initialize(0); + const bool initialized = + hrx_status_is_ok(init_status) || + hrx_status_code(init_status) == HRX_STATUS_ALREADY_EXISTS; + hrx_status_ignore(init_status); + if (initialized && + hrx_status_is_ok(hrx_gpu_device_get(0, &dev)) && + hrx_status_is_ok(hrx_stream_create(dev, 0, &stream))) { + ok = true; + if (std::getenv("HRX_DEBUG")) + std::fprintf(stderr, "[hrx] device+stream initialized (Runtime@%p)\n", + (void*)this); + } else { + std::fprintf(stderr, "[hrx] device init FAILED\n"); + ok = false; + } + } + +private: + Runtime() = default; +}; + +inline Runtime& rt() { + Runtime& r = Runtime::get(); + r.ensure(); + return r; +} + +// Report (do not swallow) an HRX error. Returns true if status was an error. +// FLM dispatch silently ignored synchronize/dispatch failures, which turns a +// failed ERT_CMD_CHAIN (e.g. a missing host patch table) into silent no-op +// dispatches -> garbage output at full speed. Always surface these. +inline bool hrx_report(hrx_status_t s, const char* where) { + if (hrx_status_is_ok(s)) return false; + char* m = nullptr; + size_t mn = 0; + hrx_status_to_string(s, &m, &mn); + std::fprintf(stderr, "[hrx][ERROR] %s: %s\n", where, m ? m : "?"); + hrx_status_free_message(m); + hrx_status_ignore(s); + return true; +} + +// --------------------------------------------------------------------------- +// Executable cache: build one HRX XADX executable per distinct executable +// identity (xclbin + control program + host patch table) and resolve its export +// ordinal once. +// --------------------------------------------------------------------------- +struct CachedExe { + hrx_executable_t exe = nullptr; + uint32_t ord = 0; +}; + +inline void append_key_bytes(std::string& key, const void* data, + size_t byte_count) { + const uint64_t length = static_cast(byte_count); + key.append(reinterpret_cast(&length), sizeof(length)); + if (data && byte_count) { + key.append(reinterpret_cast(data), byte_count); + } +} + +inline hrx_executable_t build_or_get_executable( + const std::vector& xclbin_bytes, const uint32_t* cc, size_t n, + uint32_t* ord_out) { + static std::mutex mu; + static std::unordered_map cache; + std::string key; + key.reserve(xclbin_bytes.size() + n * sizeof(uint32_t) + + 2 * sizeof(uint64_t)); + append_key_bytes(key, xclbin_bytes.data(), xclbin_bytes.size()); + append_key_bytes(key, cc, n * sizeof(uint32_t)); + std::lock_guard lk(mu); + auto it = cache.find(key); + if (it != cache.end()) { + if (ord_out) *ord_out = it->second.ord; + return it->second.exe; + } + hrx_const_byte_span_t xclbin = {xclbin_bytes.data(), xclbin_bytes.size()}; + hrx_amdxdna_executable_run_t run = {}; + run.record_length = sizeof(run); + run.abi_version = HRX_AMDXDNA_EXECUTABLE_RUN_ABI_VERSION_0; + run.transaction = {reinterpret_cast(cc), + n * sizeof(uint32_t)}; + hrx_amdxdna_executable_entry_point_t entry_point = {}; + entry_point.record_length = sizeof(entry_point); + entry_point.abi_version = HRX_AMDXDNA_EXECUTABLE_ENTRY_POINT_ABI_VERSION_0; + entry_point.name = {"MLIR_AIE", std::strlen("MLIR_AIE")}; + entry_point.context_mode = HRX_AMDXDNA_CONTEXT_MODE_CREATE; + entry_point.runs = &run; + entry_point.run_count = 1; + hrx_amdxdna_executable_create_params_t params = {}; + params.record_length = sizeof(params); + params.abi_version = HRX_AMDXDNA_EXECUTABLE_CREATE_PARAMS_ABI_VERSION_0; + params.xclbins = &xclbin; + params.xclbin_count = 1; + params.entry_points = &entry_point; + params.entry_point_count = 1; + hrx_executable_t exe = nullptr; + uint32_t ord = 0; + hrx_status_t create_status = hrx_amdxdna_executable_create( + rt().dev, ¶ms, &exe); + if (hrx_report(create_status, "hrx_amdxdna_executable_create")) { + exe = nullptr; + } + if (exe && hrx_report(hrx_executable_lookup_export_by_name( + exe, "MLIR_AIE", &ord), + "hrx_executable_lookup_export_by_name")) { + hrx_executable_release(exe); + exe = nullptr; + } + cache.emplace(std::move(key), CachedExe{exe, ord}); + if (ord_out) *ord_out = ord; + return exe; +} + +// --------------------------------------------------------------------------- +// uuid / xclbin / device / hw_context +// --------------------------------------------------------------------------- +class uuid { +public: + unsigned char m_uuid[16] = {0}; +}; + +class xclbin { +public: + std::shared_ptr> bytes_ = + std::make_shared>(); + + xclbin() = default; + explicit xclbin(const std::string& path) { + std::FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) throw std::runtime_error("hrx::xclbin: cannot open " + path); + std::fseek(f, 0, SEEK_END); + long n = std::ftell(f); + std::fseek(f, 0, SEEK_SET); + if (n > 0) { + bytes_->resize(static_cast(n)); + size_t rd = std::fread(bytes_->data(), 1, bytes_->size(), f); + (void)rd; + } + std::fclose(f); + } + + // FLM searches kernels for one whose name starts with "MLIR_AIE"; the HRX + // dispatch path always uses the "MLIR_AIE" export, so a single placeholder + // kernel is sufficient (matches the proven interposer behavior). + class kernel { + public: + std::string name = "MLIR_AIE"; + std::string get_name() const { return name; } + }; + std::vector get_kernels() const { return {kernel{}}; } + uuid get_uuid() const { return uuid{}; } + const std::vector& bytes() const { return *bytes_; } + std::shared_ptr> bytes_shared() const { return bytes_; } +}; + +namespace info { +// Argument to device::get_info<>(). FLM only queries the human-readable +// device name for diagnostics. +enum class device { name, architecture }; +} + +class device { +public: + device() = default; + explicit device(unsigned int /*index*/) { rt(); } + uuid register_xclbin(const xclbin& /*xc*/) { return uuid{}; } + void reset() {} + + // Returns a human-readable device identity string for diagnostics: the + // IREE HAL device name (e.g. "amdxdna") via hrx_device_get_property(). + template + std::string get_info() const { + char buf[128] = {0}; + hrx_device_get_property( + rt().dev, + P == info::device::architecture ? HRX_DEVICE_PROPERTY_ARCHITECTURE + : HRX_DEVICE_PROPERTY_NAME, + buf, sizeof(buf)); + return std::string(buf); + } +}; + +class hw_context { +public: + std::shared_ptr> xclbin_bytes_; + + hw_context() = default; + hw_context(const device& /*dev*/, const xclbin& xc) + : xclbin_bytes_(xc.bytes_shared()) {} + // Legacy (device, uuid) form kept for source compatibility; carries no + // xclbin bytes, so prefer the (device, xclbin) form. + hw_context(const device& /*dev*/, const uuid& /*id*/) {} + + const std::vector& xclbin_bytes() const { + static const std::vector empty; + return xclbin_bytes_ ? *xclbin_bytes_ : empty; + } +}; + +// --------------------------------------------------------------------------- +// Buffers +// --------------------------------------------------------------------------- +class bo { +public: + hrx_buffer_t hbuf_ = nullptr; + void* mapped_ = nullptr; + size_t size_ = 0; + bool owns_ = false; + + bo() = default; + virtual ~bo() { + if (owns_ && hbuf_) hrx_buffer_release(hbuf_); + hbuf_ = nullptr; + mapped_ = nullptr; + } + bo(const bo&) = delete; + bo& operator=(const bo&) = delete; + + template + T map() { + return reinterpret_cast(mapped_); + } + size_t size() const { return size_; } + hrx_buffer_t handle() const { return hbuf_; } + + void flush() { // host writes -> device (sync_to_device) + if (hbuf_) hrx_buffer_flush_range(hbuf_, 0, size_); + } + void invalidate() { // device writes -> host (sync_from_device) + if (hbuf_) hrx_buffer_invalidate_range(hbuf_, 0, size_); + } +}; + +namespace ext { +class bo : public hrx::bo { +public: + bo(const device& /*dev*/, size_t sz) { + Runtime& r = rt(); + size_ = sz; + owns_ = true; + if (!r.ok) throw std::runtime_error("hrx::ext::bo: HRX device unavailable"); + // Device-visible, host-coherent, persistent mapping (one mmap kept for + // the buffer's lifetime). Coherence maintained via flush/invalidate. + hrx_status_t s = hrx_buffer_allocate( + r.stream, sz, + HRX_MEMORY_TYPE_HOST_LOCAL | HRX_MEMORY_TYPE_DEVICE_VISIBLE, + HRX_BUFFER_USAGE_DEFAULT | HRX_BUFFER_USAGE_MAPPING_PERSISTENT, + &hbuf_); + if (!hrx_status_is_ok(s) || !hbuf_) { + hrx_status_ignore(s); + throw std::runtime_error("hrx::ext::bo: hrx_buffer_allocate failed"); + } + void* p = nullptr; + s = hrx_buffer_map_with_mode(hbuf_, HRX_MAPPING_MODE_PERSISTENT, + HRX_MAP_READ | HRX_MAP_WRITE, 0, sz, &p); + if (!hrx_status_is_ok(s) || !p) { + hrx_status_ignore(s); + hrx_buffer_release(hbuf_); + hbuf_ = nullptr; + throw std::runtime_error("hrx::ext::bo: map_persistent failed"); + } + mapped_ = p; + std::memset(p, 0, sz); + } +}; +} // namespace ext + +// --------------------------------------------------------------------------- +// run / runlist +// --------------------------------------------------------------------------- +class run { +public: + hrx_executable_t exe_ = nullptr; + uint32_t ord_ = 0; + std::vector binds_; + + run() = default; + explicit run(hrx_executable_t exe, uint32_t ord) : exe_(exe), ord_(ord) {} + + void add_binding(hrx_buffer_t b, size_t size) { + binds_.push_back({b, 0, size}); + } + + // Record the dispatch on the stream (no synchronize); wait() flushes. + void start() { + if (!exe_) { + std::fprintf(stderr, "[hrx][ERROR] run::start with null executable\n"); + return; + } + if (binds_.empty()) { + std::fprintf(stderr, "[hrx][ERROR] run::start with no bindings\n"); + return; + } + hrx_dispatch_config_t cfg = {{1, 1, 1}, {1, 1, 1}, 0}; + hrx_status_t s = hrx_stream_dispatch(rt().stream, exe_, ord_, &cfg, + nullptr, 0, binds_.data(), + binds_.size(), HRX_DISPATCH_FLAG_NONE); + hrx_report(s, "run::start hrx_stream_dispatch"); + } + + ert_cmd_state wait() { + hrx_status_t s = hrx_stream_synchronize(rt().stream); + return hrx_report(s, "run::wait hrx_stream_synchronize") + ? ERT_CMD_STATE_ERROR + : ERT_CMD_STATE_COMPLETED; + } +}; + +class runlist { +public: + std::vector runs_; + + runlist() = default; + explicit runlist(const hw_context& /*ctx*/) {} + + void add(const run& r) { runs_.push_back(r); } + void add(run&& r) { runs_.push_back(std::move(r)); } + void reset() { runs_.clear(); } + + // Record every dispatch (no per-run synchronize) so HRX submits them as a + // batch; wait() runs one synchronize for the whole list. + void execute() { + for (auto& r : runs_) r.start(); + } + ert_cmd_state wait() { + hrx_status_t s = hrx_stream_synchronize(rt().stream); + return hrx_report(s, "runlist::wait hrx_stream_synchronize") + ? ERT_CMD_STATE_ERROR + : ERT_CMD_STATE_COMPLETED; + } +}; + +} // namespace hrx diff --git a/src/include/npu_utils/instr_utils/npu_cmd.hpp b/src/include/npu_utils/instr_utils/npu_cmd.hpp index 7826087b..66a8f9d2 100644 --- a/src/include/npu_utils/instr_utils/npu_cmd.hpp +++ b/src/include/npu_utils/instr_utils/npu_cmd.hpp @@ -16,7 +16,6 @@ #include #include "buffer.hpp" #include "utils/debug_utils.hpp" -#include "xrt/xrt_bo.h" const int INSTR_PRINT_WIDTH = 80; diff --git a/src/include/npu_utils/npu_instr_utils.hpp b/src/include/npu_utils/npu_instr_utils.hpp index ee971da6..7c5a9111 100644 --- a/src/include/npu_utils/npu_instr_utils.hpp +++ b/src/include/npu_utils/npu_instr_utils.hpp @@ -14,9 +14,9 @@ #include #include #include +#include #include "buffer.hpp" #include "utils/debug_utils.hpp" -#include "xrt/xrt_bo.h" #include "instr_utils/npu_cmd.hpp" #include "instr_utils/npu_cmd_write.hpp" #include "instr_utils/npu_cmd_ddr.hpp" @@ -478,22 +478,22 @@ class npu_sequence{ constexpr int wrap_bits = 10; if(size[3] > (1<< wrap_bits)){ - header_print_r("ERROR", ": step_3 out ouf range"); + header_print("ERROR", ": step_3 out ouf range"); } if(stride[3] > (1<< step_bits)){ - header_print_r("ERROR", ": stride_3 out of range"); + header_print("ERROR", ": stride_3 out of range"); } if(size[2] > (1<< wrap_bits)){ - header_print_r("ERROR", ": step_2 out ouf range"); + header_print("ERROR", ": step_2 out ouf range"); } if(stride[2] > (1<< step_bits)){ - header_print_r("ERROR", ": stride_2 out of range"); + header_print("ERROR", ": stride_2 out of range"); } if(size[1] > (1<< wrap_bits)){ - header_print_r("ERROR", ": step_1 out ouf range"); + header_print("ERROR", ": step_1 out ouf range"); } if(stride[1] > (1<< step_bits)){ - header_print_r("ERROR", ": stride_1 out of range"); + header_print("ERROR", ": stride_1 out of range"); } // inverse of human's order cmd->dim0_size = size[3]; @@ -659,6 +659,47 @@ class npu_sequence{ return std::make_pair(this->npu_seq.data(), this->npu_seq.size()); } + ///@brief Build the host patch table for the current control code (TXN). + ///@note Replaces the relocations aiebu used to emit. For every DDR + /// address-patch op we locate the matching shim-DMA BLOCKWRITE BD + /// (same col/row/bd_id) and emit a triple + /// (byte_offset_of_BD_length_word, arg_idx, arg_offset). + /// The amdxdna host-patch path reads addr-low at offset+4 and + /// addr-high at offset+8 of that BD, adding the bound buffer's + /// device address. Word offsets are into dump() (TXN incl. header). + ///@return flat vector of (offset, arg_idx, arg_offset) triples. + std::vector dump_patch_table(){ + if (this->is_valid == false){ + this->cmds2seq(); + } + std::vector patch; + // (col,row,bd_id) -> word index of the most recent BLOCKWRITE BD. + std::unordered_map bd_to_word; + size_t word = 4; // 4-word TXN header precedes the ops + for (auto& c : this->cmds){ + if (auto* d = dynamic_cast(c.get())){ + uint32_t key = (d->col << 16) | (d->row << 8) | d->bd_id; + auto it = bd_to_word.find(key); + if (it != bd_to_word.end()){ + size_t bw = it->second; // BLOCKWRITE start word + // word4 of the BLOCKWRITE = BD length register (0x1D000); + // +4 -> addr low (0x1D004, =0), +8 -> addr high (0x1D008). + patch.push_back((uint32_t)((bw + 4) * sizeof(uint32_t))); + patch.push_back(d->arg_idx); + patch.push_back(d->arg_offset); + } + word += c->get_op_lines(); + continue; + } + if (auto* b = dynamic_cast(c.get())){ + uint32_t key = (b->col << 16) | (b->row << 8) | b->bd_id; + bd_to_word[key] = word; + } + word += c->get_op_lines(); + } + return patch; + } + inline bool sequence_valid(){return this->is_valid;} inline uint8_t sequence_version(){return this->instr_version;} npu_device device_gen; diff --git a/src/include/npu_utils/npu_utils.hpp b/src/include/npu_utils/npu_utils.hpp index 0aa6ed5f..a546b2bd 100644 --- a/src/include/npu_utils/npu_utils.hpp +++ b/src/include/npu_utils/npu_utils.hpp @@ -1,664 +1,17 @@ /// \file npu_utils.hpp -/// \brief npu_utils class -/// \author FastFlowLM Team, Alfred -/// \date 2025-09-09 -/// \note This file contains the classes for managing the npu device +/// \brief Backend dispatcher for the NPU device/app management classes. +/// \note The NPU dispatch layer genuinely differs between the two runtime +/// backends (XRT builds an ELF via aiebu + xrt::module/elf/ext::kernel; +/// HRX builds an XADX executable directly via hrx_amdxdna). Rather than +/// interleave the two implementations, each lives in its own variant +/// header and this file includes exactly one, chosen by the FLM_USE_HRX +/// build flag (0 = XRT default, 1 = HRX). This keeps each backend's code +/// clean and lets the XRT variant stay a verbatim copy of the upstream +/// implementation. #pragma once -/* - The NPU is managed in this way: - For each program, there should only be one npu_xclbin_manager, example: - npu_xclbin_manager npu_mgr(device_npu2, 0); - - Then, multiple xclbins could be used by this npu and each xclbin may have multiple runtime sequences (applications) - Therefore, each registered xclbin will have a npu_app_manager, example: - - npu_app_manager* mvm_i8_xclbin = npu_mgr.register_xclbin("mvm_i8.xclbin"); - - Notice that the return value is a pointer to the npu_app_manager. This is to avoid duplicated npu hardware context, - all xclbins are managed by the npu_xclbin_manager centrally. - - Finally, an application can be create from the npu_app_manager, example: - - npu_app app = mvm_i8_xclbin->create_app(); - - In this version, the app do not have a name. -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef __WINDOWS__ -#include -#include -#include -#include -#include -#include "amdxdna_accel.h" -#endif -#include "xrt/xrt_bo.h" -#include "xrt/xrt_device.h" -#include "xrt/xrt_kernel.h" -#include "buffer.hpp" -#include "utils/debug_utils.hpp" -#include - -#include "xrt/experimental/xrt_kernel.h" -#include "xrt/experimental/xrt_ext.h" -#include "xrt/experimental/xrt_module.h" -#include "xrt/experimental/xrt_elf.h" - -#include "aiebu/aiebu.h" - -#include "npu_instr_utils.hpp" - - -class npu_app_manager; -class npu_xclbin_manager; - - -///@brief npu_app, a application that both xclbin and instruction are specified -///@param device the pointer to the device -///@param context the pointer to the context -///@param kernel_name the name of the kernel -///@see xrt::kernel, xrt::device -class npu_app { -private: - // from external - xrt::hw_context* context; - xrt::device* device; - std::string kernel_name; - npu_device device_gen; - bool enable_preemption; - - // self-managed - bool module_valid; - uint8_t module_version; - std::unique_ptr module; - std::unique_ptr elf; - std::unique_ptr kernel; - std::unique_ptr ctrl_seq; - - uint32_t _gen_elf(char** elf_buf, std::pair& instruction_data){ - uint32_t elf_buf_size = aiebu_assembler_get_elf( - aiebu_assembler_buffer_type_blob_instr_transaction, - (char*) instruction_data.first, instruction_data.second * sizeof(uint32_t), - NULL, 0, (void**)elf_buf, NULL, 0, "", "", NULL, 0); - assert(elf_buf_size > 0); - if (elf_buf_size == 0){ - header_print_r("ERROR", "Failed to get elf from ctrl_seq"); - exit(1); - } - return elf_buf_size; - } - - ///@brief Setup the kernel - ///@note The function will create an elf file from the ctrl_seq - ///@note The function will also update the module, elf, and kernel - void _setup_kernel(){ - char* elf_buf; - this->kernel.reset(); - this->module.reset(); - this->elf.reset(); - std::pair data = this->ctrl_seq->dump(); - assert(data.first != nullptr); - assert(data.second > 0); - uint32_t elf_buf_size = this->_gen_elf(&elf_buf, data); - if (this->module_valid){ - this->module.reset(); - this->elf.reset(); - this->kernel.reset(); - } - this->elf = std::make_unique(elf_buf, elf_buf_size); - this->module = std::make_unique(*this->elf); - this->kernel = std::make_unique(*this->context, *this->module, this->kernel_name); - this->module_valid = true; - this->module_version = this->ctrl_seq->sequence_version(); - - free((void*)elf_buf); - } - -public: - // enum ert_cmd_state { - // ERT_CMD_STATE_NEW = 1, - // ERT_CMD_STATE_QUEUED = 2, - // ERT_CMD_STATE_RUNNING = 3, - // ERT_CMD_STATE_COMPLETED = 4, - // ERT_CMD_STATE_ERROR = 5, - // ERT_CMD_STATE_ABORT = 6, - // ERT_CMD_STATE_SUBMITTED = 7, - // ERT_CMD_STATE_TIMEOUT = 8, - // ERT_CMD_STATE_NORESPONSE = 9, - // ERT_CMD_STATE_SKERROR = 10, //Check for error return code from Soft Kernel - // ERT_CMD_STATE_SKCRASHED = 11, //Soft kernel has crashed - // ERT_CMD_STATE_MAX, // Always the last one - // }; - std::map cmd_state_map = { - {ERT_CMD_STATE_NEW, "new"}, - {ERT_CMD_STATE_QUEUED, "queued"}, - {ERT_CMD_STATE_RUNNING, "running"}, - {ERT_CMD_STATE_COMPLETED, "completed"}, - {ERT_CMD_STATE_ERROR, "error"}, - {ERT_CMD_STATE_ABORT, "abort"}, - {ERT_CMD_STATE_SUBMITTED, "submitted"}, - {ERT_CMD_STATE_TIMEOUT, "timeout"}, - {ERT_CMD_STATE_NORESPONSE, "noresponse"}, - {ERT_CMD_STATE_SKERROR, "skerror"}, - {ERT_CMD_STATE_SKCRASHED, "skcrashed"}, - {ERT_CMD_STATE_MAX, "max"}, - }; - - ///@brief Default Constructor - ///@note Initialize the npu_app to nullptr - npu_app() { - this->device_gen = device_npu2; - this->kernel = nullptr; - this->device = nullptr; - this->module_valid = false; - this->module = nullptr; - this->elf = nullptr; - this->kernel = nullptr; - this->module_version = 0xFF; - } - - ///@brief Constructor, this shall not invoke by user, it shall only be invoked by npu_app_manager - ///@param device_gen the npu device - ///@param device the pointer to the device - ///@param context the pointer to the context - ///@param kernel_name the name of the kernel - ///@see xrt::device, xrt::hw_context, xrt::kernel - npu_app(npu_device device_gen, xrt::device* device, xrt::hw_context* context, std::string kernel_name, bool enable_preemption = false): - device_gen(device_gen), device(device), context(context), kernel_name(kernel_name), enable_preemption(enable_preemption){ - this->module_valid = false; - this->module = nullptr; - this->elf = nullptr; - this->kernel = nullptr; - this->ctrl_seq = std::make_unique(device_gen, enable_preemption); - this->module_version = 0xFF; - } - - - void update_ctrl_seq(){ - assert(this->ctrl_seq != nullptr); - this->_setup_kernel(); - } - - void load_elf(std::string elf_name){ - this->ctrl_seq->clear_cmds(); - this->ctrl_seq->cmds2seq(); // just for making sure the sequence is valid - this->elf = std::make_unique(elf_name); - this->module = std::make_unique(*this->elf); - this->kernel = std::make_unique(*this->context, *this->module, this->kernel_name); - this->module_valid = true; - this->module_version = this->ctrl_seq->sequence_version(); // force sync the sequence version - } - - void store_elf(std::string elf_name){ - char* elf_buf; - if (this->module_valid == false) { - this->_setup_kernel(); - } - std::pair data = this->ctrl_seq->dump(); - uint32_t elf_buf_size = this->_gen_elf(&elf_buf, data); - std::ofstream fout(elf_name, std::ios::binary); - if (fout.is_open() == false) { - header_print_r("ERROR", "Failed to open file: " << elf_name); - exit(1); - } - fout.write((char*)elf_buf, elf_buf_size); - fout.close(); - free((void*)elf_buf); - } - - ///@brief Operator() for running the kernel - ///@param args arguments, shall be the buffers with real bo - ///@see xrt::run - template - ert_cmd_state operator()(BoArgs&&... args){ - if (this->module_valid == false || this->ctrl_seq->sequence_valid() == false || this->module_version != this->ctrl_seq->sequence_version()) { - this->_setup_kernel(); - } - auto run = this->kernel->operator()(3, 0, 0, args.bo()...); - ert_cmd_state state = run.wait(); - LOG_VERBOSE(2, "ending state: " << this->cmd_state_map[state]); - return state; - } - - - ///@brief Operator() for running the kernel - ///@param args arguments, shall be the buffers with real bo - ///@see xrt::run - template - ert_cmd_state safe_run(BoArgs&&... args){ - if (this->module_valid == false || this->ctrl_seq->sequence_valid() == false || this->module_version != this->ctrl_seq->sequence_version()) { - this->_setup_kernel(); - } - std::array bo_args = { &args... }; - for (size_t i = 0; i < sizeof...(args); i++){ - bo_args[i]->sync_to_device(); - } - auto run = this->kernel->operator()(3, 0, 0, args.bo()...); - ert_cmd_state state = run.wait(); - for (size_t i = 0; i < sizeof...(args); i++){ - bo_args[i]->sync_from_device(); - } - LOG_VERBOSE(2, "ending state: " << this->cmd_state_map[state]); - return state; - } - - ///@brief Create a run object - ///@param args arguments, shall be the buffers with real bo - ///@return a run object for waiting - ///@see xrt::run - template - xrt::run create_run(BoArgs&&... args){ - if (this->module_valid == false || this->ctrl_seq->sequence_valid() == false || this->module_version != this->ctrl_seq->sequence_version()) { - this->_setup_kernel(); - } - xrt::run run = xrt::run(*this->kernel); - run.set_arg(0, 3); - run.set_arg(1, 0); - run.set_arg(2, 0); - std::array bo_args = { &args... }; - for (size_t i = 0; i < sizeof...(args); i++){ - run.set_arg(3 + i, bo_args[i]->bo()); - } - return run; - } - - ///@brief Create a buffer with real bo - ///@param size size of the buffer - ///@see buffer - template - buffer create_bo_buffer(size_t size){ - assert(size > 0); - LOG_VERBOSE(2, "Creating buffer buffer with size: " << size); - return buffer(*this->device, size); - } - - npu_sequence* seq() { - return this->ctrl_seq.get(); - } -}; - -class npu_app_manager { -private: - npu_device device_gen; - bool enable_preemption; - xrt::device* device; - std::unique_ptr context; - std::string kernel_name; - std::string xclbin_name; - bool xclbin_valid; -public: - ///@brief Default Constructor - ///@param device_gen the npu device - ///@param device device object - ///@param xclbin_name name of the xclbin file - ///@see xrt::device, xrt::xclbin - ///@note The function will initialize the npu_app_manager to nullptr - npu_app_manager(){ - this->device_gen = device_npu2; - this->device = nullptr; - this->context = nullptr; - this->kernel_name = ""; - this->xclbin_name = ""; - this->xclbin_valid = false; - this->enable_preemption = false; - } - - ///@brief Copy constructor, this shall not invoke by user, it shall only be invoked by npu_xclbin_manager - ///@param other the other npu_app_manager to copy from - npu_app_manager(const npu_app_manager& other) { - this->device_gen = other.device_gen; - this->device = other.device; - this->kernel_name = other.kernel_name; - this->xclbin_name = other.xclbin_name; - this->xclbin_valid = other.xclbin_valid; - this->enable_preemption = other.enable_preemption; - if (other.context) { - this->context = std::make_unique(*other.context); - } else { - this->context = nullptr; - } - } - - ///@brief Copy assignment operator - ///@param other the other npu_app_manager to copy from - ///@return reference to this object - npu_app_manager& operator=(const npu_app_manager& other) { - if (this != &other) { - this->device_gen = other.device_gen; - this->device = other.device; - this->kernel_name = other.kernel_name; - this->xclbin_name = other.xclbin_name; - this->xclbin_valid = other.xclbin_valid; - this->enable_preemption = other.enable_preemption; - if (other.context) { - this->context = std::make_unique(*other.context); - } else { - this->context = nullptr; - } - } - return *this; - } - - ///@brief Move constructor - ///@param other the other npu_app_manager to move from - npu_app_manager(npu_app_manager&& other) noexcept { - this->device_gen = other.device_gen; - this->device = other.device; - this->context = std::move(other.context); - this->kernel_name = std::move(other.kernel_name); - this->xclbin_name = std::move(other.xclbin_name); - this->xclbin_valid = other.xclbin_valid; - this->enable_preemption = other.enable_preemption; - // Reset the moved-from object - other.device = nullptr; - other.xclbin_valid = false; - } - - ///@brief Move assignment operator - ///@param other the other npu_app_manager to move from - ///@return reference to this object - npu_app_manager& operator=(npu_app_manager&& other) noexcept { - if (this != &other) { - this->device_gen = other.device_gen; - this->device = other.device; - this->context = std::move(other.context); - this->kernel_name = std::move(other.kernel_name); - this->xclbin_name = std::move(other.xclbin_name); - this->xclbin_valid = other.xclbin_valid; - this->enable_preemption = other.enable_preemption; - // Reset the moved-from object - other.device = nullptr; - other.xclbin_valid = false; - } - return *this; - } - - ///@brief Constructor, this shall not invoke by user, it shall only be invoked by npu_xclbin_manager - ///@param device_gen the npu device - ///@param device device object - ///@param xclbin_name name of the xclbin file - ///@see xrt::device, xrt::xclbin - npu_app_manager(npu_device device_gen, xrt::device* device, std::string xclbin_name, bool enable_preemption = false){ - assert(device != nullptr); - assert(xclbin_name != ""); - this->device_gen = device_gen; - this->device = device; - this->xclbin_name = xclbin_name; - this->enable_preemption = enable_preemption; - #ifndef __WINDOWS__ - if (this->enable_preemption){ - header_print("warning", "Preemption is not supported on Linux host currently."); - header_print("warning", "Preemption is set as disabled for NPU."); - this->enable_preemption = false; - } - #endif - LOG_VERBOSE(2, "Loading xclbin: " << xclbin_name); - auto this_xclbin = xrt::xclbin(xclbin_name); - // int verbosity = VERBOSE; - std::string Node = "MLIR_AIE"; - auto xkernels = this_xclbin.get_kernels(); - auto xkernel = *std::find_if( - xkernels.begin(), - xkernels.end(), - [Node](xrt::xclbin::kernel &k) { - auto name = k.get_name(); - return name.rfind(Node, 0) == 0; - } - ); - this->device->register_xclbin(this_xclbin); - auto kernelName = xkernel.get_name(); - this->context = std::make_unique(*this->device, this_xclbin.get_uuid()); - this->kernel_name = kernelName; - this->xclbin_valid = true; - LOG_VERBOSE(2, "Xclbin: " << xclbin_name << " loaded successfully!"); - } - - ~npu_app_manager() = default; - - ///@brief Create a npu_app - ///@return a npu_app object - ///@see npu_app - npu_app create_app(){ - assert(this->xclbin_valid); - return npu_app(this->device_gen, this->device, this->context.get(), this->kernel_name, this->enable_preemption); - } - - ///@brief Create a buffer with real bo - ///@param size size of the buffer - ///@see buffer - template - buffer create_bo_buffer(size_t size){ - assert(size > 0); - LOG_VERBOSE(2, "Creating buffer buffer with size: " << size); - return buffer(*this->device, size); - } - - ///@brief Get the name of the xclbin - ///@return the name of the xclbin - std::string get_xclbin_name(){ - assert(this->xclbin_valid); - return this->xclbin_name; - } - - ///@brief Create a runlist - ///@return a runlist object - ///@see xrt::runlist - xrt::runlist create_runlist(){ - assert(this->xclbin_valid); - assert(this->enable_preemption == false); // preemption is not supported for runlist - return xrt::runlist(*this->context); - } -}; - -///@brief npu_xclbin_manager -///@note There should be only one npu_xclbin_manager inside main. -///@note It handles all xclbins -class npu_xclbin_manager{ - -private: - std::vector> npu_xclbins; - - size_t xclbin_count; - // the only device instance - xrt::device* device; - bool enable_preemption; - npu_device npu_gen; -public: - constexpr static int max_xclbins = 16; // This is hard constraint from the XRT driver - - ///@brief Constructor, this shall not invoke by user, it shall only be invoked by main - ///@param device the npu device - ///@param device_id the device id - ///@see xrt::device - npu_xclbin_manager(npu_device device = device_npu2, xrt::device* device_inst = nullptr, bool enable_preemption = false){ - this->device = device_inst; - // this->npu_xclbins.resize(max_xclbins); - this->npu_xclbins.reserve(max_xclbins); - this->xclbin_count = 0; - this->npu_gen = device; - this->enable_preemption = enable_preemption; - } - - ~npu_xclbin_manager() = default; - - ///@brief register an accel_user_desc to the npu_manager - ///@param xclbin_name the name of the xclbin - ///@return the npu_app_manager object - ///@see npu_app_manager - ///@note Different apps may have the same xclbin, but the sequence is unique. - ///@note To avoid creating duplicated applications, the function checks if the xclbin is registered. - ///@note If the xclbin is not registered, the function will register the xclbin and create a new application. - npu_app_manager* register_xclbin(std::string xclbin_name){ - assert(xclbin_name != ""); - int xclbin_id = -1; - for (size_t i = 0; i < this->xclbin_count; i++){ - - if (this->npu_xclbins[i]->get_xclbin_name() == xclbin_name){ - xclbin_id = i; - break; - } - } - LOG_VERBOSE_IF_ELSE(2, xclbin_id > -1, - "Found xclbin: " << xclbin_name << "registered as id " << xclbin_id << "!", - "Xclbin: " << xclbin_name << " not registered yet!" - ); - - if (xclbin_id == -1){ // the xclbin is not registered yet - if (this->xclbin_count >= max_xclbins){ - throw std::runtime_error("Max number of xclbins reached"); - } - this->npu_xclbins.push_back(std::make_unique(this->npu_gen, this->device, xclbin_name, this->enable_preemption)); - xclbin_id = this->xclbin_count; - LOG_VERBOSE(2, "Xclbin: " << xclbin_name << " registered as id " << xclbin_id << "!"); - this->xclbin_count++; - xclbin_id = this->xclbin_count - 1; - } - return this->npu_xclbins[xclbin_id].get(); - } - - void list_xclbins(){ - for (size_t i = 0; i < this->xclbin_count; i++){ - std::cout << "Xclbin " << i << " name: " << this->npu_xclbins[i]->get_xclbin_name() << std::endl; - } - } - - /// @brief Helper function to write out the trace to a file - /// @param traceOutPtr pointer to the trace - /// @param trace_size size of the trace - /// @param path path to the file - static void write_out_trace(char *traceOutPtr, size_t trace_size, std::string path) { - std::ofstream fout(path); - LOG_VERBOSE(1, "Writing out trace to: " << path); - uint32_t *traceOut = (uint32_t *)traceOutPtr; - for (size_t i = 0; i < trace_size / sizeof(traceOut[0]); i++) { - fout << std::setfill('0') << std::setw(8) << std::hex << (int)traceOut[i]; - fout << std::endl; - } - fout.close(); - LOG_VERBOSE(1, "Trace written successfully!"); - } - - ///@brief Create a buffer with real bo - ///@param size size of the buffer - ///@see buffer - template - buffer create_bo_buffer(size_t size){ - assert(size > 0); - LOG_VERBOSE(2, "Creating buffer buffer with size: " << size); - return buffer(*this->device, size); - } - - inline bool is_preemption_enabled(){ - return this->enable_preemption; - } - - #ifndef __WINDOWS__ - ///@brief print the npu information - ///@note The function will print the npu version, clock frequency, column count, row count, core info, mem info, shim info. - ///@note The information is read via the IOCTL interface. - void print_npu_info(){ - int fd = open("/dev/accel/accel0", O_RDWR); - if (fd < 0) { - perror("Failed to open amdgpu device"); - return; - } - amdxdna_drm_query_clock_metadata query_clock_metadata; - amdxdna_drm_get_info get_info = { - .param = DRM_AMDXDNA_QUERY_CLOCK_METADATA, - .buffer_size = sizeof(amdxdna_drm_query_clock_metadata), - .buffer = (unsigned long)&query_clock_metadata, - }; - int ret = ioctl(fd, DRM_IOCTL_AMDXDNA_GET_INFO, &get_info); - if (ret < 0) { - std::cout << "Error code: " << ret << std::endl; - perror("Failed to get telemetry information"); - close(fd); - return; - } - - amdxdna_drm_query_aie_metadata query_aie_metadata = {}; - get_info.param = DRM_AMDXDNA_QUERY_AIE_METADATA; - get_info.buffer_size = sizeof(amdxdna_drm_query_aie_metadata); - get_info.buffer = (unsigned long)&query_aie_metadata; - ret = ioctl(fd, DRM_IOCTL_AMDXDNA_GET_INFO, &get_info); - if (ret < 0) { - std::cout << "Error code: " << ret << std::endl; - perror("Failed to get telemetry information"); - close(fd); - return; - } - - close(fd); - MSG_BONDLINE(40); - MSG_BOX_LINE(40, "NPU version: " << query_aie_metadata.version.major << "." << query_aie_metadata.version.minor); - MSG_BOX_LINE(40, "MP-NPU clock frequency: " << query_clock_metadata.mp_npu_clock.freq_mhz << " MHz"); - MSG_BOX_LINE(40, "H clock frequency: " << query_clock_metadata.h_clock.freq_mhz << " MHz"); - // What is the meaning of the column size? - // std::cout << "NPU column size: " << query_aie_metadata.col_size << std::endl; - MSG_BOX_LINE(40, "NPU column count: " << query_aie_metadata.cols); - MSG_BOX_LINE(40, "NPU row count: " << query_aie_metadata.rows); - MSG_BOX_LINE(40, "NPU core Info: "); - MSG_BOX_LINE(40, "--Row count: " << query_aie_metadata.core.row_count); - MSG_BOX_LINE(40, "--Row start: " << query_aie_metadata.core.row_start); - MSG_BOX_LINE(40, "--DMA channel count: " << query_aie_metadata.core.dma_channel_count); - MSG_BOX_LINE(40, "--Lock count: " << query_aie_metadata.core.lock_count); - MSG_BOX_LINE(40, "--Event reg count: " << query_aie_metadata.core.event_reg_count); - MSG_BOX_LINE(40, "NPU mem Info: "); - MSG_BOX_LINE(40, "--Row count: " << query_aie_metadata.mem.row_count); - MSG_BOX_LINE(40, "--Row start: " << query_aie_metadata.mem.row_start); - MSG_BOX_LINE(40, "--DMA channel count: " << query_aie_metadata.mem.dma_channel_count); - MSG_BOX_LINE(40, "--Lock count: " << query_aie_metadata.mem.lock_count); - MSG_BOX_LINE(40, "--Event reg count: " << query_aie_metadata.mem.event_reg_count); - MSG_BOX_LINE(40, "NPU shim Info: "); - MSG_BOX_LINE(40, "--Row count: " << query_aie_metadata.shim.row_count); - MSG_BOX_LINE(40, "--Row start: " << query_aie_metadata.shim.row_start); - MSG_BOX_LINE(40, "--DMA channel count: " << query_aie_metadata.shim.dma_channel_count); - MSG_BOX_LINE(40, "--Lock count: " << query_aie_metadata.shim.lock_count); - MSG_BOX_LINE(40, "--Event reg count: " << query_aie_metadata.shim.event_reg_count); - MSG_BONDLINE(40); - } - - ///@brief get the npu power consumption - ///@param print whether to print the power consumption - ///@return the power consumption, unit is Watt - float get_npu_power(bool print){ - // get the npu power consumption, unit is Watt - int fd = open("/dev/accel/accel0", O_RDWR); - if (fd < 0) { - perror("Failed to open amdgpu device"); - return -1; - } - amdxdna_drm_query_sensor query_sensor; - - amdxdna_drm_get_info get_info = { - .param = DRM_AMDXDNA_QUERY_SENSORS, - .buffer_size = sizeof(amdxdna_drm_query_sensor), - .buffer = (unsigned long)&query_sensor, - }; - int ret = ioctl(fd, DRM_IOCTL_AMDXDNA_GET_INFO, &get_info); - if (ret < 0) { - std::cout << "Error code: " << ret << std::endl; - perror("Failed to get telemetry information"); - close(fd); - return -1; - } - if (print){ - MSG_BOX(40, "NPU power: " << query_sensor.input << " " << query_sensor.units); - } - close(fd); - return (float)query_sensor.input * pow(10, query_sensor.unitm); - } - #endif -}; \ No newline at end of file +#if defined(FLM_USE_HRX) +#include "npu_utils_hrx.hpp" +#else +#include "npu_utils_xrt.hpp" +#endif \ No newline at end of file diff --git a/src/include/npu_utils/npu_utils_hrx.hpp b/src/include/npu_utils/npu_utils_hrx.hpp new file mode 100644 index 00000000..6268d698 --- /dev/null +++ b/src/include/npu_utils/npu_utils_hrx.hpp @@ -0,0 +1,628 @@ +/// \file npu_utils.hpp +/// \brief npu_utils class +/// \author FastFlowLM Team, Alfred +/// \date 2025-09-09 +/// \note This file contains the classes for managing the npu device +#pragma once + +/* + The NPU is managed in this way: + For each program, there should only be one npu_xclbin_manager, example: + npu_xclbin_manager npu_mgr(device_npu2, 0); + + Then, multiple xclbins could be used by this npu and each xclbin may have multiple runtime sequences (applications) + Therefore, each registered xclbin will have a npu_app_manager, example: + + npu_app_manager* mvm_i8_xclbin = npu_mgr.register_xclbin("mvm_i8.xclbin"); + + Notice that the return value is a pointer to the npu_app_manager. This is to avoid duplicated npu hardware context, + all xclbins are managed by the npu_xclbin_manager centrally. + + Finally, an application can be create from the npu_app_manager, example: + + npu_app app = mvm_i8_xclbin->create_app(); + + In this version, the app do not have a name. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef __WINDOWS__ +#include +#include +#include +#include +#include +#include "amdxdna_accel.h" +#endif +#include "hrx_cpp/hrx_cpp.hpp" +#include "buffer.hpp" +#include "utils/debug_utils.hpp" +#include + +#include "npu_instr_utils.hpp" + + +class npu_app_manager; +class npu_xclbin_manager; + + +///@brief npu_app, a application that both xclbin and instruction are specified +///@param device the pointer to the device +///@param context the pointer to the context +///@param kernel_name the name of the kernel +///@see hrx::kernel, hrx::device +class npu_app { +private: + // from external + hrx::hw_context* context; + hrx::device* device; + std::string kernel_name; + npu_device device_gen; + bool enable_preemption; + + // self-managed + bool module_valid; + uint8_t module_version; + hrx_executable_t exe; // HRX direct executable built from ctrl_seq->dump() + uint32_t exe_ord; // resolved "MLIR_AIE" export ordinal + std::unique_ptr ctrl_seq; + + ///@brief Setup the kernel + ///@note Builds (or fetches a cached) HRX XADX executable directly from the + /// ctrl_seq TXN stream (npu_sequence::dump()). + void _setup_kernel(){ + std::pair data = this->ctrl_seq->dump(); + assert(data.first != nullptr); + assert(data.second > 0); + assert(this->context != nullptr); + this->exe = hrx::build_or_get_executable( + this->context->xclbin_bytes(), data.first, data.second, + &this->exe_ord); + if (this->exe == nullptr){ + header_print_r("ERROR", "Failed to build HRX executable from ctrl_seq"); + exit(1); + } + this->module_valid = true; + this->module_version = this->ctrl_seq->sequence_version(); + } + +public: + // enum ert_cmd_state { + // ERT_CMD_STATE_NEW = 1, + // ERT_CMD_STATE_QUEUED = 2, + // ERT_CMD_STATE_RUNNING = 3, + // ERT_CMD_STATE_COMPLETED = 4, + // ERT_CMD_STATE_ERROR = 5, + // ERT_CMD_STATE_ABORT = 6, + // ERT_CMD_STATE_SUBMITTED = 7, + // ERT_CMD_STATE_TIMEOUT = 8, + // ERT_CMD_STATE_NORESPONSE = 9, + // ERT_CMD_STATE_SKERROR = 10, //Check for error return code from Soft Kernel + // ERT_CMD_STATE_SKCRASHED = 11, //Soft kernel has crashed + // ERT_CMD_STATE_MAX, // Always the last one + // }; + std::map cmd_state_map = { + {ERT_CMD_STATE_NEW, "new"}, + {ERT_CMD_STATE_QUEUED, "queued"}, + {ERT_CMD_STATE_RUNNING, "running"}, + {ERT_CMD_STATE_COMPLETED, "completed"}, + {ERT_CMD_STATE_ERROR, "error"}, + {ERT_CMD_STATE_ABORT, "abort"}, + {ERT_CMD_STATE_SUBMITTED, "submitted"}, + {ERT_CMD_STATE_TIMEOUT, "timeout"}, + {ERT_CMD_STATE_NORESPONSE, "noresponse"}, + {ERT_CMD_STATE_SKERROR, "skerror"}, + {ERT_CMD_STATE_SKCRASHED, "skcrashed"}, + {ERT_CMD_STATE_MAX, "max"}, + }; + + ///@brief Default Constructor + ///@note Initialize the npu_app to nullptr + npu_app() { + this->device_gen = device_npu2; + this->device = nullptr; + this->context = nullptr; + this->module_valid = false; + this->exe = nullptr; + this->exe_ord = 0; + this->module_version = 0xFF; + } + + ///@brief Constructor, this shall not invoke by user, it shall only be invoked by npu_app_manager + ///@param device_gen the npu device + ///@param device the pointer to the device + ///@param context the pointer to the context + ///@param kernel_name the name of the kernel + ///@see hrx::device, hrx::hw_context, hrx::kernel + npu_app(npu_device device_gen, hrx::device* device, hrx::hw_context* context, std::string kernel_name, bool enable_preemption = false): + device_gen(device_gen), device(device), context(context), kernel_name(kernel_name), enable_preemption(enable_preemption){ + this->module_valid = false; + this->exe = nullptr; + this->exe_ord = 0; + this->ctrl_seq = std::make_unique(device_gen, enable_preemption); + this->module_version = 0xFF; + } + + + void update_ctrl_seq(){ + assert(this->ctrl_seq != nullptr); + this->_setup_kernel(); + } + + void load_elf(std::string elf_name){ + // ELF-based control loading is not used on the HRX backend (control + // code comes from ctrl_seq->dump()). + (void)elf_name; + header_print("warning", "npu_app::load_elf is not supported on the HRX backend (ignored)"); + } + + void store_elf(std::string elf_name){ + (void)elf_name; + header_print("warning", "npu_app::store_elf is not supported on the HRX backend (ignored)"); + } + + ///@brief Operator() for running the kernel + ///@param args arguments, shall be the buffers with real bo + ///@see hrx::run + template + ert_cmd_state operator()(BoArgs&&... args){ + if (this->module_valid == false || this->ctrl_seq->sequence_valid() == false || this->module_version != this->ctrl_seq->sequence_version()) { + this->_setup_kernel(); + } + hrx::run run(this->exe, this->exe_ord); + std::array bo_args = { &args... }; + for (size_t i = 0; i < sizeof...(args); i++){ + run.add_binding(bo_args[i]->bo().handle(), bo_args[i]->bo().size()); + } + run.start(); + ert_cmd_state state = run.wait(); + LOG_VERBOSE(2, "ending state: " << this->cmd_state_map[state]); + return state; + } + + + ///@brief Operator() for running the kernel + ///@param args arguments, shall be the buffers with real bo + ///@see hrx::run + template + ert_cmd_state safe_run(BoArgs&&... args){ + if (this->module_valid == false || this->ctrl_seq->sequence_valid() == false || this->module_version != this->ctrl_seq->sequence_version()) { + this->_setup_kernel(); + } + std::array bo_args = { &args... }; + for (size_t i = 0; i < sizeof...(args); i++){ + bo_args[i]->sync_to_device(); + } + hrx::run run(this->exe, this->exe_ord); + for (size_t i = 0; i < sizeof...(args); i++){ + run.add_binding(bo_args[i]->bo().handle(), bo_args[i]->bo().size()); + } + run.start(); + ert_cmd_state state = run.wait(); + for (size_t i = 0; i < sizeof...(args); i++){ + bo_args[i]->sync_from_device(); + } + LOG_VERBOSE(2, "ending state: " << this->cmd_state_map[state]); + return state; + } + + ///@brief Create a run object + ///@param args arguments, shall be the buffers with real bo + ///@return a run object for waiting + ///@see hrx::run + template + hrx::run create_run(BoArgs&&... args){ + if (this->module_valid == false || this->ctrl_seq->sequence_valid() == false || this->module_version != this->ctrl_seq->sequence_version()) { + this->_setup_kernel(); + } + hrx::run run(this->exe, this->exe_ord); + // HRX bindings are the buffer args only, in order (arg 3 = output). The + // (3,0,0) opcode scalars are encoded in the TXN stream, not as bindings. + std::array bo_args = { &args... }; + for (size_t i = 0; i < sizeof...(args); i++){ + run.add_binding(bo_args[i]->bo().handle(), bo_args[i]->bo().size()); + } + return run; + } + + ///@brief Create a buffer with real bo + ///@param size size of the buffer + ///@see buffer + template + buffer create_bo_buffer(size_t size){ + assert(size > 0); + LOG_VERBOSE(2, "Creating buffer buffer with size: " << size); + return buffer(*this->device, size); + } + + npu_sequence* seq() { + return this->ctrl_seq.get(); + } +}; + +class npu_app_manager { +private: + npu_device device_gen; + bool enable_preemption; + hrx::device* device; + std::unique_ptr context; + std::string kernel_name; + std::string xclbin_name; + bool xclbin_valid; +public: + ///@brief Default Constructor + ///@param device_gen the npu device + ///@param device device object + ///@param xclbin_name name of the xclbin file + ///@see hrx::device, hrx::xclbin + ///@note The function will initialize the npu_app_manager to nullptr + npu_app_manager(){ + this->device_gen = device_npu2; + this->device = nullptr; + this->context = nullptr; + this->kernel_name = ""; + this->xclbin_name = ""; + this->xclbin_valid = false; + this->enable_preemption = false; + } + + ///@brief Copy constructor, this shall not invoke by user, it shall only be invoked by npu_xclbin_manager + ///@param other the other npu_app_manager to copy from + npu_app_manager(const npu_app_manager& other) { + this->device_gen = other.device_gen; + this->device = other.device; + this->kernel_name = other.kernel_name; + this->xclbin_name = other.xclbin_name; + this->xclbin_valid = other.xclbin_valid; + this->enable_preemption = other.enable_preemption; + if (other.context) { + this->context = std::make_unique(*other.context); + } else { + this->context = nullptr; + } + } + + ///@brief Copy assignment operator + ///@param other the other npu_app_manager to copy from + ///@return reference to this object + npu_app_manager& operator=(const npu_app_manager& other) { + if (this != &other) { + this->device_gen = other.device_gen; + this->device = other.device; + this->kernel_name = other.kernel_name; + this->xclbin_name = other.xclbin_name; + this->xclbin_valid = other.xclbin_valid; + this->enable_preemption = other.enable_preemption; + if (other.context) { + this->context = std::make_unique(*other.context); + } else { + this->context = nullptr; + } + } + return *this; + } + + ///@brief Move constructor + ///@param other the other npu_app_manager to move from + npu_app_manager(npu_app_manager&& other) noexcept { + this->device_gen = other.device_gen; + this->device = other.device; + this->context = std::move(other.context); + this->kernel_name = std::move(other.kernel_name); + this->xclbin_name = std::move(other.xclbin_name); + this->xclbin_valid = other.xclbin_valid; + this->enable_preemption = other.enable_preemption; + // Reset the moved-from object + other.device = nullptr; + other.xclbin_valid = false; + } + + ///@brief Move assignment operator + ///@param other the other npu_app_manager to move from + ///@return reference to this object + npu_app_manager& operator=(npu_app_manager&& other) noexcept { + if (this != &other) { + this->device_gen = other.device_gen; + this->device = other.device; + this->context = std::move(other.context); + this->kernel_name = std::move(other.kernel_name); + this->xclbin_name = std::move(other.xclbin_name); + this->xclbin_valid = other.xclbin_valid; + this->enable_preemption = other.enable_preemption; + // Reset the moved-from object + other.device = nullptr; + other.xclbin_valid = false; + } + return *this; + } + + ///@brief Constructor, this shall not invoke by user, it shall only be invoked by npu_xclbin_manager + ///@param device_gen the npu device + ///@param device device object + ///@param xclbin_name name of the xclbin file + ///@see hrx::device, hrx::xclbin + npu_app_manager(npu_device device_gen, hrx::device* device, std::string xclbin_name, bool enable_preemption = false){ + assert(device != nullptr); + assert(xclbin_name != ""); + this->device_gen = device_gen; + this->device = device; + this->xclbin_name = xclbin_name; + this->enable_preemption = enable_preemption; + #ifndef __WINDOWS__ + if (this->enable_preemption){ + header_print("warning", "Preemption is not supported on Linux host currently."); + header_print("warning", "Preemption is set as disabled for NPU."); + this->enable_preemption = false; + } + #endif + LOG_VERBOSE(2, "Loading xclbin: " << xclbin_name); + auto this_xclbin = hrx::xclbin(xclbin_name); + // int verbosity = VERBOSE; + std::string Node = "MLIR_AIE"; + auto xkernels = this_xclbin.get_kernels(); + auto xkernel = *std::find_if( + xkernels.begin(), + xkernels.end(), + [Node](hrx::xclbin::kernel &k) { + auto name = k.get_name(); + return name.rfind(Node, 0) == 0; + } + ); + this->device->register_xclbin(this_xclbin); + auto kernelName = xkernel.get_name(); + // Pass the xclbin (its raw bytes) into the context so npu_app can build + // the HRX XADX executable that wraps it. + this->context = std::make_unique(*this->device, this_xclbin); + this->kernel_name = kernelName; + this->xclbin_valid = true; + LOG_VERBOSE(2, "Xclbin: " << xclbin_name << " loaded successfully!"); + } + + ~npu_app_manager() = default; + + ///@brief Create a npu_app + ///@return a npu_app object + ///@see npu_app + npu_app create_app(){ + assert(this->xclbin_valid); + return npu_app(this->device_gen, this->device, this->context.get(), this->kernel_name, this->enable_preemption); + } + + ///@brief Create a buffer with real bo + ///@param size size of the buffer + ///@see buffer + template + buffer create_bo_buffer(size_t size){ + assert(size > 0); + LOG_VERBOSE(2, "Creating buffer buffer with size: " << size); + return buffer(*this->device, size); + } + + ///@brief Get the name of the xclbin + ///@return the name of the xclbin + std::string get_xclbin_name(){ + assert(this->xclbin_valid); + return this->xclbin_name; + } + + ///@brief Create a runlist + ///@return a runlist object + ///@see hrx::runlist + hrx::runlist create_runlist(){ + assert(this->xclbin_valid); + assert(this->enable_preemption == false); // preemption is not supported for runlist + return hrx::runlist(*this->context); + } +}; + +///@brief npu_xclbin_manager +///@note There should be only one npu_xclbin_manager inside main. +///@note It handles all xclbins +class npu_xclbin_manager{ + +private: + std::vector> npu_xclbins; + + size_t xclbin_count; + // the only device instance + hrx::device* device; + bool enable_preemption; + npu_device npu_gen; +public: + constexpr static int max_xclbins = 16; // hard constraint inherited from the NPU runtime + + ///@brief Constructor, this shall not invoke by user, it shall only be invoked by main + ///@param device the npu device + ///@param device_id the device id + ///@see hrx::device + npu_xclbin_manager(npu_device device = device_npu2, hrx::device* device_inst = nullptr, bool enable_preemption = false){ + this->device = device_inst; + // this->npu_xclbins.resize(max_xclbins); + this->npu_xclbins.reserve(max_xclbins); + this->xclbin_count = 0; + this->npu_gen = device; + this->enable_preemption = enable_preemption; + } + + ~npu_xclbin_manager() = default; + + ///@brief register an accel_user_desc to the npu_manager + ///@param xclbin_name the name of the xclbin + ///@return the npu_app_manager object + ///@see npu_app_manager + ///@note Different apps may have the same xclbin, but the sequence is unique. + ///@note To avoid creating duplicated applications, the function checks if the xclbin is registered. + ///@note If the xclbin is not registered, the function will register the xclbin and create a new application. + npu_app_manager* register_xclbin(std::string xclbin_name){ + assert(xclbin_name != ""); + int xclbin_id = -1; + for (size_t i = 0; i < this->xclbin_count; i++){ + + if (this->npu_xclbins[i]->get_xclbin_name() == xclbin_name){ + xclbin_id = i; + break; + } + } + LOG_VERBOSE_IF_ELSE(2, xclbin_id > -1, + "Found xclbin: " << xclbin_name << "registered as id " << xclbin_id << "!", + "Xclbin: " << xclbin_name << " not registered yet!" + ); + + if (xclbin_id == -1){ // the xclbin is not registered yet + if (this->xclbin_count >= max_xclbins){ + throw std::runtime_error("Max number of xclbins reached"); + } + this->npu_xclbins.push_back(std::make_unique(this->npu_gen, this->device, xclbin_name, this->enable_preemption)); + xclbin_id = this->xclbin_count; + LOG_VERBOSE(2, "Xclbin: " << xclbin_name << " registered as id " << xclbin_id << "!"); + this->xclbin_count++; + xclbin_id = this->xclbin_count - 1; + } + return this->npu_xclbins[xclbin_id].get(); + } + + void list_xclbins(){ + for (size_t i = 0; i < this->xclbin_count; i++){ + std::cout << "Xclbin " << i << " name: " << this->npu_xclbins[i]->get_xclbin_name() << std::endl; + } + } + + /// @brief Helper function to write out the trace to a file + /// @param traceOutPtr pointer to the trace + /// @param trace_size size of the trace + /// @param path path to the file + static void write_out_trace(char *traceOutPtr, size_t trace_size, std::string path) { + std::ofstream fout(path); + LOG_VERBOSE(1, "Writing out trace to: " << path); + uint32_t *traceOut = (uint32_t *)traceOutPtr; + for (size_t i = 0; i < trace_size / sizeof(traceOut[0]); i++) { + fout << std::setfill('0') << std::setw(8) << std::hex << (int)traceOut[i]; + fout << std::endl; + } + fout.close(); + LOG_VERBOSE(1, "Trace written successfully!"); + } + + ///@brief Create a buffer with real bo + ///@param size size of the buffer + ///@see buffer + template + buffer create_bo_buffer(size_t size){ + assert(size > 0); + LOG_VERBOSE(2, "Creating buffer buffer with size: " << size); + return buffer(*this->device, size); + } + + inline bool is_preemption_enabled(){ + return this->enable_preemption; + } + + #ifndef __WINDOWS__ + ///@brief print the npu information + ///@note The function will print the npu version, clock frequency, column count, row count, core info, mem info, shim info. + ///@note The information is read via the IOCTL interface. + void print_npu_info(){ + int fd = open("/dev/accel/accel0", O_RDWR); + if (fd < 0) { + perror("Failed to open amdgpu device"); + return; + } + amdxdna_drm_query_clock_metadata query_clock_metadata; + amdxdna_drm_get_info get_info = { + .param = DRM_AMDXDNA_QUERY_CLOCK_METADATA, + .buffer_size = sizeof(amdxdna_drm_query_clock_metadata), + .buffer = (unsigned long)&query_clock_metadata, + }; + int ret = ioctl(fd, DRM_IOCTL_AMDXDNA_GET_INFO, &get_info); + if (ret < 0) { + std::cout << "Error code: " << ret << std::endl; + perror("Failed to get telemetry information"); + close(fd); + return; + } + + amdxdna_drm_query_aie_metadata query_aie_metadata = {}; + get_info.param = DRM_AMDXDNA_QUERY_AIE_METADATA; + get_info.buffer_size = sizeof(amdxdna_drm_query_aie_metadata); + get_info.buffer = (unsigned long)&query_aie_metadata; + ret = ioctl(fd, DRM_IOCTL_AMDXDNA_GET_INFO, &get_info); + if (ret < 0) { + std::cout << "Error code: " << ret << std::endl; + perror("Failed to get telemetry information"); + close(fd); + return; + } + + close(fd); + MSG_BONDLINE(40); + MSG_BOX_LINE(40, "NPU version: " << query_aie_metadata.version.major << "." << query_aie_metadata.version.minor); + MSG_BOX_LINE(40, "MP-NPU clock frequency: " << query_clock_metadata.mp_npu_clock.freq_mhz << " MHz"); + MSG_BOX_LINE(40, "H clock frequency: " << query_clock_metadata.h_clock.freq_mhz << " MHz"); + // What is the meaning of the column size? + // std::cout << "NPU column size: " << query_aie_metadata.col_size << std::endl; + MSG_BOX_LINE(40, "NPU column count: " << query_aie_metadata.cols); + MSG_BOX_LINE(40, "NPU row count: " << query_aie_metadata.rows); + MSG_BOX_LINE(40, "NPU core Info: "); + MSG_BOX_LINE(40, "--Row count: " << query_aie_metadata.core.row_count); + MSG_BOX_LINE(40, "--Row start: " << query_aie_metadata.core.row_start); + MSG_BOX_LINE(40, "--DMA channel count: " << query_aie_metadata.core.dma_channel_count); + MSG_BOX_LINE(40, "--Lock count: " << query_aie_metadata.core.lock_count); + MSG_BOX_LINE(40, "--Event reg count: " << query_aie_metadata.core.event_reg_count); + MSG_BOX_LINE(40, "NPU mem Info: "); + MSG_BOX_LINE(40, "--Row count: " << query_aie_metadata.mem.row_count); + MSG_BOX_LINE(40, "--Row start: " << query_aie_metadata.mem.row_start); + MSG_BOX_LINE(40, "--DMA channel count: " << query_aie_metadata.mem.dma_channel_count); + MSG_BOX_LINE(40, "--Lock count: " << query_aie_metadata.mem.lock_count); + MSG_BOX_LINE(40, "--Event reg count: " << query_aie_metadata.mem.event_reg_count); + MSG_BOX_LINE(40, "NPU shim Info: "); + MSG_BOX_LINE(40, "--Row count: " << query_aie_metadata.shim.row_count); + MSG_BOX_LINE(40, "--Row start: " << query_aie_metadata.shim.row_start); + MSG_BOX_LINE(40, "--DMA channel count: " << query_aie_metadata.shim.dma_channel_count); + MSG_BOX_LINE(40, "--Lock count: " << query_aie_metadata.shim.lock_count); + MSG_BOX_LINE(40, "--Event reg count: " << query_aie_metadata.shim.event_reg_count); + MSG_BONDLINE(40); + } + + ///@brief get the npu power consumption + ///@param print whether to print the power consumption + ///@return the power consumption, unit is Watt + float get_npu_power(bool print){ + // get the npu power consumption, unit is Watt + int fd = open("/dev/accel/accel0", O_RDWR); + if (fd < 0) { + perror("Failed to open amdgpu device"); + return -1; + } + amdxdna_drm_query_sensor query_sensor; + + amdxdna_drm_get_info get_info = { + .param = DRM_AMDXDNA_QUERY_SENSORS, + .buffer_size = sizeof(amdxdna_drm_query_sensor), + .buffer = (unsigned long)&query_sensor, + }; + int ret = ioctl(fd, DRM_IOCTL_AMDXDNA_GET_INFO, &get_info); + if (ret < 0) { + std::cout << "Error code: " << ret << std::endl; + perror("Failed to get telemetry information"); + close(fd); + return -1; + } + if (print){ + MSG_BOX(40, "NPU power: " << query_sensor.input << " " << query_sensor.units); + } + close(fd); + return (float)query_sensor.input * pow(10, query_sensor.unitm); + } + #endif +}; diff --git a/src/include/npu_utils/npu_utils_xrt.hpp b/src/include/npu_utils/npu_utils_xrt.hpp new file mode 100644 index 00000000..0aa6ed5f --- /dev/null +++ b/src/include/npu_utils/npu_utils_xrt.hpp @@ -0,0 +1,664 @@ +/// \file npu_utils.hpp +/// \brief npu_utils class +/// \author FastFlowLM Team, Alfred +/// \date 2025-09-09 +/// \note This file contains the classes for managing the npu device +#pragma once + +/* + The NPU is managed in this way: + For each program, there should only be one npu_xclbin_manager, example: + npu_xclbin_manager npu_mgr(device_npu2, 0); + + Then, multiple xclbins could be used by this npu and each xclbin may have multiple runtime sequences (applications) + Therefore, each registered xclbin will have a npu_app_manager, example: + + npu_app_manager* mvm_i8_xclbin = npu_mgr.register_xclbin("mvm_i8.xclbin"); + + Notice that the return value is a pointer to the npu_app_manager. This is to avoid duplicated npu hardware context, + all xclbins are managed by the npu_xclbin_manager centrally. + + Finally, an application can be create from the npu_app_manager, example: + + npu_app app = mvm_i8_xclbin->create_app(); + + In this version, the app do not have a name. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef __WINDOWS__ +#include +#include +#include +#include +#include +#include "amdxdna_accel.h" +#endif +#include "xrt/xrt_bo.h" +#include "xrt/xrt_device.h" +#include "xrt/xrt_kernel.h" +#include "buffer.hpp" +#include "utils/debug_utils.hpp" +#include + +#include "xrt/experimental/xrt_kernel.h" +#include "xrt/experimental/xrt_ext.h" +#include "xrt/experimental/xrt_module.h" +#include "xrt/experimental/xrt_elf.h" + +#include "aiebu/aiebu.h" + +#include "npu_instr_utils.hpp" + + +class npu_app_manager; +class npu_xclbin_manager; + + +///@brief npu_app, a application that both xclbin and instruction are specified +///@param device the pointer to the device +///@param context the pointer to the context +///@param kernel_name the name of the kernel +///@see xrt::kernel, xrt::device +class npu_app { +private: + // from external + xrt::hw_context* context; + xrt::device* device; + std::string kernel_name; + npu_device device_gen; + bool enable_preemption; + + // self-managed + bool module_valid; + uint8_t module_version; + std::unique_ptr module; + std::unique_ptr elf; + std::unique_ptr kernel; + std::unique_ptr ctrl_seq; + + uint32_t _gen_elf(char** elf_buf, std::pair& instruction_data){ + uint32_t elf_buf_size = aiebu_assembler_get_elf( + aiebu_assembler_buffer_type_blob_instr_transaction, + (char*) instruction_data.first, instruction_data.second * sizeof(uint32_t), + NULL, 0, (void**)elf_buf, NULL, 0, "", "", NULL, 0); + assert(elf_buf_size > 0); + if (elf_buf_size == 0){ + header_print_r("ERROR", "Failed to get elf from ctrl_seq"); + exit(1); + } + return elf_buf_size; + } + + ///@brief Setup the kernel + ///@note The function will create an elf file from the ctrl_seq + ///@note The function will also update the module, elf, and kernel + void _setup_kernel(){ + char* elf_buf; + this->kernel.reset(); + this->module.reset(); + this->elf.reset(); + std::pair data = this->ctrl_seq->dump(); + assert(data.first != nullptr); + assert(data.second > 0); + uint32_t elf_buf_size = this->_gen_elf(&elf_buf, data); + if (this->module_valid){ + this->module.reset(); + this->elf.reset(); + this->kernel.reset(); + } + this->elf = std::make_unique(elf_buf, elf_buf_size); + this->module = std::make_unique(*this->elf); + this->kernel = std::make_unique(*this->context, *this->module, this->kernel_name); + this->module_valid = true; + this->module_version = this->ctrl_seq->sequence_version(); + + free((void*)elf_buf); + } + +public: + // enum ert_cmd_state { + // ERT_CMD_STATE_NEW = 1, + // ERT_CMD_STATE_QUEUED = 2, + // ERT_CMD_STATE_RUNNING = 3, + // ERT_CMD_STATE_COMPLETED = 4, + // ERT_CMD_STATE_ERROR = 5, + // ERT_CMD_STATE_ABORT = 6, + // ERT_CMD_STATE_SUBMITTED = 7, + // ERT_CMD_STATE_TIMEOUT = 8, + // ERT_CMD_STATE_NORESPONSE = 9, + // ERT_CMD_STATE_SKERROR = 10, //Check for error return code from Soft Kernel + // ERT_CMD_STATE_SKCRASHED = 11, //Soft kernel has crashed + // ERT_CMD_STATE_MAX, // Always the last one + // }; + std::map cmd_state_map = { + {ERT_CMD_STATE_NEW, "new"}, + {ERT_CMD_STATE_QUEUED, "queued"}, + {ERT_CMD_STATE_RUNNING, "running"}, + {ERT_CMD_STATE_COMPLETED, "completed"}, + {ERT_CMD_STATE_ERROR, "error"}, + {ERT_CMD_STATE_ABORT, "abort"}, + {ERT_CMD_STATE_SUBMITTED, "submitted"}, + {ERT_CMD_STATE_TIMEOUT, "timeout"}, + {ERT_CMD_STATE_NORESPONSE, "noresponse"}, + {ERT_CMD_STATE_SKERROR, "skerror"}, + {ERT_CMD_STATE_SKCRASHED, "skcrashed"}, + {ERT_CMD_STATE_MAX, "max"}, + }; + + ///@brief Default Constructor + ///@note Initialize the npu_app to nullptr + npu_app() { + this->device_gen = device_npu2; + this->kernel = nullptr; + this->device = nullptr; + this->module_valid = false; + this->module = nullptr; + this->elf = nullptr; + this->kernel = nullptr; + this->module_version = 0xFF; + } + + ///@brief Constructor, this shall not invoke by user, it shall only be invoked by npu_app_manager + ///@param device_gen the npu device + ///@param device the pointer to the device + ///@param context the pointer to the context + ///@param kernel_name the name of the kernel + ///@see xrt::device, xrt::hw_context, xrt::kernel + npu_app(npu_device device_gen, xrt::device* device, xrt::hw_context* context, std::string kernel_name, bool enable_preemption = false): + device_gen(device_gen), device(device), context(context), kernel_name(kernel_name), enable_preemption(enable_preemption){ + this->module_valid = false; + this->module = nullptr; + this->elf = nullptr; + this->kernel = nullptr; + this->ctrl_seq = std::make_unique(device_gen, enable_preemption); + this->module_version = 0xFF; + } + + + void update_ctrl_seq(){ + assert(this->ctrl_seq != nullptr); + this->_setup_kernel(); + } + + void load_elf(std::string elf_name){ + this->ctrl_seq->clear_cmds(); + this->ctrl_seq->cmds2seq(); // just for making sure the sequence is valid + this->elf = std::make_unique(elf_name); + this->module = std::make_unique(*this->elf); + this->kernel = std::make_unique(*this->context, *this->module, this->kernel_name); + this->module_valid = true; + this->module_version = this->ctrl_seq->sequence_version(); // force sync the sequence version + } + + void store_elf(std::string elf_name){ + char* elf_buf; + if (this->module_valid == false) { + this->_setup_kernel(); + } + std::pair data = this->ctrl_seq->dump(); + uint32_t elf_buf_size = this->_gen_elf(&elf_buf, data); + std::ofstream fout(elf_name, std::ios::binary); + if (fout.is_open() == false) { + header_print_r("ERROR", "Failed to open file: " << elf_name); + exit(1); + } + fout.write((char*)elf_buf, elf_buf_size); + fout.close(); + free((void*)elf_buf); + } + + ///@brief Operator() for running the kernel + ///@param args arguments, shall be the buffers with real bo + ///@see xrt::run + template + ert_cmd_state operator()(BoArgs&&... args){ + if (this->module_valid == false || this->ctrl_seq->sequence_valid() == false || this->module_version != this->ctrl_seq->sequence_version()) { + this->_setup_kernel(); + } + auto run = this->kernel->operator()(3, 0, 0, args.bo()...); + ert_cmd_state state = run.wait(); + LOG_VERBOSE(2, "ending state: " << this->cmd_state_map[state]); + return state; + } + + + ///@brief Operator() for running the kernel + ///@param args arguments, shall be the buffers with real bo + ///@see xrt::run + template + ert_cmd_state safe_run(BoArgs&&... args){ + if (this->module_valid == false || this->ctrl_seq->sequence_valid() == false || this->module_version != this->ctrl_seq->sequence_version()) { + this->_setup_kernel(); + } + std::array bo_args = { &args... }; + for (size_t i = 0; i < sizeof...(args); i++){ + bo_args[i]->sync_to_device(); + } + auto run = this->kernel->operator()(3, 0, 0, args.bo()...); + ert_cmd_state state = run.wait(); + for (size_t i = 0; i < sizeof...(args); i++){ + bo_args[i]->sync_from_device(); + } + LOG_VERBOSE(2, "ending state: " << this->cmd_state_map[state]); + return state; + } + + ///@brief Create a run object + ///@param args arguments, shall be the buffers with real bo + ///@return a run object for waiting + ///@see xrt::run + template + xrt::run create_run(BoArgs&&... args){ + if (this->module_valid == false || this->ctrl_seq->sequence_valid() == false || this->module_version != this->ctrl_seq->sequence_version()) { + this->_setup_kernel(); + } + xrt::run run = xrt::run(*this->kernel); + run.set_arg(0, 3); + run.set_arg(1, 0); + run.set_arg(2, 0); + std::array bo_args = { &args... }; + for (size_t i = 0; i < sizeof...(args); i++){ + run.set_arg(3 + i, bo_args[i]->bo()); + } + return run; + } + + ///@brief Create a buffer with real bo + ///@param size size of the buffer + ///@see buffer + template + buffer create_bo_buffer(size_t size){ + assert(size > 0); + LOG_VERBOSE(2, "Creating buffer buffer with size: " << size); + return buffer(*this->device, size); + } + + npu_sequence* seq() { + return this->ctrl_seq.get(); + } +}; + +class npu_app_manager { +private: + npu_device device_gen; + bool enable_preemption; + xrt::device* device; + std::unique_ptr context; + std::string kernel_name; + std::string xclbin_name; + bool xclbin_valid; +public: + ///@brief Default Constructor + ///@param device_gen the npu device + ///@param device device object + ///@param xclbin_name name of the xclbin file + ///@see xrt::device, xrt::xclbin + ///@note The function will initialize the npu_app_manager to nullptr + npu_app_manager(){ + this->device_gen = device_npu2; + this->device = nullptr; + this->context = nullptr; + this->kernel_name = ""; + this->xclbin_name = ""; + this->xclbin_valid = false; + this->enable_preemption = false; + } + + ///@brief Copy constructor, this shall not invoke by user, it shall only be invoked by npu_xclbin_manager + ///@param other the other npu_app_manager to copy from + npu_app_manager(const npu_app_manager& other) { + this->device_gen = other.device_gen; + this->device = other.device; + this->kernel_name = other.kernel_name; + this->xclbin_name = other.xclbin_name; + this->xclbin_valid = other.xclbin_valid; + this->enable_preemption = other.enable_preemption; + if (other.context) { + this->context = std::make_unique(*other.context); + } else { + this->context = nullptr; + } + } + + ///@brief Copy assignment operator + ///@param other the other npu_app_manager to copy from + ///@return reference to this object + npu_app_manager& operator=(const npu_app_manager& other) { + if (this != &other) { + this->device_gen = other.device_gen; + this->device = other.device; + this->kernel_name = other.kernel_name; + this->xclbin_name = other.xclbin_name; + this->xclbin_valid = other.xclbin_valid; + this->enable_preemption = other.enable_preemption; + if (other.context) { + this->context = std::make_unique(*other.context); + } else { + this->context = nullptr; + } + } + return *this; + } + + ///@brief Move constructor + ///@param other the other npu_app_manager to move from + npu_app_manager(npu_app_manager&& other) noexcept { + this->device_gen = other.device_gen; + this->device = other.device; + this->context = std::move(other.context); + this->kernel_name = std::move(other.kernel_name); + this->xclbin_name = std::move(other.xclbin_name); + this->xclbin_valid = other.xclbin_valid; + this->enable_preemption = other.enable_preemption; + // Reset the moved-from object + other.device = nullptr; + other.xclbin_valid = false; + } + + ///@brief Move assignment operator + ///@param other the other npu_app_manager to move from + ///@return reference to this object + npu_app_manager& operator=(npu_app_manager&& other) noexcept { + if (this != &other) { + this->device_gen = other.device_gen; + this->device = other.device; + this->context = std::move(other.context); + this->kernel_name = std::move(other.kernel_name); + this->xclbin_name = std::move(other.xclbin_name); + this->xclbin_valid = other.xclbin_valid; + this->enable_preemption = other.enable_preemption; + // Reset the moved-from object + other.device = nullptr; + other.xclbin_valid = false; + } + return *this; + } + + ///@brief Constructor, this shall not invoke by user, it shall only be invoked by npu_xclbin_manager + ///@param device_gen the npu device + ///@param device device object + ///@param xclbin_name name of the xclbin file + ///@see xrt::device, xrt::xclbin + npu_app_manager(npu_device device_gen, xrt::device* device, std::string xclbin_name, bool enable_preemption = false){ + assert(device != nullptr); + assert(xclbin_name != ""); + this->device_gen = device_gen; + this->device = device; + this->xclbin_name = xclbin_name; + this->enable_preemption = enable_preemption; + #ifndef __WINDOWS__ + if (this->enable_preemption){ + header_print("warning", "Preemption is not supported on Linux host currently."); + header_print("warning", "Preemption is set as disabled for NPU."); + this->enable_preemption = false; + } + #endif + LOG_VERBOSE(2, "Loading xclbin: " << xclbin_name); + auto this_xclbin = xrt::xclbin(xclbin_name); + // int verbosity = VERBOSE; + std::string Node = "MLIR_AIE"; + auto xkernels = this_xclbin.get_kernels(); + auto xkernel = *std::find_if( + xkernels.begin(), + xkernels.end(), + [Node](xrt::xclbin::kernel &k) { + auto name = k.get_name(); + return name.rfind(Node, 0) == 0; + } + ); + this->device->register_xclbin(this_xclbin); + auto kernelName = xkernel.get_name(); + this->context = std::make_unique(*this->device, this_xclbin.get_uuid()); + this->kernel_name = kernelName; + this->xclbin_valid = true; + LOG_VERBOSE(2, "Xclbin: " << xclbin_name << " loaded successfully!"); + } + + ~npu_app_manager() = default; + + ///@brief Create a npu_app + ///@return a npu_app object + ///@see npu_app + npu_app create_app(){ + assert(this->xclbin_valid); + return npu_app(this->device_gen, this->device, this->context.get(), this->kernel_name, this->enable_preemption); + } + + ///@brief Create a buffer with real bo + ///@param size size of the buffer + ///@see buffer + template + buffer create_bo_buffer(size_t size){ + assert(size > 0); + LOG_VERBOSE(2, "Creating buffer buffer with size: " << size); + return buffer(*this->device, size); + } + + ///@brief Get the name of the xclbin + ///@return the name of the xclbin + std::string get_xclbin_name(){ + assert(this->xclbin_valid); + return this->xclbin_name; + } + + ///@brief Create a runlist + ///@return a runlist object + ///@see xrt::runlist + xrt::runlist create_runlist(){ + assert(this->xclbin_valid); + assert(this->enable_preemption == false); // preemption is not supported for runlist + return xrt::runlist(*this->context); + } +}; + +///@brief npu_xclbin_manager +///@note There should be only one npu_xclbin_manager inside main. +///@note It handles all xclbins +class npu_xclbin_manager{ + +private: + std::vector> npu_xclbins; + + size_t xclbin_count; + // the only device instance + xrt::device* device; + bool enable_preemption; + npu_device npu_gen; +public: + constexpr static int max_xclbins = 16; // This is hard constraint from the XRT driver + + ///@brief Constructor, this shall not invoke by user, it shall only be invoked by main + ///@param device the npu device + ///@param device_id the device id + ///@see xrt::device + npu_xclbin_manager(npu_device device = device_npu2, xrt::device* device_inst = nullptr, bool enable_preemption = false){ + this->device = device_inst; + // this->npu_xclbins.resize(max_xclbins); + this->npu_xclbins.reserve(max_xclbins); + this->xclbin_count = 0; + this->npu_gen = device; + this->enable_preemption = enable_preemption; + } + + ~npu_xclbin_manager() = default; + + ///@brief register an accel_user_desc to the npu_manager + ///@param xclbin_name the name of the xclbin + ///@return the npu_app_manager object + ///@see npu_app_manager + ///@note Different apps may have the same xclbin, but the sequence is unique. + ///@note To avoid creating duplicated applications, the function checks if the xclbin is registered. + ///@note If the xclbin is not registered, the function will register the xclbin and create a new application. + npu_app_manager* register_xclbin(std::string xclbin_name){ + assert(xclbin_name != ""); + int xclbin_id = -1; + for (size_t i = 0; i < this->xclbin_count; i++){ + + if (this->npu_xclbins[i]->get_xclbin_name() == xclbin_name){ + xclbin_id = i; + break; + } + } + LOG_VERBOSE_IF_ELSE(2, xclbin_id > -1, + "Found xclbin: " << xclbin_name << "registered as id " << xclbin_id << "!", + "Xclbin: " << xclbin_name << " not registered yet!" + ); + + if (xclbin_id == -1){ // the xclbin is not registered yet + if (this->xclbin_count >= max_xclbins){ + throw std::runtime_error("Max number of xclbins reached"); + } + this->npu_xclbins.push_back(std::make_unique(this->npu_gen, this->device, xclbin_name, this->enable_preemption)); + xclbin_id = this->xclbin_count; + LOG_VERBOSE(2, "Xclbin: " << xclbin_name << " registered as id " << xclbin_id << "!"); + this->xclbin_count++; + xclbin_id = this->xclbin_count - 1; + } + return this->npu_xclbins[xclbin_id].get(); + } + + void list_xclbins(){ + for (size_t i = 0; i < this->xclbin_count; i++){ + std::cout << "Xclbin " << i << " name: " << this->npu_xclbins[i]->get_xclbin_name() << std::endl; + } + } + + /// @brief Helper function to write out the trace to a file + /// @param traceOutPtr pointer to the trace + /// @param trace_size size of the trace + /// @param path path to the file + static void write_out_trace(char *traceOutPtr, size_t trace_size, std::string path) { + std::ofstream fout(path); + LOG_VERBOSE(1, "Writing out trace to: " << path); + uint32_t *traceOut = (uint32_t *)traceOutPtr; + for (size_t i = 0; i < trace_size / sizeof(traceOut[0]); i++) { + fout << std::setfill('0') << std::setw(8) << std::hex << (int)traceOut[i]; + fout << std::endl; + } + fout.close(); + LOG_VERBOSE(1, "Trace written successfully!"); + } + + ///@brief Create a buffer with real bo + ///@param size size of the buffer + ///@see buffer + template + buffer create_bo_buffer(size_t size){ + assert(size > 0); + LOG_VERBOSE(2, "Creating buffer buffer with size: " << size); + return buffer(*this->device, size); + } + + inline bool is_preemption_enabled(){ + return this->enable_preemption; + } + + #ifndef __WINDOWS__ + ///@brief print the npu information + ///@note The function will print the npu version, clock frequency, column count, row count, core info, mem info, shim info. + ///@note The information is read via the IOCTL interface. + void print_npu_info(){ + int fd = open("/dev/accel/accel0", O_RDWR); + if (fd < 0) { + perror("Failed to open amdgpu device"); + return; + } + amdxdna_drm_query_clock_metadata query_clock_metadata; + amdxdna_drm_get_info get_info = { + .param = DRM_AMDXDNA_QUERY_CLOCK_METADATA, + .buffer_size = sizeof(amdxdna_drm_query_clock_metadata), + .buffer = (unsigned long)&query_clock_metadata, + }; + int ret = ioctl(fd, DRM_IOCTL_AMDXDNA_GET_INFO, &get_info); + if (ret < 0) { + std::cout << "Error code: " << ret << std::endl; + perror("Failed to get telemetry information"); + close(fd); + return; + } + + amdxdna_drm_query_aie_metadata query_aie_metadata = {}; + get_info.param = DRM_AMDXDNA_QUERY_AIE_METADATA; + get_info.buffer_size = sizeof(amdxdna_drm_query_aie_metadata); + get_info.buffer = (unsigned long)&query_aie_metadata; + ret = ioctl(fd, DRM_IOCTL_AMDXDNA_GET_INFO, &get_info); + if (ret < 0) { + std::cout << "Error code: " << ret << std::endl; + perror("Failed to get telemetry information"); + close(fd); + return; + } + + close(fd); + MSG_BONDLINE(40); + MSG_BOX_LINE(40, "NPU version: " << query_aie_metadata.version.major << "." << query_aie_metadata.version.minor); + MSG_BOX_LINE(40, "MP-NPU clock frequency: " << query_clock_metadata.mp_npu_clock.freq_mhz << " MHz"); + MSG_BOX_LINE(40, "H clock frequency: " << query_clock_metadata.h_clock.freq_mhz << " MHz"); + // What is the meaning of the column size? + // std::cout << "NPU column size: " << query_aie_metadata.col_size << std::endl; + MSG_BOX_LINE(40, "NPU column count: " << query_aie_metadata.cols); + MSG_BOX_LINE(40, "NPU row count: " << query_aie_metadata.rows); + MSG_BOX_LINE(40, "NPU core Info: "); + MSG_BOX_LINE(40, "--Row count: " << query_aie_metadata.core.row_count); + MSG_BOX_LINE(40, "--Row start: " << query_aie_metadata.core.row_start); + MSG_BOX_LINE(40, "--DMA channel count: " << query_aie_metadata.core.dma_channel_count); + MSG_BOX_LINE(40, "--Lock count: " << query_aie_metadata.core.lock_count); + MSG_BOX_LINE(40, "--Event reg count: " << query_aie_metadata.core.event_reg_count); + MSG_BOX_LINE(40, "NPU mem Info: "); + MSG_BOX_LINE(40, "--Row count: " << query_aie_metadata.mem.row_count); + MSG_BOX_LINE(40, "--Row start: " << query_aie_metadata.mem.row_start); + MSG_BOX_LINE(40, "--DMA channel count: " << query_aie_metadata.mem.dma_channel_count); + MSG_BOX_LINE(40, "--Lock count: " << query_aie_metadata.mem.lock_count); + MSG_BOX_LINE(40, "--Event reg count: " << query_aie_metadata.mem.event_reg_count); + MSG_BOX_LINE(40, "NPU shim Info: "); + MSG_BOX_LINE(40, "--Row count: " << query_aie_metadata.shim.row_count); + MSG_BOX_LINE(40, "--Row start: " << query_aie_metadata.shim.row_start); + MSG_BOX_LINE(40, "--DMA channel count: " << query_aie_metadata.shim.dma_channel_count); + MSG_BOX_LINE(40, "--Lock count: " << query_aie_metadata.shim.lock_count); + MSG_BOX_LINE(40, "--Event reg count: " << query_aie_metadata.shim.event_reg_count); + MSG_BONDLINE(40); + } + + ///@brief get the npu power consumption + ///@param print whether to print the power consumption + ///@return the power consumption, unit is Watt + float get_npu_power(bool print){ + // get the npu power consumption, unit is Watt + int fd = open("/dev/accel/accel0", O_RDWR); + if (fd < 0) { + perror("Failed to open amdgpu device"); + return -1; + } + amdxdna_drm_query_sensor query_sensor; + + amdxdna_drm_get_info get_info = { + .param = DRM_AMDXDNA_QUERY_SENSORS, + .buffer_size = sizeof(amdxdna_drm_query_sensor), + .buffer = (unsigned long)&query_sensor, + }; + int ret = ioctl(fd, DRM_IOCTL_AMDXDNA_GET_INFO, &get_info); + if (ret < 0) { + std::cout << "Error code: " << ret << std::endl; + perror("Failed to get telemetry information"); + close(fd); + return -1; + } + if (print){ + MSG_BOX(40, "NPU power: " << query_sensor.input << " " << query_sensor.units); + } + close(fd); + return (float)query_sensor.input * pow(10, query_sensor.unitm); + } + #endif +}; \ No newline at end of file diff --git a/src/include/typedef.hpp b/src/include/typedef.hpp index 0bb6e203..60a312a6 100644 --- a/src/include/typedef.hpp +++ b/src/include/typedef.hpp @@ -13,6 +13,9 @@ #include #include #include +#ifdef _WIN32 +#include +#endif #include "biovault_bfloat16.h" typedef float f32; diff --git a/src/include/utils/debug_utils.hpp b/src/include/utils/debug_utils.hpp index 1034fa00..9189b826 100644 --- a/src/include/utils/debug_utils.hpp +++ b/src/include/utils/debug_utils.hpp @@ -7,6 +7,7 @@ #pragma once #include #include +#include #include #ifndef VERBOSE diff --git a/src/include/whisper/modeling_whisper.hpp b/src/include/whisper/modeling_whisper.hpp index 7a17173c..8bb8e587 100644 --- a/src/include/whisper/modeling_whisper.hpp +++ b/src/include/whisper/modeling_whisper.hpp @@ -134,7 +134,7 @@ class Whisper { std::unique_ptr whisper_engine; std::unique_ptr npu; std::unique_ptr lm_config = nullptr; - xrt::device* device; + flm_rt::device* device; bool enable_preemption; @@ -241,7 +241,7 @@ class Whisper { e_transcribe = 1 } whisper_task_type_t; - Whisper(xrt::device* npu_device_inst); + Whisper(flm_rt::device* npu_device_inst); void load_model(std::string model_path, nlohmann::ordered_json model_inf, bool enable_preemption = false); //void toggle_enable_think() override; diff --git a/src/lib/hrx/dequant.dll b/src/lib/hrx/dequant.dll new file mode 100644 index 00000000..07783f87 Binary files /dev/null and b/src/lib/hrx/dequant.dll differ diff --git a/src/lib/hrx/dequant.lib b/src/lib/hrx/dequant.lib new file mode 100644 index 00000000..40928aae Binary files /dev/null and b/src/lib/hrx/dequant.lib differ diff --git a/src/lib/hrx/dequant_new.dll b/src/lib/hrx/dequant_new.dll new file mode 100644 index 00000000..8a82d4ae Binary files /dev/null and b/src/lib/hrx/dequant_new.dll differ diff --git a/src/lib/hrx/dequant_new.lib b/src/lib/hrx/dequant_new.lib new file mode 100644 index 00000000..af2f9584 Binary files /dev/null and b/src/lib/hrx/dequant_new.lib differ diff --git a/src/lib/hrx/gemm.dll b/src/lib/hrx/gemm.dll new file mode 100644 index 00000000..e47cbeba Binary files /dev/null and b/src/lib/hrx/gemm.dll differ diff --git a/src/lib/hrx/gemm.lib b/src/lib/hrx/gemm.lib new file mode 100644 index 00000000..afd6c330 Binary files /dev/null and b/src/lib/hrx/gemm.lib differ diff --git a/src/lib/hrx/gemma4e_npu.dll b/src/lib/hrx/gemma4e_npu.dll new file mode 100644 index 00000000..9b22bd4f Binary files /dev/null and b/src/lib/hrx/gemma4e_npu.dll differ diff --git a/src/lib/hrx/gemma4e_npu.lib b/src/lib/hrx/gemma4e_npu.lib new file mode 100644 index 00000000..dd98bad0 Binary files /dev/null and b/src/lib/hrx/gemma4e_npu.lib differ diff --git a/src/lib/hrx/gemma_embedding.dll b/src/lib/hrx/gemma_embedding.dll new file mode 100644 index 00000000..b75e11e5 Binary files /dev/null and b/src/lib/hrx/gemma_embedding.dll differ diff --git a/src/lib/hrx/gemma_embedding.lib b/src/lib/hrx/gemma_embedding.lib new file mode 100644 index 00000000..7e22e6c2 Binary files /dev/null and b/src/lib/hrx/gemma_embedding.lib differ diff --git a/src/lib/hrx/gemma_npu.dll b/src/lib/hrx/gemma_npu.dll new file mode 100644 index 00000000..42cfb464 Binary files /dev/null and b/src/lib/hrx/gemma_npu.dll differ diff --git a/src/lib/hrx/gemma_npu.lib b/src/lib/hrx/gemma_npu.lib new file mode 100644 index 00000000..f09ad7fd Binary files /dev/null and b/src/lib/hrx/gemma_npu.lib differ diff --git a/src/lib/hrx/gemma_text_npu.dll b/src/lib/hrx/gemma_text_npu.dll new file mode 100644 index 00000000..f610ca88 Binary files /dev/null and b/src/lib/hrx/gemma_text_npu.dll differ diff --git a/src/lib/hrx/gemma_text_npu.lib b/src/lib/hrx/gemma_text_npu.lib new file mode 100644 index 00000000..3af93529 Binary files /dev/null and b/src/lib/hrx/gemma_text_npu.lib differ diff --git a/src/lib/hrx/gpt_oss_npu.dll b/src/lib/hrx/gpt_oss_npu.dll new file mode 100644 index 00000000..13cdd894 Binary files /dev/null and b/src/lib/hrx/gpt_oss_npu.dll differ diff --git a/src/lib/hrx/gpt_oss_npu.lib b/src/lib/hrx/gpt_oss_npu.lib new file mode 100644 index 00000000..aec6d1ec Binary files /dev/null and b/src/lib/hrx/gpt_oss_npu.lib differ diff --git a/src/lib/hrx/lfm2_npu.dll b/src/lib/hrx/lfm2_npu.dll new file mode 100644 index 00000000..9494ca84 Binary files /dev/null and b/src/lib/hrx/lfm2_npu.dll differ diff --git a/src/lib/hrx/lfm2_npu.lib b/src/lib/hrx/lfm2_npu.lib new file mode 100644 index 00000000..27514a8d Binary files /dev/null and b/src/lib/hrx/lfm2_npu.lib differ diff --git a/src/lib/hrx/libdequant.so b/src/lib/hrx/libdequant.so new file mode 100755 index 00000000..bf2a9fea Binary files /dev/null and b/src/lib/hrx/libdequant.so differ diff --git a/src/lib/hrx/libdequant_new.so b/src/lib/hrx/libdequant_new.so new file mode 100755 index 00000000..4851f5b9 Binary files /dev/null and b/src/lib/hrx/libdequant_new.so differ diff --git a/src/lib/hrx/libgemm.so b/src/lib/hrx/libgemm.so new file mode 100755 index 00000000..1476dac1 Binary files /dev/null and b/src/lib/hrx/libgemm.so differ diff --git a/src/lib/hrx/libgemma4e_npu.so b/src/lib/hrx/libgemma4e_npu.so new file mode 100755 index 00000000..5e04e8f7 Binary files /dev/null and b/src/lib/hrx/libgemma4e_npu.so differ diff --git a/src/lib/hrx/libgemma_embedding.so b/src/lib/hrx/libgemma_embedding.so new file mode 100755 index 00000000..f2ff03e6 Binary files /dev/null and b/src/lib/hrx/libgemma_embedding.so differ diff --git a/src/lib/hrx/libgemma_npu.so b/src/lib/hrx/libgemma_npu.so new file mode 100755 index 00000000..5ad370c6 Binary files /dev/null and b/src/lib/hrx/libgemma_npu.so differ diff --git a/src/lib/hrx/libgemma_text_npu.so b/src/lib/hrx/libgemma_text_npu.so new file mode 100755 index 00000000..a5f482c9 Binary files /dev/null and b/src/lib/hrx/libgemma_text_npu.so differ diff --git a/src/lib/hrx/libgpt_oss_npu.so b/src/lib/hrx/libgpt_oss_npu.so new file mode 100755 index 00000000..e170e18b Binary files /dev/null and b/src/lib/hrx/libgpt_oss_npu.so differ diff --git a/src/lib/hrx/liblfm2_npu.so b/src/lib/hrx/liblfm2_npu.so new file mode 100755 index 00000000..c11ea295 Binary files /dev/null and b/src/lib/hrx/liblfm2_npu.so differ diff --git a/src/lib/hrx/libllama_npu.so b/src/lib/hrx/libllama_npu.so new file mode 100755 index 00000000..c1a39c88 Binary files /dev/null and b/src/lib/hrx/libllama_npu.so differ diff --git a/src/lib/hrx/liblm_head.so b/src/lib/hrx/liblm_head.so new file mode 100755 index 00000000..dd555289 Binary files /dev/null and b/src/lib/hrx/liblm_head.so differ diff --git a/src/lib/hrx/libmha.so b/src/lib/hrx/libmha.so new file mode 100755 index 00000000..01b72b16 Binary files /dev/null and b/src/lib/hrx/libmha.so differ diff --git a/src/lib/hrx/libnanbeige_npu.so b/src/lib/hrx/libnanbeige_npu.so new file mode 100755 index 00000000..26abf96d Binary files /dev/null and b/src/lib/hrx/libnanbeige_npu.so differ diff --git a/src/lib/hrx/libphi4_npu.so b/src/lib/hrx/libphi4_npu.so new file mode 100755 index 00000000..18c46f65 Binary files /dev/null and b/src/lib/hrx/libphi4_npu.so differ diff --git a/src/lib/hrx/libq4_npu_eXpress.so b/src/lib/hrx/libq4_npu_eXpress.so new file mode 100755 index 00000000..9490f777 Binary files /dev/null and b/src/lib/hrx/libq4_npu_eXpress.so differ diff --git a/src/lib/hrx/libqwen2_npu.so b/src/lib/hrx/libqwen2_npu.so new file mode 100755 index 00000000..c165e82f Binary files /dev/null and b/src/lib/hrx/libqwen2_npu.so differ diff --git a/src/lib/hrx/libqwen2vl_npu.so b/src/lib/hrx/libqwen2vl_npu.so new file mode 100755 index 00000000..0a315ff8 Binary files /dev/null and b/src/lib/hrx/libqwen2vl_npu.so differ diff --git a/src/lib/hrx/libqwen3_5_omni_npu.so b/src/lib/hrx/libqwen3_5_omni_npu.so new file mode 100755 index 00000000..6440739d Binary files /dev/null and b/src/lib/hrx/libqwen3_5_omni_npu.so differ diff --git a/src/lib/hrx/libqwen3_5vl_npu.so b/src/lib/hrx/libqwen3_5vl_npu.so new file mode 100755 index 00000000..f509eb76 Binary files /dev/null and b/src/lib/hrx/libqwen3_5vl_npu.so differ diff --git a/src/lib/hrx/libqwen3_6_moe_npu.so b/src/lib/hrx/libqwen3_6_moe_npu.so new file mode 100755 index 00000000..014370a3 Binary files /dev/null and b/src/lib/hrx/libqwen3_6_moe_npu.so differ diff --git a/src/lib/hrx/libqwen3_npu.so b/src/lib/hrx/libqwen3_npu.so new file mode 100755 index 00000000..a87c653e Binary files /dev/null and b/src/lib/hrx/libqwen3_npu.so differ diff --git a/src/lib/hrx/libqwen3vl_npu.so b/src/lib/hrx/libqwen3vl_npu.so new file mode 100755 index 00000000..9e8d5501 Binary files /dev/null and b/src/lib/hrx/libqwen3vl_npu.so differ diff --git a/src/lib/hrx/libwhisper_npu.so b/src/lib/hrx/libwhisper_npu.so new file mode 100755 index 00000000..04159f14 Binary files /dev/null and b/src/lib/hrx/libwhisper_npu.so differ diff --git a/src/lib/hrx/llama_npu.dll b/src/lib/hrx/llama_npu.dll new file mode 100644 index 00000000..dfae8cca Binary files /dev/null and b/src/lib/hrx/llama_npu.dll differ diff --git a/src/lib/hrx/llama_npu.lib b/src/lib/hrx/llama_npu.lib new file mode 100644 index 00000000..81acf502 Binary files /dev/null and b/src/lib/hrx/llama_npu.lib differ diff --git a/src/lib/hrx/lm_head.dll b/src/lib/hrx/lm_head.dll new file mode 100644 index 00000000..e96585a2 Binary files /dev/null and b/src/lib/hrx/lm_head.dll differ diff --git a/src/lib/hrx/lm_head.lib b/src/lib/hrx/lm_head.lib new file mode 100644 index 00000000..f07d5258 Binary files /dev/null and b/src/lib/hrx/lm_head.lib differ diff --git a/src/lib/hrx/mha.dll b/src/lib/hrx/mha.dll new file mode 100644 index 00000000..8a6f6595 Binary files /dev/null and b/src/lib/hrx/mha.dll differ diff --git a/src/lib/hrx/mha.lib b/src/lib/hrx/mha.lib new file mode 100644 index 00000000..98adbe51 Binary files /dev/null and b/src/lib/hrx/mha.lib differ diff --git a/src/lib/hrx/nanbeige_npu.dll b/src/lib/hrx/nanbeige_npu.dll new file mode 100644 index 00000000..3eccb842 Binary files /dev/null and b/src/lib/hrx/nanbeige_npu.dll differ diff --git a/src/lib/hrx/nanbeige_npu.lib b/src/lib/hrx/nanbeige_npu.lib new file mode 100644 index 00000000..91b9fb37 Binary files /dev/null and b/src/lib/hrx/nanbeige_npu.lib differ diff --git a/src/lib/hrx/phi4_npu.dll b/src/lib/hrx/phi4_npu.dll new file mode 100644 index 00000000..5e603716 Binary files /dev/null and b/src/lib/hrx/phi4_npu.dll differ diff --git a/src/lib/hrx/phi4_npu.lib b/src/lib/hrx/phi4_npu.lib new file mode 100644 index 00000000..25e9e803 Binary files /dev/null and b/src/lib/hrx/phi4_npu.lib differ diff --git a/src/lib/hrx/q4_npu_eXpress.dll b/src/lib/hrx/q4_npu_eXpress.dll new file mode 100644 index 00000000..a6f99cc1 Binary files /dev/null and b/src/lib/hrx/q4_npu_eXpress.dll differ diff --git a/src/lib/hrx/q4_npu_eXpress.lib b/src/lib/hrx/q4_npu_eXpress.lib new file mode 100644 index 00000000..f101d2b7 Binary files /dev/null and b/src/lib/hrx/q4_npu_eXpress.lib differ diff --git a/src/lib/hrx/qwen2_npu.dll b/src/lib/hrx/qwen2_npu.dll new file mode 100644 index 00000000..d4bdb73b Binary files /dev/null and b/src/lib/hrx/qwen2_npu.dll differ diff --git a/src/lib/hrx/qwen2_npu.lib b/src/lib/hrx/qwen2_npu.lib new file mode 100644 index 00000000..f731b890 Binary files /dev/null and b/src/lib/hrx/qwen2_npu.lib differ diff --git a/src/lib/hrx/qwen2vl_npu.dll b/src/lib/hrx/qwen2vl_npu.dll new file mode 100644 index 00000000..939f4564 Binary files /dev/null and b/src/lib/hrx/qwen2vl_npu.dll differ diff --git a/src/lib/hrx/qwen2vl_npu.lib b/src/lib/hrx/qwen2vl_npu.lib new file mode 100644 index 00000000..deb8cbee Binary files /dev/null and b/src/lib/hrx/qwen2vl_npu.lib differ diff --git a/src/lib/hrx/qwen3_5_omni_npu.dll b/src/lib/hrx/qwen3_5_omni_npu.dll new file mode 100644 index 00000000..7a2c372a Binary files /dev/null and b/src/lib/hrx/qwen3_5_omni_npu.dll differ diff --git a/src/lib/hrx/qwen3_5_omni_npu.lib b/src/lib/hrx/qwen3_5_omni_npu.lib new file mode 100644 index 00000000..7ef7e232 Binary files /dev/null and b/src/lib/hrx/qwen3_5_omni_npu.lib differ diff --git a/src/lib/hrx/qwen3_5vl_npu.dll b/src/lib/hrx/qwen3_5vl_npu.dll new file mode 100644 index 00000000..82a3f572 Binary files /dev/null and b/src/lib/hrx/qwen3_5vl_npu.dll differ diff --git a/src/lib/hrx/qwen3_5vl_npu.lib b/src/lib/hrx/qwen3_5vl_npu.lib new file mode 100644 index 00000000..a99e0a79 Binary files /dev/null and b/src/lib/hrx/qwen3_5vl_npu.lib differ diff --git a/src/lib/hrx/qwen3_6_moe_npu.dll b/src/lib/hrx/qwen3_6_moe_npu.dll new file mode 100644 index 00000000..997b62df Binary files /dev/null and b/src/lib/hrx/qwen3_6_moe_npu.dll differ diff --git a/src/lib/hrx/qwen3_6_moe_npu.lib b/src/lib/hrx/qwen3_6_moe_npu.lib new file mode 100644 index 00000000..1e41ddad Binary files /dev/null and b/src/lib/hrx/qwen3_6_moe_npu.lib differ diff --git a/src/lib/hrx/qwen3_npu.dll b/src/lib/hrx/qwen3_npu.dll new file mode 100644 index 00000000..7d4de7ec Binary files /dev/null and b/src/lib/hrx/qwen3_npu.dll differ diff --git a/src/lib/hrx/qwen3_npu.lib b/src/lib/hrx/qwen3_npu.lib new file mode 100644 index 00000000..059a5c97 Binary files /dev/null and b/src/lib/hrx/qwen3_npu.lib differ diff --git a/src/lib/hrx/qwen3vl_npu.dll b/src/lib/hrx/qwen3vl_npu.dll new file mode 100644 index 00000000..6cbaf66b Binary files /dev/null and b/src/lib/hrx/qwen3vl_npu.dll differ diff --git a/src/lib/hrx/qwen3vl_npu.lib b/src/lib/hrx/qwen3vl_npu.lib new file mode 100644 index 00000000..8769f7b2 Binary files /dev/null and b/src/lib/hrx/qwen3vl_npu.lib differ diff --git a/src/lib/hrx/whisper_npu.dll b/src/lib/hrx/whisper_npu.dll new file mode 100644 index 00000000..07f5bd6c Binary files /dev/null and b/src/lib/hrx/whisper_npu.dll differ diff --git a/src/lib/hrx/whisper_npu.lib b/src/lib/hrx/whisper_npu.lib new file mode 100644 index 00000000..9217433b Binary files /dev/null and b/src/lib/hrx/whisper_npu.lib differ diff --git a/src/lib/aiebu_static.lib b/src/lib/xrt/aiebu_static.lib similarity index 100% rename from src/lib/aiebu_static.lib rename to src/lib/xrt/aiebu_static.lib diff --git a/src/lib/dequant.dll b/src/lib/xrt/dequant.dll similarity index 100% rename from src/lib/dequant.dll rename to src/lib/xrt/dequant.dll diff --git a/src/lib/dequant.lib b/src/lib/xrt/dequant.lib similarity index 100% rename from src/lib/dequant.lib rename to src/lib/xrt/dequant.lib diff --git a/src/lib/dequant_new.dll b/src/lib/xrt/dequant_new.dll similarity index 100% rename from src/lib/dequant_new.dll rename to src/lib/xrt/dequant_new.dll diff --git a/src/lib/dequant_new.lib b/src/lib/xrt/dequant_new.lib similarity index 100% rename from src/lib/dequant_new.lib rename to src/lib/xrt/dequant_new.lib diff --git a/src/lib/gemm.dll b/src/lib/xrt/gemm.dll similarity index 100% rename from src/lib/gemm.dll rename to src/lib/xrt/gemm.dll diff --git a/src/lib/gemm.lib b/src/lib/xrt/gemm.lib similarity index 100% rename from src/lib/gemm.lib rename to src/lib/xrt/gemm.lib diff --git a/src/lib/gemma4e_npu.dll b/src/lib/xrt/gemma4e_npu.dll similarity index 100% rename from src/lib/gemma4e_npu.dll rename to src/lib/xrt/gemma4e_npu.dll diff --git a/src/lib/gemma4e_npu.lib b/src/lib/xrt/gemma4e_npu.lib similarity index 100% rename from src/lib/gemma4e_npu.lib rename to src/lib/xrt/gemma4e_npu.lib diff --git a/src/lib/gemma_embedding.dll b/src/lib/xrt/gemma_embedding.dll similarity index 100% rename from src/lib/gemma_embedding.dll rename to src/lib/xrt/gemma_embedding.dll diff --git a/src/lib/gemma_embedding.lib b/src/lib/xrt/gemma_embedding.lib similarity index 100% rename from src/lib/gemma_embedding.lib rename to src/lib/xrt/gemma_embedding.lib diff --git a/src/lib/gemma_npu.dll b/src/lib/xrt/gemma_npu.dll similarity index 100% rename from src/lib/gemma_npu.dll rename to src/lib/xrt/gemma_npu.dll diff --git a/src/lib/gemma_npu.lib b/src/lib/xrt/gemma_npu.lib similarity index 100% rename from src/lib/gemma_npu.lib rename to src/lib/xrt/gemma_npu.lib diff --git a/src/lib/gemma_text_npu.dll b/src/lib/xrt/gemma_text_npu.dll similarity index 100% rename from src/lib/gemma_text_npu.dll rename to src/lib/xrt/gemma_text_npu.dll diff --git a/src/lib/gemma_text_npu.lib b/src/lib/xrt/gemma_text_npu.lib similarity index 100% rename from src/lib/gemma_text_npu.lib rename to src/lib/xrt/gemma_text_npu.lib diff --git a/src/lib/gpt_oss_npu.dll b/src/lib/xrt/gpt_oss_npu.dll similarity index 100% rename from src/lib/gpt_oss_npu.dll rename to src/lib/xrt/gpt_oss_npu.dll diff --git a/src/lib/gpt_oss_npu.lib b/src/lib/xrt/gpt_oss_npu.lib similarity index 100% rename from src/lib/gpt_oss_npu.lib rename to src/lib/xrt/gpt_oss_npu.lib diff --git a/src/lib/lfm2_npu.dll b/src/lib/xrt/lfm2_npu.dll similarity index 100% rename from src/lib/lfm2_npu.dll rename to src/lib/xrt/lfm2_npu.dll diff --git a/src/lib/lfm2_npu.lib b/src/lib/xrt/lfm2_npu.lib similarity index 100% rename from src/lib/lfm2_npu.lib rename to src/lib/xrt/lfm2_npu.lib diff --git a/src/lib/libaiebu.a b/src/lib/xrt/libaiebu.a similarity index 100% rename from src/lib/libaiebu.a rename to src/lib/xrt/libaiebu.a diff --git a/src/lib/libdequant.so b/src/lib/xrt/libdequant.so similarity index 100% rename from src/lib/libdequant.so rename to src/lib/xrt/libdequant.so diff --git a/src/lib/libdequant_new.so b/src/lib/xrt/libdequant_new.so similarity index 100% rename from src/lib/libdequant_new.so rename to src/lib/xrt/libdequant_new.so diff --git a/src/lib/libgemm.so b/src/lib/xrt/libgemm.so similarity index 100% rename from src/lib/libgemm.so rename to src/lib/xrt/libgemm.so diff --git a/src/lib/libgemma4e_npu.so b/src/lib/xrt/libgemma4e_npu.so similarity index 100% rename from src/lib/libgemma4e_npu.so rename to src/lib/xrt/libgemma4e_npu.so diff --git a/src/lib/libgemma_embedding.so b/src/lib/xrt/libgemma_embedding.so similarity index 100% rename from src/lib/libgemma_embedding.so rename to src/lib/xrt/libgemma_embedding.so diff --git a/src/lib/libgemma_npu.so b/src/lib/xrt/libgemma_npu.so similarity index 100% rename from src/lib/libgemma_npu.so rename to src/lib/xrt/libgemma_npu.so diff --git a/src/lib/libgemma_text_npu.so b/src/lib/xrt/libgemma_text_npu.so similarity index 100% rename from src/lib/libgemma_text_npu.so rename to src/lib/xrt/libgemma_text_npu.so diff --git a/src/lib/libgpt_oss_npu.so b/src/lib/xrt/libgpt_oss_npu.so similarity index 100% rename from src/lib/libgpt_oss_npu.so rename to src/lib/xrt/libgpt_oss_npu.so diff --git a/src/lib/liblfm2_npu.so b/src/lib/xrt/liblfm2_npu.so similarity index 100% rename from src/lib/liblfm2_npu.so rename to src/lib/xrt/liblfm2_npu.so diff --git a/src/lib/libllama_npu.so b/src/lib/xrt/libllama_npu.so similarity index 100% rename from src/lib/libllama_npu.so rename to src/lib/xrt/libllama_npu.so diff --git a/src/lib/liblm_head.so b/src/lib/xrt/liblm_head.so similarity index 100% rename from src/lib/liblm_head.so rename to src/lib/xrt/liblm_head.so diff --git a/src/lib/libmha.so b/src/lib/xrt/libmha.so similarity index 100% rename from src/lib/libmha.so rename to src/lib/xrt/libmha.so diff --git a/src/lib/libnanbeige_npu.so b/src/lib/xrt/libnanbeige_npu.so similarity index 100% rename from src/lib/libnanbeige_npu.so rename to src/lib/xrt/libnanbeige_npu.so diff --git a/src/lib/libphi4_npu.so b/src/lib/xrt/libphi4_npu.so similarity index 100% rename from src/lib/libphi4_npu.so rename to src/lib/xrt/libphi4_npu.so diff --git a/src/lib/libq4_npu_eXpress.so b/src/lib/xrt/libq4_npu_eXpress.so similarity index 100% rename from src/lib/libq4_npu_eXpress.so rename to src/lib/xrt/libq4_npu_eXpress.so diff --git a/src/lib/libqwen2_npu.so b/src/lib/xrt/libqwen2_npu.so similarity index 100% rename from src/lib/libqwen2_npu.so rename to src/lib/xrt/libqwen2_npu.so diff --git a/src/lib/libqwen2vl_npu.so b/src/lib/xrt/libqwen2vl_npu.so similarity index 100% rename from src/lib/libqwen2vl_npu.so rename to src/lib/xrt/libqwen2vl_npu.so diff --git a/src/lib/libqwen3_5_omni_npu.so b/src/lib/xrt/libqwen3_5_omni_npu.so similarity index 100% rename from src/lib/libqwen3_5_omni_npu.so rename to src/lib/xrt/libqwen3_5_omni_npu.so diff --git a/src/lib/libqwen3_5vl_npu.so b/src/lib/xrt/libqwen3_5vl_npu.so similarity index 100% rename from src/lib/libqwen3_5vl_npu.so rename to src/lib/xrt/libqwen3_5vl_npu.so diff --git a/src/lib/libqwen3_6_moe_npu.so b/src/lib/xrt/libqwen3_6_moe_npu.so similarity index 100% rename from src/lib/libqwen3_6_moe_npu.so rename to src/lib/xrt/libqwen3_6_moe_npu.so diff --git a/src/lib/libqwen3_npu.so b/src/lib/xrt/libqwen3_npu.so similarity index 100% rename from src/lib/libqwen3_npu.so rename to src/lib/xrt/libqwen3_npu.so diff --git a/src/lib/libqwen3vl_npu.so b/src/lib/xrt/libqwen3vl_npu.so similarity index 100% rename from src/lib/libqwen3vl_npu.so rename to src/lib/xrt/libqwen3vl_npu.so diff --git a/src/lib/libwhisper_npu.so b/src/lib/xrt/libwhisper_npu.so similarity index 100% rename from src/lib/libwhisper_npu.so rename to src/lib/xrt/libwhisper_npu.so diff --git a/src/lib/llama_npu.dll b/src/lib/xrt/llama_npu.dll similarity index 100% rename from src/lib/llama_npu.dll rename to src/lib/xrt/llama_npu.dll diff --git a/src/lib/llama_npu.lib b/src/lib/xrt/llama_npu.lib similarity index 100% rename from src/lib/llama_npu.lib rename to src/lib/xrt/llama_npu.lib diff --git a/src/lib/lm_head.dll b/src/lib/xrt/lm_head.dll similarity index 100% rename from src/lib/lm_head.dll rename to src/lib/xrt/lm_head.dll diff --git a/src/lib/lm_head.lib b/src/lib/xrt/lm_head.lib similarity index 100% rename from src/lib/lm_head.lib rename to src/lib/xrt/lm_head.lib diff --git a/src/lib/mha.dll b/src/lib/xrt/mha.dll similarity index 100% rename from src/lib/mha.dll rename to src/lib/xrt/mha.dll diff --git a/src/lib/mha.lib b/src/lib/xrt/mha.lib similarity index 100% rename from src/lib/mha.lib rename to src/lib/xrt/mha.lib diff --git a/src/lib/nanbeige_npu.dll b/src/lib/xrt/nanbeige_npu.dll similarity index 100% rename from src/lib/nanbeige_npu.dll rename to src/lib/xrt/nanbeige_npu.dll diff --git a/src/lib/nanbeige_npu.lib b/src/lib/xrt/nanbeige_npu.lib similarity index 100% rename from src/lib/nanbeige_npu.lib rename to src/lib/xrt/nanbeige_npu.lib diff --git a/src/lib/phi4_npu.dll b/src/lib/xrt/phi4_npu.dll similarity index 100% rename from src/lib/phi4_npu.dll rename to src/lib/xrt/phi4_npu.dll diff --git a/src/lib/phi4_npu.lib b/src/lib/xrt/phi4_npu.lib similarity index 100% rename from src/lib/phi4_npu.lib rename to src/lib/xrt/phi4_npu.lib diff --git a/src/lib/q4_npu_eXpress.dll b/src/lib/xrt/q4_npu_eXpress.dll similarity index 100% rename from src/lib/q4_npu_eXpress.dll rename to src/lib/xrt/q4_npu_eXpress.dll diff --git a/src/lib/q4_npu_eXpress.lib b/src/lib/xrt/q4_npu_eXpress.lib similarity index 100% rename from src/lib/q4_npu_eXpress.lib rename to src/lib/xrt/q4_npu_eXpress.lib diff --git a/src/lib/qwen2_npu.dll b/src/lib/xrt/qwen2_npu.dll similarity index 100% rename from src/lib/qwen2_npu.dll rename to src/lib/xrt/qwen2_npu.dll diff --git a/src/lib/qwen2_npu.lib b/src/lib/xrt/qwen2_npu.lib similarity index 100% rename from src/lib/qwen2_npu.lib rename to src/lib/xrt/qwen2_npu.lib diff --git a/src/lib/qwen2vl_npu.dll b/src/lib/xrt/qwen2vl_npu.dll similarity index 100% rename from src/lib/qwen2vl_npu.dll rename to src/lib/xrt/qwen2vl_npu.dll diff --git a/src/lib/qwen2vl_npu.lib b/src/lib/xrt/qwen2vl_npu.lib similarity index 100% rename from src/lib/qwen2vl_npu.lib rename to src/lib/xrt/qwen2vl_npu.lib diff --git a/src/lib/qwen3_5_omni_npu.dll b/src/lib/xrt/qwen3_5_omni_npu.dll similarity index 100% rename from src/lib/qwen3_5_omni_npu.dll rename to src/lib/xrt/qwen3_5_omni_npu.dll diff --git a/src/lib/qwen3_5_omni_npu.lib b/src/lib/xrt/qwen3_5_omni_npu.lib similarity index 100% rename from src/lib/qwen3_5_omni_npu.lib rename to src/lib/xrt/qwen3_5_omni_npu.lib diff --git a/src/lib/qwen3_5vl_npu.dll b/src/lib/xrt/qwen3_5vl_npu.dll similarity index 100% rename from src/lib/qwen3_5vl_npu.dll rename to src/lib/xrt/qwen3_5vl_npu.dll diff --git a/src/lib/qwen3_5vl_npu.lib b/src/lib/xrt/qwen3_5vl_npu.lib similarity index 100% rename from src/lib/qwen3_5vl_npu.lib rename to src/lib/xrt/qwen3_5vl_npu.lib diff --git a/src/lib/qwen3_6_moe_npu.dll b/src/lib/xrt/qwen3_6_moe_npu.dll similarity index 100% rename from src/lib/qwen3_6_moe_npu.dll rename to src/lib/xrt/qwen3_6_moe_npu.dll diff --git a/src/lib/qwen3_6_moe_npu.lib b/src/lib/xrt/qwen3_6_moe_npu.lib similarity index 100% rename from src/lib/qwen3_6_moe_npu.lib rename to src/lib/xrt/qwen3_6_moe_npu.lib diff --git a/src/lib/qwen3_npu.dll b/src/lib/xrt/qwen3_npu.dll similarity index 100% rename from src/lib/qwen3_npu.dll rename to src/lib/xrt/qwen3_npu.dll diff --git a/src/lib/qwen3_npu.lib b/src/lib/xrt/qwen3_npu.lib similarity index 100% rename from src/lib/qwen3_npu.lib rename to src/lib/xrt/qwen3_npu.lib diff --git a/src/lib/qwen3vl_npu.dll b/src/lib/xrt/qwen3vl_npu.dll similarity index 100% rename from src/lib/qwen3vl_npu.dll rename to src/lib/xrt/qwen3vl_npu.dll diff --git a/src/lib/qwen3vl_npu.lib b/src/lib/xrt/qwen3vl_npu.lib similarity index 100% rename from src/lib/qwen3vl_npu.lib rename to src/lib/xrt/qwen3vl_npu.lib diff --git a/src/lib/whisper_npu.dll b/src/lib/xrt/whisper_npu.dll similarity index 100% rename from src/lib/whisper_npu.dll rename to src/lib/xrt/whisper_npu.dll diff --git a/src/lib/whisper_npu.lib b/src/lib/xrt/whisper_npu.lib similarity index 100% rename from src/lib/whisper_npu.lib rename to src/lib/xrt/whisper_npu.lib diff --git a/src/pull/download_model.cpp b/src/pull/download_model.cpp index 30e434ec..38ccf44a 100644 --- a/src/pull/download_model.cpp +++ b/src/pull/download_model.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "utils/utils.hpp" #include "nlohmann/json.hpp" #include "picosha2.h" diff --git a/src/runner/runner.cpp b/src/runner/runner.cpp index 2be1495a..d6027f8e 100644 --- a/src/runner/runner.cpp +++ b/src/runner/runner.cpp @@ -18,6 +18,8 @@ #include #include #include +#include +#include /// \brief Command map for command line input std::map cmd_map = { @@ -39,7 +41,7 @@ std::map cmd_map = { Runner::Runner(model_list& supported_models, ModelDownloader& downloader, program_args_t& args) : supported_models(supported_models), downloader(downloader), tag(args.model_tag), modelscope(args.modelscope), asr(args.asr), embed(args.embed), img_pre_resize(args.img_pre_resize), preemption(args.preemption) { - this->npu_device_inst = xrt::device(0); + this->npu_device_inst = flm_rt::device(0); if (args.ctx_length != -1) { this->ctx_length = args.ctx_length >= 512 ? args.ctx_length : 512; diff --git a/src/runner/runner.hpp b/src/runner/runner.hpp index c88c0312..0e5459cb 100644 --- a/src/runner/runner.hpp +++ b/src/runner/runner.hpp @@ -67,7 +67,7 @@ class Runner { int img_pre_resize; // CLI instance for interactive input CLIWide cli; - xrt::device npu_device_inst; + flm_rt::device npu_device_inst; /// \brief Command functions void cmd_set(std::vector& input_list); diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 26540966..a135d5ad 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -323,7 +323,7 @@ static json convert_tool_responses_gemma4(json messages) { ///@return the rest handler RestHandler::RestHandler(model_list& models, ModelDownloader& downloader, program_args_t& args) : supported_models(models), downloader(downloader), default_model_tag(args.model_tag), current_model_tag(""), modelscope(args.modelscope), asr(args.asr), embed(args.embed), img_pre_resize(args.img_pre_resize), preemption(args.preemption){ - this->npu_device_inst = xrt::device(0); + this->npu_device_inst = flm_rt::device(0); if (args.ctx_length != -1) { this->ctx_length = args.ctx_length >= 512 ? args.ctx_length : 512; @@ -399,7 +399,7 @@ bool RestHandler::ensure_model_loaded(const std::string& model_tag) { header_print("ERROR", "Failed to load model: " + std::string(e.what())); this->auto_chat_engine.reset(); this->npu_device_inst.reset(); - this->npu_device_inst = xrt::device(0); + this->npu_device_inst = flm_rt::device(0); this->current_model_tag = "model-faker"; return false; } diff --git a/src/server/rest_handler.hpp b/src/server/rest_handler.hpp index 4438d253..40f7e948 100644 --- a/src/server/rest_handler.hpp +++ b/src/server/rest_handler.hpp @@ -120,7 +120,7 @@ class RestHandler { std::unique_ptr whisper_engine; std::unique_ptr auto_embedding_engine; #endif - xrt::device npu_device_inst; + flm_rt::device npu_device_inst; model_list& supported_models; ModelDownloader& downloader; std::string current_model_tag; diff --git a/src/src/benchmarking.hpp b/src/src/benchmarking.hpp index a41016f6..27f4626a 100644 --- a/src/src/benchmarking.hpp +++ b/src/src/benchmarking.hpp @@ -267,7 +267,7 @@ BenchmarkResults_t run_benchmarks(std::string model_tag, std::string bench_confi input_file.close(); } - xrt::device npu_device_inst = xrt::device(0); + flm_rt::device npu_device_inst = flm_rt::device(0); std::unique_ptr auto_chat_engine; if (!availble_models.is_model_supported(model_tag)) { header_print_r("ERROR", "Model not found: " << model_tag << "; Please check with `flm list` and try again."); diff --git a/src/src/main.cpp b/src/src/main.cpp index 1ad70f55..f5ee1ad0 100644 --- a/src/src/main.cpp +++ b/src/src/main.cpp @@ -59,10 +59,11 @@ std::atomic should_exit(false); std::mutex exit_mutex; std::condition_variable exit_cv; -#ifndef _WIN32 +#if !defined(FLM_USE_HRX) && !defined(_WIN32) ///@brief Preload critical XRT libraries from the executable directory ///@details This ensures that dlopen() calls within libraries find the bundled versions -///@note Only on Linux/Unix; Windows handles DLL loading differently +///@note Only on Linux/Unix for the XRT backend; Windows handles DLL loading +/// differently and the HRX backend has no equivalent preload requirement. void preload_bundled_libraries() { std::string exe_dir = utils::get_executable_directory(); @@ -463,8 +464,8 @@ int main(int argc, char* argv[]) { #ifdef _WIN32 SetConsoleOutputCP(CP_UTF8); SetConsoleCP(CP_UTF8); -#else - // Preload bundled libraries from executable directory +#elif !defined(FLM_USE_HRX) + // XRT backend: preload bundled XRT libraries from the executable directory. preload_bundled_libraries(); #endif diff --git a/src/test/common.mk b/src/test/common.mk index fb53068a..12fddcd1 100644 --- a/src/test/common.mk +++ b/src/test/common.mk @@ -10,7 +10,13 @@ endif CURRENT_DIR := $(notdir $(CURDIR)) BUILD_DIR := ../../build/test/$(CURRENT_DIR) -LIB_DIR := ../../lib + +# ———————————————————————————————————————————————————————————————— +# NPU runtime backend selection (mirrors the CMake FLM_USE_HRX flag). +# FLM_USE_HRX=0 -> XRT (default) FLM_USE_HRX=1 -> HRX +# Prebuilt engine libs are consumed from the matching per-backend subdir. +# ———————————————————————————————————————————————————————————————— +FLM_USE_HRX ?= 0 ifeq ($(WSL), 0) # Linux build environment @@ -21,7 +27,6 @@ CXX_FLAGS := -std=c++20 -fPIC -Wall -DUSEAVX2=1 CXX_FLAGS += -mavx2 -mfma -march=native -ffast-math CXX_FLAGS += -fmax-errors=1 CXX_FLAGS += -I../../include -CXX_FLAGS += -I/opt/xilinx/xrt/include CXX_FLAGS += -MMD -MP CXX_FLAGS += -DDEV_BUILD CXX_FLAGS += -fopenmp @@ -32,14 +37,44 @@ CXX_FLAGS += -D__FLM_VERSION__="\"0.9.34\"" CXX_FLAGS += -D__NPU_VERSION__="\"32.0.203.304\"" CXX_FLAGS += -DDISABLE_ABI_CHECK=1 -LDFLAGS += -L/opt/xilinx/xrt/lib -LDFLAGS += -Wl,-rpath,/opt/xilinx/xrt/lib -LDFLAGS += -lxrt_coreutil +ifeq ($(FLM_USE_HRX),1) +# --- HRX backend --- +# Consumed from the pinned release artifact fetched by +# hrx-integration/fetch-hrx-release.sh (pin: hrx-integration/hrx-release.env). +# The artifact's env.sh exports HRX_DIR / HRX_BUILD; source it before building +# or override HRX_DIR / HRX_BUILD on the make command line. +LIB_DIR := ../../lib/hrx +HRX_MK_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +REPO_ROOT := $(abspath $(HRX_MK_DIR)/../..) +HRX_RELEASE_DIR ?= $(REPO_ROOT)/hrx-integration/.hrx-release +HRX_ARTIFACT_ROOT ?= $(firstword $(wildcard $(HRX_RELEASE_DIR)/hrx-amdxdna-*/)) +HRX_DIR ?= $(HRX_ARTIFACT_ROOT)HRX_DIR +HRX_BUILD ?= $(HRX_ARTIFACT_ROOT)HRX_BUILD +ifeq ($(wildcard $(HRX_BUILD)/libhrx/src/libhrx/libhrx.so),) +$(error HRX artifact not found at "$(HRX_BUILD)". Run hrx-integration/fetch-hrx-release.sh (or `source /env.sh`) before building, or override HRX_DIR/HRX_BUILD on the make command line.) +endif +HRX_INC := -I$(HRX_DIR)/libhrx/include -I$(HRX_DIR)/runtime/src \ + -I$(HRX_BUILD)/runtime/src -I$(HRX_BUILD)/_deps/flatcc-src/include +HRX_LIBS := $(HRX_BUILD)/libhrx/src/libhrx/libhrx.so $(HRX_BUILD)/libflatcc_runtime.a +HRX_RPATH := -Wl,-rpath,$(HRX_BUILD)/libhrx/src/libhrx +CXX_FLAGS += -DFLM_USE_HRX=1 $(HRX_INC) +RUNTIME_LDFLAGS := $(HRX_RPATH) $(HRX_LIBS) +else +# --- XRT backend (default) --- +LIB_DIR := ../../lib/xrt +XRT_PREFIX ?= /opt/xilinx/xrt +CXX_FLAGS += -I$(XRT_PREFIX)/include +RUNTIME_LDFLAGS := -L$(XRT_PREFIX)/lib -Wl,-rpath,$(XRT_PREFIX)/lib -lxrt_coreutil -laiebu +endif + LDFLAGS += -lboost_program_options -lboost_filesystem LDFLAGS += -L$(LIB_DIR) +# Resolve the prebuilt engine .so (and bundled aiebu) at run time. +LDFLAGS += -Wl,-rpath,$(abspath $(LIB_DIR)) LDFLAGS += -L../../build/tokenizers-cpp LDFLAGS += -L../../build/tokenizers-cpp/sentencepiece/src -LDFLAGS += -ltokenizers_cpp -ltokenizers_c -lsentencepiece -laiebu +LDFLAGS += -ltokenizers_cpp -ltokenizers_c -lsentencepiece +LDFLAGS += $(RUNTIME_LDFLAGS) DEPENDENCY_LDFLAGS += -lmha -ldequant -lgemm -llm_head -lq4_npu_eXpress diff --git a/src/test/gemma4e_npu/test.cpp b/src/test/gemma4e_npu/test.cpp index fcc541c2..f668ea2d 100644 --- a/src/test/gemma4e_npu/test.cpp +++ b/src/test/gemma4e_npu/test.cpp @@ -5,7 +5,7 @@ #include "AutoModel/modeling_gemma4e.hpp" #include "model_list.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; // // Model-specific factory function for Qwen family and DeepSeek_r1_0528_8b // inline std::pair> get_qwen3_5vl_model(const std::string& model_tag) { @@ -59,8 +59,8 @@ int main(int argc, char* argv[]) { std::unique_ptr chat = std::make_unique(&npu_device_global); std::cout <<"Chat model initialized" << std::endl; - npu_device_global = xrt::device(0); - std::cout << "NPU Device initialized: " << npu_device_global.get_info() << std::endl; + npu_device_global = flm_rt::device(0); + std::cout << "NPU Device initialized: " << npu_device_global.get_info() << std::endl; chat->load_model(model_path, model_info, -1, preemption); header_print("info", "Model loaded"); chat_meta_info_t meta_info; diff --git a/src/test/gemma_embedding/test.cpp b/src/test/gemma_embedding/test.cpp index 3a201ac4..0c61fdad 100644 --- a/src/test/gemma_embedding/test.cpp +++ b/src/test/gemma_embedding/test.cpp @@ -58,7 +58,7 @@ static float reference[] = { 4.077148e-02, 1.806641e-02, 1.843262e-02, -2.661133e-02, 7.446289e-03, 3.768921e-03, 5.566406e-02, -4.687500e-02, -4.174805e-02, 1.531982e-02, 9.765625e-03, 2.343750e-02, 2.905273e-02, 2.246094e-02, 2.160645e-02, 5.859375e-03 }; // Model-specific factory function for Gemma3 Text family only -inline std::pair> get_gemma_embedding_model(const std::string& model_tag, xrt::device* npu_device_inst) { +inline std::pair> get_gemma_embedding_model(const std::string& model_tag, flm_rt::device* npu_device_inst) { static std::unordered_set gemma_embedding_Tags = { "embed-gemma", "embed-gemma:300m" }; @@ -101,7 +101,7 @@ int main(int argc, char* argv[]) { std::cout << "Model path: " << model_path << std::endl; std::cout << "Model info: " << model_info.dump() << std::endl; - auto npu_device_global = xrt::device(0); + auto npu_device_global = flm_rt::device(0); // Use model-specific factory std::unique_ptr embedding = std::make_unique(&npu_device_global); diff --git a/src/test/gemma_npu/test.cpp b/src/test/gemma_npu/test.cpp index 5b59b52b..dadccde8 100644 --- a/src/test/gemma_npu/test.cpp +++ b/src/test/gemma_npu/test.cpp @@ -5,7 +5,7 @@ #include "AutoModel/modeling_gemma3.hpp" #include "model_list.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; int main(int argc, char* argv[]) { #ifdef __WINDOWS__ @@ -39,7 +39,7 @@ int main(int argc, char* argv[]) { std::cout << "Model info" << model_info.dump(4) << std::endl; - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); std::unique_ptr chat = std::make_unique(&npu_device_global); chat->load_model(model_path, model_info, -1, preemption); diff --git a/src/test/gemma_text_npu/test.cpp b/src/test/gemma_text_npu/test.cpp index a46be828..a6cee06a 100644 --- a/src/test/gemma_text_npu/test.cpp +++ b/src/test/gemma_text_npu/test.cpp @@ -5,9 +5,9 @@ #include "AutoModel/modeling_gemma3_text.hpp" #include "model_list.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; // Model-specific factory function for Gemma3 Text family only -inline std::pair> get_gemma_text_model(const std::string& model_tag, xrt::device* npu_device_inst) { +inline std::pair> get_gemma_text_model(const std::string& model_tag, flm_rt::device* npu_device_inst) { static std::unordered_set gemma3_text_Tags = { "gemma3", "gemma3:270m", "gemma3:1b" }; @@ -55,7 +55,7 @@ int main(int argc, char* argv[]) { std::cout << "Model info" << model_info.dump(4) << std::endl; - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); std::unique_ptr chat = std::make_unique(&npu_device_global); chat->load_model(model_path, model_info, -1, preemption); diff --git a/src/test/gpt_oss_npu/test.cpp b/src/test/gpt_oss_npu/test.cpp index 8035cd56..c29d0b25 100644 --- a/src/test/gpt_oss_npu/test.cpp +++ b/src/test/gpt_oss_npu/test.cpp @@ -10,7 +10,7 @@ #include "model_list.hpp" #include "utils/vm_args.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; int main(int argc, char* argv[]) { @@ -46,7 +46,7 @@ int main(int argc, char* argv[]) { std::cout << "Model path: " << model_path << std::endl; std::unique_ptr chat = std::make_unique(&npu_device_global); - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); try { chat->load_model(model_path, model_info, -1, preemption); diff --git a/src/test/lfm2_npu/test.cpp b/src/test/lfm2_npu/test.cpp index f3b2ecb1..cbc33ac2 100644 --- a/src/test/lfm2_npu/test.cpp +++ b/src/test/lfm2_npu/test.cpp @@ -8,7 +8,7 @@ #include "utils/vm_args.hpp" #include "metrices.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; // Model-specific factory function for Llama family and DeepSeek_r1_8b inline std::pair> get_lfm2_model(const std::string& model_tag) { @@ -65,7 +65,7 @@ int main(int argc, char* argv[]) { std::unique_ptr chat = std::make_unique(&npu_device_global); - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); chat->load_model(model_path, model_info, -1, preemption); chat->set_topk(1); diff --git a/src/test/llama_npu/test.cpp b/src/test/llama_npu/test.cpp index d5752bdc..cf2959ba 100644 --- a/src/test/llama_npu/test.cpp +++ b/src/test/llama_npu/test.cpp @@ -10,7 +10,7 @@ #include "model_list.hpp" #include "utils/vm_args.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; int main(int argc, char* argv[]) { #ifdef __WINDOWS__ @@ -49,7 +49,7 @@ int main(int argc, char* argv[]) { if (model_info_pair.first.find("llama3") != std::string::npos) { chat = std::make_unique(&npu_device_global); } - else if (model_info_pair.first.find("deepseek-r1-8b") != std::string::npos) { + else if (model_info_pair.first == "deepseek-r1:8b") { chat = std::make_unique(&npu_device_global); } else { @@ -57,7 +57,7 @@ int main(int argc, char* argv[]) { return 1; } - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); chat->load_model(model_path, model_info, -1, preemption); chat_meta_info_t meta_info; diff --git a/src/test/nanbeige_npu/test.cpp b/src/test/nanbeige_npu/test.cpp index e8243734..e2a5e46f 100644 --- a/src/test/nanbeige_npu/test.cpp +++ b/src/test/nanbeige_npu/test.cpp @@ -10,7 +10,7 @@ #include "model_list.hpp" #include "utils/vm_args.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; int main(int argc, char* argv[]) { #ifdef __WINDOWS__ @@ -54,7 +54,7 @@ int main(int argc, char* argv[]) { return 1; } - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); chat->load_model(model_path, model_info, -1, preemption); chat_meta_info_t meta_info; diff --git a/src/test/phi4_npu/test.cpp b/src/test/phi4_npu/test.cpp index 7e5ff98e..e7e76d2e 100644 --- a/src/test/phi4_npu/test.cpp +++ b/src/test/phi4_npu/test.cpp @@ -11,7 +11,7 @@ #include "utils/vm_args.hpp" #include "metrices.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; @@ -46,7 +46,7 @@ int main(int argc, char* argv[]) { std::pair model_info_pair = model_list.get_model_info(tag); nlohmann::json model_info = model_info_pair.second; std::cout << "Model path: " << model_path << std::endl; - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); std::unique_ptr chat = std::make_unique(&npu_device_global); diff --git a/src/test/qwen2_npu/test.cpp b/src/test/qwen2_npu/test.cpp index 3d948177..68f4b3e0 100644 --- a/src/test/qwen2_npu/test.cpp +++ b/src/test/qwen2_npu/test.cpp @@ -5,7 +5,7 @@ #include "AutoModel/modeling_qwen2.hpp" #include "model_list.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; int main(int argc, char* argv[]) { #ifdef __WINDOWS__ @@ -38,7 +38,7 @@ int main(int argc, char* argv[]) { std::pair model_info_pair = model_list.get_model_info(tag); nlohmann::json model_info = model_info_pair.second; std::cout << "Model path: " << model_path << std::endl; - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); std::unique_ptr chat = std::make_unique(&npu_device_global); diff --git a/src/test/qwen2vl_npu/test.cpp b/src/test/qwen2vl_npu/test.cpp index 9526bcb3..9f0313bd 100644 --- a/src/test/qwen2vl_npu/test.cpp +++ b/src/test/qwen2vl_npu/test.cpp @@ -5,7 +5,7 @@ #include "AutoModel/modeling_qwen2vl.hpp" #include "model_list.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; int main(int argc, char* argv[]) { @@ -42,7 +42,7 @@ int main(int argc, char* argv[]) { std::unique_ptr chat = std::make_unique(&npu_device_global); - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); chat->load_model(model_path, model_info, -1, preemption); chat->set_topk(1); diff --git a/src/test/qwen3_5vl_npu/test.cpp b/src/test/qwen3_5vl_npu/test.cpp index d69fb011..7511f6f6 100644 --- a/src/test/qwen3_5vl_npu/test.cpp +++ b/src/test/qwen3_5vl_npu/test.cpp @@ -5,7 +5,7 @@ #include "AutoModel/modeling_qwen3_5vl.hpp" #include "model_list.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; // Model-specific factory function for Qwen family and DeepSeek_r1_0528_8b inline std::pair> get_qwen3_5vl_model(const std::string& model_tag) { @@ -58,7 +58,7 @@ int main(int argc, char* argv[]) { std::cout << "Model path: " << model_path << std::endl; std::unique_ptr chat = std::make_unique(&npu_device_global); - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); chat->load_model(model_path, model_info, -1, preemption); header_print("info", "Model loaded"); diff --git a/src/test/qwen3_6_moe_npu/test.cpp b/src/test/qwen3_6_moe_npu/test.cpp index b6c6e4d4..fc5f4492 100644 --- a/src/test/qwen3_6_moe_npu/test.cpp +++ b/src/test/qwen3_6_moe_npu/test.cpp @@ -10,7 +10,7 @@ #include "model_list.hpp" #include "utils/vm_args.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; int main(int argc, char* argv[]) { #ifdef __WINDOWS__ @@ -45,7 +45,7 @@ int main(int argc, char* argv[]) { std::cout << "Model path: " << model_path << std::endl; std::unique_ptr chat = std::make_unique(&npu_device_global); - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); chat->load_model(model_path, model_info, -1, preemption); diff --git a/src/test/qwen3_npu/test.cpp b/src/test/qwen3_npu/test.cpp index fa85e086..9270abcc 100644 --- a/src/test/qwen3_npu/test.cpp +++ b/src/test/qwen3_npu/test.cpp @@ -5,7 +5,7 @@ #include "AutoModel/modeling_qwen3.hpp" #include "model_list.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; // Model-specific factory function for Qwen family and DeepSeek_r1_0528_8b inline std::unique_ptr get_qwen_model(const std::string& model_tag) { @@ -72,7 +72,7 @@ int main(int argc, char* argv[]) { std::pair model_info_pair = model_list.get_model_info(tag); nlohmann::json model_info = model_info_pair.second; std::cout << "Model path: " << model_path << std::endl; - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); std::unique_ptr chat = get_qwen_model(tag); diff --git a/src/test/qwen3vl_npu/test.cpp b/src/test/qwen3vl_npu/test.cpp index 8cbc1579..9f62e7fc 100644 --- a/src/test/qwen3vl_npu/test.cpp +++ b/src/test/qwen3vl_npu/test.cpp @@ -5,7 +5,7 @@ #include "AutoModel/modeling_qwen3vl.hpp" #include "model_list.hpp" -xrt::device npu_device_global; +flm_rt::device npu_device_global; // Model-specific factory function for Qwen family and DeepSeek_r1_0528_8b inline std::pair> get_qwen3vl_model(const std::string& model_tag) { @@ -58,7 +58,7 @@ int main(int argc, char* argv[]) { std::cout << "Model path: " << model_path << std::endl; std::unique_ptr chat = std::make_unique(&npu_device_global); - npu_device_global = xrt::device(0); + npu_device_global = flm_rt::device(0); chat->load_model(model_path, model_info, -1, preemption); header_print("info", "Model loaded"); diff --git a/src/test/whisper_npu/test.cpp b/src/test/whisper_npu/test.cpp index d3b99256..353b0205 100644 --- a/src/test/whisper_npu/test.cpp +++ b/src/test/whisper_npu/test.cpp @@ -46,7 +46,7 @@ int main(int argc, char* argv[]) { std::pair model_info_pair = model_list.get_model_info(tag); nlohmann::json model_info = model_info_pair.second; - xrt::device npu_device_global = xrt::device(0); + flm_rt::device npu_device_global = flm_rt::device(0); Whisper whisper(&npu_device_global);