diff --git a/README.md b/README.md index 5f2dc17a..0da34fe1 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ details. ## One-level benchmark catalog -The one-level manifests contain **53 build variants**. Every name below +The one-level manifests contain **62 build variants**. Every name below has a source-complete page with its build command and PTO intrinsic surface in the website's **Benchmarks** section. @@ -176,6 +176,12 @@ the website's **Benchmarks** section. +
One-level / pto_kernels (9 names, 9 variants) + +`pto_add`, `pto_flash_attention`, `pto_gemm`, `pto_gemm_basic`, `pto_gemm_demo`, `pto_gemm_performance`, `pto_mamulb`, `pto_tload_store`, `pto_tmatmul_acc` + +
+
One-level / reduction/reducemax_col (1 name, 1 variant) `reducemax_col` diff --git a/benchmark/one-level-arch/compile_all.sh b/benchmark/one-level-arch/compile_all.sh index 488720fe..eb6396f3 100755 --- a/benchmark/one-level-arch/compile_all.sh +++ b/benchmark/one-level-arch/compile_all.sh @@ -61,6 +61,7 @@ compile_operator "$REPO_ROOT/test/kernel/reduction/reducesum_row" "reducesum_row compile_operator "$REPO_ROOT/test/kernel/control" "control" || failures+=("control") compile_operator "$REPO_ROOT/test/kernel/fa" "fa" || failures+=("fa") compile_operator "$REPO_ROOT/test/kernel/sort" "sort" || failures+=("sort") +compile_operator "$REPO_ROOT/test/kernel/pto_kernels" "pto_kernels" || failures+=("pto_kernels") echo "" echo "==========================================" diff --git a/benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp b/benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp new file mode 100644 index 00000000..5d62c14d --- /dev/null +++ b/benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp @@ -0,0 +1,212 @@ +#ifndef PTO_COMMON_LINX_LOWP_TYPES_HPP +#define PTO_COMMON_LINX_LOWP_TYPES_HPP + +#include + +namespace pto { + +struct fp16_t { + uint16_t bits; +}; + +#if !defined(__CPU_SIM) +using half = fp16_t; +#endif + +struct fp8_e4m3_t { + uint8_t bits; +}; + +struct fp4_e2m1_t { + uint8_t bits; +}; + +inline float fp16_to_float(fp16_t x) { + const uint16_t v = x.bits; + const uint32_t sign = static_cast(v & 0x8000u) << 16; + const uint32_t exp = (v >> 10) & 0x1fu; + const uint32_t mant = v & 0x03ffu; + + uint32_t out_bits = 0u; + if (exp == 0u) { + if (mant == 0u) { + out_bits = sign; + } else { + int e = -14; + uint32_t m = mant; + while ((m & 0x0400u) == 0u) { + m <<= 1u; + --e; + } + m &= 0x03ffu; + const uint32_t exp32 = static_cast(e + 127); + out_bits = sign | (exp32 << 23) | (m << 13); + } + } else if (exp == 0x1fu) { + out_bits = sign | 0x7f800000u | (mant << 13); + } else { + const uint32_t exp32 = exp + (127u - 15u); + out_bits = sign | (exp32 << 23) | (mant << 13); + } + + union { + uint32_t u; + float f; + } cvt = {out_bits}; + return cvt.f; +} + +inline fp16_t float_to_fp16(float x) { + union { + float f; + uint32_t u; + } cvt = {x}; + + const uint32_t sign = (cvt.u >> 16) & 0x8000u; + const int exp32 = static_cast((cvt.u >> 23) & 0xffu); + const uint32_t mant32 = cvt.u & 0x7fffffu; + + if (exp32 == 0xff) { + const uint16_t nan_inf = static_cast(sign | 0x7c00u | (mant32 ? 0x0200u : 0u)); + return fp16_t{nan_inf}; + } + + const int exp16 = exp32 - 127 + 15; + if (exp16 <= 0) { + if (exp16 < -10) + return fp16_t{static_cast(sign)}; + uint32_t mant = mant32 | 0x800000u; + const int shift = 14 - exp16; + uint32_t rounded = mant >> static_cast(shift); + if (((mant >> static_cast(shift - 1)) & 1u) != 0u) + ++rounded; + return fp16_t{static_cast(sign | (rounded & 0x03ffu))}; + } + + if (exp16 >= 31) + return fp16_t{static_cast(sign | 0x7c00u)}; + + uint32_t mant = mant32; + mant += 0x1000u; // round-to-nearest-even at fp16 mantissa boundary + if (mant & 0x800000u) { + mant = 0u; + if (exp16 + 1 >= 31) + return fp16_t{static_cast(sign | 0x7c00u)}; + return fp16_t{static_cast(sign | (static_cast(exp16 + 1) << 10))}; + } + + return fp16_t{static_cast(sign | (static_cast(exp16) << 10) | (mant >> 13))}; +} + +inline float fp8_e4m3_to_float(fp8_e4m3_t x) { + auto pow2i = [](int e) -> float { + float s = 1.0f; + if (e >= 0) { + for (int i = 0; i < e; ++i) + s *= 2.0f; + } else { + for (int i = 0; i < -e; ++i) + s *= 0.5f; + } + return s; + }; + const uint8_t bits = x.bits; + const float sign = (bits & 0x80u) ? -1.0f : 1.0f; + const uint8_t exp = static_cast((bits >> 3) & 0x0fu); + const uint8_t mant = static_cast(bits & 0x07u); + + if (exp == 0u) { + if (mant == 0u) + return 0.0f * sign; + return sign * (static_cast(mant) / 8.0f) * pow2i(-6); + } + if (exp == 0x0fu) { + const float sat = (1.0f + (7.0f / 8.0f)) * pow2i(7); + return sign * sat; + } + return sign * (1.0f + static_cast(mant) / 8.0f) * + pow2i(static_cast(exp) - 7); +} + +inline fp8_e4m3_t float_to_fp8_e4m3(float x) { + if (x == 0.0f) + return fp8_e4m3_t{0u}; + + const bool neg = x < 0.0f; + float ax = neg ? -x : x; + int e = 0; + float norm = ax; + while (norm >= 2.0f && e < 30) { + norm *= 0.5f; + ++e; + } + while (norm < 1.0f && e > -30) { + norm *= 2.0f; + --e; + } + int ef = e + 7; + + uint8_t sign = neg ? 0x80u : 0u; + + if (ef <= 0) { + int mant = static_cast(ax * 512.0f + 0.5f); // ax * 2^(6+3) + if (mant < 0) + mant = 0; + if (mant > 7) + mant = 7; + return fp8_e4m3_t{static_cast(sign | mant)}; + } + + if (ef >= 0x0f) + return fp8_e4m3_t{static_cast(sign | 0x7eu)}; + + float frac = norm - 1.0f; + int mant = static_cast(frac * 8.0f + 0.5f); + if (mant >= 8) { + mant = 0; + ++ef; + if (ef >= 0x0f) + return fp8_e4m3_t{static_cast(sign | 0x7eu)}; + } + if (mant < 0) + mant = 0; + return fp8_e4m3_t{static_cast(sign | (static_cast(ef) << 3) | static_cast(mant))}; +} + +inline float fp4_e2m1_to_float(fp4_e2m1_t x) { + const uint8_t b = static_cast(x.bits & 0x0fu); + static constexpr float kTable[16] = { + 0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, + -0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f}; + return kTable[b]; +} + +inline fp4_e2m1_t float_to_fp4_e2m1(float x) { + static constexpr float kTable[16] = { + 0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, + -0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f}; + + auto absf = [](float v) -> float { return v < 0.0f ? -v : v; }; + uint8_t best = 0u; + float best_err = absf(x - kTable[0]); + for (uint8_t i = 1u; i < 16u; ++i) { + const float err = absf(x - kTable[i]); + if (err < best_err) { + best_err = err; + best = i; + } + } + return fp4_e2m1_t{static_cast(best & 0x0fu)}; +} + +inline uint32_t lowp_word_from_fp16(fp16_t x) { return static_cast(x.bits); } +inline uint32_t lowp_word_from_fp8(fp8_e4m3_t x) { return static_cast(x.bits); } +inline uint32_t lowp_word_from_fp4(fp4_e2m1_t x) { return static_cast(x.bits & 0x0fu); } + +inline fp16_t fp16_from_lowp_word(uint32_t word) { return fp16_t{static_cast(word & 0xffffu)}; } +inline fp8_e4m3_t fp8_from_lowp_word(uint32_t word) { return fp8_e4m3_t{static_cast(word & 0xffu)}; } +inline fp4_e2m1_t fp4_from_lowp_word(uint32_t word) { return fp4_e2m1_t{static_cast(word & 0x0fu)}; } + +} // namespace pto + +#endif // PTO_COMMON_LINX_LOWP_TYPES_HPP diff --git a/benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp b/benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp new file mode 100644 index 00000000..0f4aaf2c --- /dev/null +++ b/benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp @@ -0,0 +1,865 @@ +#ifndef PTO_COMMON_PTO_TILEOP_HPP +#define PTO_COMMON_PTO_TILEOP_HPP + +#include + +#include + +namespace pto { + +constexpr int DYNAMIC = -1; + +enum class Location : uint8_t { + Vec, + Left, + Right, + Acc, +}; + +enum class BLayout : uint8_t { + RowMajor = 0, + ColMajor = 1, +}; + +template struct RowMajor { + static constexpr int Rows = Rows_; + static constexpr int Cols = Cols_; + static constexpr bool IsRowMajor = true; +}; + +template struct ColMajor { + static constexpr int Rows = Rows_; + static constexpr int Cols = Cols_; + static constexpr bool IsRowMajor = false; +}; + +template struct global_tensor { + using DType = Element_; + using Layout = Layout_; +}; + +namespace detail { + +using ptrdiff_builtin_t = __PTRDIFF_TYPE__; + +template using void_t = void; + +template struct StaticIndex { + static constexpr int value = Value; + constexpr operator int() const { return Value; } +}; + +template +inline __attribute__((always_inline)) void static_for(Fn &&fn) { + if constexpr (Begin < End) { + fn(StaticIndex{}); + static_for(fn); + } +} + +// TMA format selectors used by B.ARG in canonical v0.57. +constexpr long long kLayoutNorm = 0ll; // NORM.normal +constexpr long long kLayoutND2NZ = 2ll; // ND2NZ.normal +constexpr long long kLayoutND2ZN = 3ll; // ND2ZN.normal +constexpr long long kLayoutDN2ZN = 8ll; // DN2ZN.normal +constexpr long long kLayoutDN2NZ = 9ll; // DN2NZ.normal + +template constexpr unsigned tileBytes() { + constexpr int rows = TileT::Rows; + constexpr int cols = TileT::Cols; + constexpr unsigned bytes = + static_cast(rows * cols * sizeof(typename TileT::DType)); + static_assert(bytes > 0u, + "PTO Linx canonical v0.57: tile bytes must be positive"); + return bytes; +} + +template constexpr unsigned tileSizeCode() { + static_assert(tileBytes() <= linx::detail::kMaxTileBytes, + "PTO Linx canonical v0.57: tile size exceeds 4KB"); + // Keep one 4 KiB carrier across data types. TCVT changes the element type, + // but not the architectural tile-register capacity or its SSA identity. + return 8u; +} + +template constexpr unsigned tileDTypeCode() { + return linx::detail::DTypeCode::value; +} + +template constexpr long long tileLayoutCode() { + return TileT::LayoutTag == BLayout::RowMajor ? 0ll : 1ll; +} + +template constexpr long long gmStrideBytes() { + constexpr long long elemBytes = + static_cast(sizeof(typename GTensor::DType)); + if constexpr (GTensor::Layout::IsRowMajor) + return static_cast(GTensor::Layout::Cols) * elemBytes; + return static_cast(GTensor::Layout::Rows) * elemBytes; +} + +template +constexpr long long tensorTileLayoutCode() { + if constexpr (TileT::Loc == Location::Left || TileT::Loc == Location::Acc) { + return GTensor::Layout::IsRowMajor ? kLayoutND2ZN : kLayoutDN2ZN; + } + if constexpr (TileT::Loc == Location::Right) { + return GTensor::Layout::IsRowMajor ? kLayoutND2NZ : kLayoutDN2NZ; + } + return kLayoutNorm; +} + +template constexpr long long tileLB0() { + return TileT::ColValid > 0 ? static_cast(TileT::ColValid) + : static_cast(TileT::Cols); +} + +template constexpr long long tileLB1() { + return TileT::RowValid > 0 ? static_cast(TileT::RowValid) + : static_cast(TileT::Rows); +} + +template +inline ptrdiff_builtin_t tileOffset(int tileRow, int tileCol) { + const int row = tileRow * TileT::Rows; + const int col = tileCol * TileT::Cols; + if constexpr (GTensor::Layout::IsRowMajor) { + return static_cast(row) * GTensor::Layout::Cols + col; + } + return static_cast(col) * GTensor::Layout::Rows + row; +} + +template +inline auto addressPtr(const AddressLike &addr) -> decltype(addr.ptr()) { + return addr.ptr(); +} + +template inline T *addressPtr(T *addr) { return addr; } + +template inline const T *addressPtr(const T *addr) { return addr; } + +template +struct AddressDesc { + static constexpr long long Layout = tileLayoutCode(); + static constexpr long long LB0 = tileLB0(); + static constexpr long long LB1 = tileLB1(); + static constexpr long long StrideBytes = 0ll; +}; + +template +struct AddressDesc< + AddressLike, TileT, + void_t> { + static constexpr long long Layout = AddressLike::kLayoutCode; + static constexpr long long LB0 = AddressLike::kLB0; + static constexpr long long LB1 = AddressLike::kLB1; + static constexpr long long StrideBytes = AddressLike::kStrideBytes; +}; + +template +constexpr long long addressLayoutCode() { + return AddressDesc::Layout; +} + +template +constexpr long long addressLB0() { + return AddressDesc::LB0; +} + +template +constexpr long long addressLB1() { + return AddressDesc::LB1; +} + +template +constexpr long long addressStrideBytes() { + return AddressDesc::StrideBytes; +} + +} // namespace detail + +template +struct Tile { + using DType = Element_; + using RawTile = linx::detail::RawTile; + using TileDType = Tile *; + using ConstTileDType = const Tile *; + + static constexpr Location Loc = Loc_; + static constexpr int Rows = Rows_; + static constexpr int Cols = Cols_; + static constexpr int RowValid = RowValid_; + static constexpr int ColValid = ColValid_; + static constexpr int ValidRow = RowValid_; + static constexpr int ValidCol = ColValid_; + static constexpr BLayout LayoutTag = Layout_; + static constexpr int RowStride = LayoutTag == BLayout::RowMajor ? Cols_ : 1; + static constexpr int ColStride = LayoutTag == BLayout::RowMajor ? 1 : Rows_; + + static_assert(RowValid_ == DYNAMIC || (RowValid_ > 0 && RowValid_ <= Rows_), + "PTO Linx: valid rows must fit the physical tile"); + static_assert(ColValid_ == DYNAMIC || (ColValid_ > 0 && ColValid_ <= Cols_), + "PTO Linx: valid columns must fit the physical tile"); + + Tile() + : valid_rows_(RowValid_ == DYNAMIC ? Rows_ : RowValid_), + valid_cols_(ColValid_ == DYNAMIC ? Cols_ : ColValid_) {} + + Tile(int validRows, int validCols) + : valid_rows_(validRows), valid_cols_(validCols) {} + + template explicit Tile(Scalar scalar) { + raw_ = linx::detail::teplSplat<0x019u, detail::tileSizeCode(), + detail::tileDTypeCode(), 2u>( + scalar, GetValidCol(), GetValidRow(), Cols); + } + + int GetValidRow() const { return valid_rows_; } + int GetValidCol() const { return valid_cols_; } + + void SetValidShape(int validRows, int validCols) { + valid_rows_ = validRows; + valid_cols_ = validCols; + } + + RawTile &raw() { return raw_; } + const RawTile &raw() const { return raw_; } + TileDType data() { return this; } + ConstTileDType data() const { return this; } + +private: + RawTile raw_{}; + int valid_rows_ = RowValid_ == DYNAMIC ? Rows_ : RowValid_; + int valid_cols_ = ColValid_ == DYNAMIC ? Cols_ : ColValid_; +}; + +template +using TileLeft = Tile; + +template +using TileRight = Tile; + +template +using TileAcc = Tile; + +template class global_iterator { +public: + using Element = typename GTensor::DType; + + explicit global_iterator(Element *base) : base_(base) {} + + struct tile_address { + using TensorType = GTensor; + using TileType = TileT; + static constexpr long long kLayoutCode = + detail::tensorTileLayoutCode(); + // TMA contract: LB0/LB1 are GM-side inner/outer counts. + // ND(row-major): inner=cols, outer=rows; DN(column-major): inner=rows, + // outer=cols. + static constexpr long long kLB0 = GTensor::Layout::IsRowMajor + ? detail::tileLB1() + : detail::tileLB0(); + static constexpr long long kLB1 = GTensor::Layout::IsRowMajor + ? detail::tileLB0() + : detail::tileLB1(); + static constexpr long long kStrideBytes = detail::gmStrideBytes(); + + Element *base; + int tileRow; + int tileCol; + + Element *ptr() const { + return base + detail::tileOffset(tileRow, tileCol); + } + }; + + tile_address operator()(int tileRow, int tileCol) const { + return tile_address{base_, tileRow, tileCol}; + } + +private: + Element *base_; +}; + +namespace tepl { +constexpr unsigned TADD = 0x000u; +constexpr unsigned TSUB = 0x001u; +constexpr unsigned TMUL = 0x002u; +constexpr unsigned TDIV = 0x003u; +constexpr unsigned TMAX = 0x004u; +constexpr unsigned TMIN = 0x005u; +constexpr unsigned TAND = 0x006u; +constexpr unsigned TOR = 0x007u; +constexpr unsigned TXOR = 0x008u; +constexpr unsigned TSHL = 0x009u; +constexpr unsigned TSHR = 0x00au; +constexpr unsigned TRELU = 0x00bu; +constexpr unsigned TPRELU = 0x00cu; +constexpr unsigned TCVT = 0x00du; +constexpr unsigned TEXP = 0x00eu; +constexpr unsigned TLOG = 0x00fu; +constexpr unsigned TSQRT = 0x010u; +constexpr unsigned TRSQRT = 0x011u; +constexpr unsigned TROWMAX = 0x012u; +constexpr unsigned TROWMIN = 0x013u; +constexpr unsigned TROWSUM = 0x014u; +constexpr unsigned TCOLMAX = 0x015u; +constexpr unsigned TCOLMIN = 0x016u; +constexpr unsigned TCOLSUM = 0x017u; +constexpr unsigned TRECIP = 0x018u; +constexpr unsigned TEXPANDS = 0x019u; +constexpr unsigned TGATHER = 0x01au; +constexpr unsigned TSCATTER = 0x01bu; +constexpr unsigned TRESHAPE = 0x01cu; +constexpr unsigned TTRANSPOSE = 0x01du; +constexpr unsigned TCOLEXPAND = 0x01eu; +constexpr unsigned TROWEXPAND = 0x01fu; +constexpr unsigned TADDS = 0x020u; +constexpr unsigned TSUBS = 0x021u; +constexpr unsigned TMULS = 0x022u; +constexpr unsigned TDIVS = 0x023u; +constexpr unsigned TMAXS = 0x024u; +constexpr unsigned TMINS = 0x025u; +constexpr unsigned TANDS = 0x026u; +constexpr unsigned TORS = 0x027u; +constexpr unsigned TXORS = 0x028u; +constexpr unsigned TSHLS = 0x029u; +constexpr unsigned TSHRS = 0x02au; +constexpr unsigned TCMP = 0x02bu; +constexpr unsigned TSEL = 0x02cu; +constexpr unsigned TABS = 0x02du; +constexpr unsigned TNOT = 0x02eu; +constexpr unsigned TCMPS = 0x033u; +constexpr unsigned TSELS = 0x034u; +constexpr unsigned TCONCAT = 0x087u; +constexpr unsigned TSORT = 0x0c0u; +constexpr unsigned TMRGSORT = 0x0c1u; +constexpr unsigned THISTOGRAM = 0x0c2u; +constexpr unsigned TPARTADD = 0x0c3u; +constexpr unsigned TPARTMUL = 0x0c4u; +constexpr unsigned TPARTMAX = 0x0c5u; +constexpr unsigned TPARTMIN = 0x0c6u; +constexpr unsigned TPARTARGMAX = 0x0c7u; +constexpr unsigned TPARTARGMIN = 0x0c8u; +} // namespace tepl + +// Core tile ops used by PR5 FlashAttention bring-up. +template +inline void TLOAD(DstTile &dst, const SrcAddress &src) { + dst.raw() = + linx::detail::tileTLoad(), + detail::tileDTypeCode(), + detail::addressLayoutCode()>( + reinterpret_cast(detail::addressPtr(src)), + dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols, + detail::addressStrideBytes()); +} + +template +inline void TSTORE(const DstAddress &dst, SrcTile &src) { + linx::detail::tileTStore(), + detail::tileDTypeCode(), + detail::addressLayoutCode()>( + reinterpret_cast(detail::addressPtr(dst)), src.raw(), + src.GetValidCol(), src.GetValidRow(), SrcTile::Cols, + detail::addressStrideBytes()); +} + +template +inline void TMOV(DstTile &dst, const SrcTile &src, unsigned mode = 0u) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + if (mode == 1u) { + dst.raw() = + linx::detail::tileTMov(), + detail::tileDTypeCode(), + detail::tileLayoutCode(), 1u, 1u>( + src.raw()); + } else { + dst.raw() = + linx::detail::tileTMov(), + detail::tileDTypeCode(), + detail::tileLayoutCode(), 1u, 0u>( + src.raw()); + } +} + +template +inline void TMATMUL(TileRes &dst, const TileLeft_ &lhs, const TileRight_ &rhs) { + // Canonical v0.57 compiler policy: + // tile_bytes = ceil(m*n*k*elem_bits/8) must fit <=4KB + // (m=Rows, n=Cols, k=lhs.Cols). + constexpr unsigned M = static_cast(TileRes::Rows); + constexpr unsigned N = static_cast(TileRes::Cols); + constexpr unsigned K = static_cast(TileLeft_::Cols); + dst.raw() = linx::detail::cubeMamulb(lhs.raw(), rhs.raw()); +} + +template +inline void TMATMUL_ACC(TileRes &dst, TileRes &acc, const TileLeft_ &lhs, + const TileRight_ &rhs) { + constexpr unsigned M = static_cast(TileRes::Rows); + constexpr unsigned N = static_cast(TileRes::Cols); + constexpr unsigned K = static_cast(TileLeft_::Cols); + dst.raw() = + linx::detail::cubeMamulbAcc(acc.raw(), lhs.raw(), rhs.raw()); +} + +template +inline void MATMACC(TileRes &dst, const TileLeft_ &lhs, const TileRight_ &rhs) { + // Keep strict CUBE accumulator-chain legality: materialize the product with + // TMATMUL, then accumulate explicitly with TEPL add. + TileRes product; + TMATMUL(product, lhs, rhs); + TADD(dst, dst, product); +} + +template +inline void TCVT(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TADD(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TSUB(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TMUL(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TMAX(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TDIV(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TMIN(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TAND(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TOR(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TXOR(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TSHL(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TSHR(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TROWMAX(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); + dst.SetValidShape(src.GetValidRow(), 1); +} + +template +inline void TROWMIN(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); + dst.SetValidShape(src.GetValidRow(), 1); +} + +template +inline void TROWSUM(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); + dst.SetValidShape(src.GetValidRow(), 1); +} + +template +inline void TCOLMAX(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); + dst.SetValidShape(1, src.GetValidCol()); +} + +template +inline void TCOLMIN(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); + dst.SetValidShape(1, src.GetValidCol()); +} + +template +inline void TCOLSUM(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); + dst.SetValidShape(1, src.GetValidCol()); +} + +template +inline void TRELU(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TEXP(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TLOG(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TSQRT(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TRSQRT(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TRECIP(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TABS(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TNOT(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TRESHAPE(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TTRANSPOSE(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); + dst.SetValidShape(src.GetValidCol(), src.GetValidRow()); +} + +template +inline void TSORT(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void THISTOGRAM(DstTile &dst, const SrcTile &src) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); + dst.SetValidShape(1, src.GetValidCol()); +} + +template +inline void TGATHER(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TSCATTER(DstTile &dst, const SrcTile0 &src0, const SrcTile1 &src1) { + dst.SetValidShape(src0.GetValidRow(), src0.GetValidCol()); + dst.raw() = + linx::detail::teplBinary(), + detail::tileDTypeCode()>( + src0.raw(), src1.raw(), dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TMULS(DstTile &dst, const SrcTile &src, Scalar scalar) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplBinaryScalar(), + detail::tileDTypeCode(), 1u>( + src.raw(), scalar, dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TADDS(DstTile &dst, const SrcTile &src, Scalar scalar) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplBinaryScalar(), + detail::tileDTypeCode(), 1u>( + src.raw(), scalar, dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TSUBS(DstTile &dst, const SrcTile &src, Scalar scalar) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplBinaryScalar(), + detail::tileDTypeCode(), 1u>( + src.raw(), scalar, dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TDIVS(DstTile &dst, const SrcTile &src, Scalar scalar) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplBinaryScalar(), + detail::tileDTypeCode(), 1u>( + src.raw(), scalar, dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TMAXS(DstTile &dst, const SrcTile &src, Scalar scalar) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplBinaryScalar(), + detail::tileDTypeCode(), 1u>( + src.raw(), scalar, dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TMINS(DstTile &dst, const SrcTile &src, Scalar scalar) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplBinaryScalar(), + detail::tileDTypeCode(), 1u>( + src.raw(), scalar, dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TXORS(DstTile &dst, const SrcTile &src, Scalar scalar) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplBinaryScalar(), + detail::tileDTypeCode(), 1u>( + src.raw(), scalar, dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TSHLS(DstTile &dst, const SrcTile &src, Scalar scalar) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplBinaryScalar(), + detail::tileDTypeCode(), 1u>( + src.raw(), scalar, dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TSHRS(DstTile &dst, const SrcTile &src, Scalar scalar) { + dst.SetValidShape(src.GetValidRow(), src.GetValidCol()); + dst.raw() = + linx::detail::teplBinaryScalar(), + detail::tileDTypeCode(), 1u>( + src.raw(), scalar, dst.GetValidCol(), dst.GetValidRow(), + DstTile::Cols); +} + +template +inline void TEXPANDS(DstTile &dst, Scalar scalar) { + dst.raw() = + linx::detail::teplSplat(), + detail::tileDTypeCode(), 2u>( + scalar, dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TCOLEXPAND(DstTile &dst, const SrcTile &src) { + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TROWEXPAND(DstTile &dst, const SrcTile &src) { + dst.raw() = + linx::detail::teplUnary(), + detail::tileDTypeCode()>( + src.raw(), dst.GetValidCol(), dst.GetValidRow(), DstTile::Cols); +} + +template +inline void TEXPANDCOL(DstTile &dst, const SrcTile &src) { + TCOLEXPAND(dst, src); +} + +} // namespace pto + +#endif // PTO_COMMON_PTO_TILEOP_HPP diff --git a/benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp b/benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp new file mode 100644 index 00000000..773e934d --- /dev/null +++ b/benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp @@ -0,0 +1,28 @@ +#ifndef PTO_COMMON_RUNTIME_KERNEL_ENV_HPP +#define PTO_COMMON_RUNTIME_KERNEL_ENV_HPP + +#ifndef PTO_QEMU_SMOKE +#define PTO_QEMU_SMOKE 0 +#endif + +#ifndef PTO_USE_MIXED_TILE_SIMT +#define PTO_USE_MIXED_TILE_SIMT 0 +#endif + +namespace pto { +namespace kernels { +namespace env { + +inline constexpr bool kQemuSmoke = PTO_QEMU_SMOKE != 0; +inline constexpr bool kMixedTileSimt = PTO_USE_MIXED_TILE_SIMT != 0; + +template +inline constexpr T select(T smoke_value, T full_value) { + return kQemuSmoke ? smoke_value : full_value; +} + +} // namespace env +} // namespace kernels +} // namespace pto + +#endif // PTO_COMMON_RUNTIME_KERNEL_ENV_HPP diff --git a/benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp b/benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp new file mode 100644 index 00000000..a8c07b04 --- /dev/null +++ b/benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp @@ -0,0 +1,77 @@ +#ifndef PTO_COMMON_RUNTIME_KERNEL_SHAPES_HPP +#define PTO_COMMON_RUNTIME_KERNEL_SHAPES_HPP + +#include + +#ifndef PTO_ATTENTION_SMOKE_SEQ +#define PTO_ATTENTION_SMOKE_SEQ 16 +#endif + +#ifndef PTO_ATTENTION_LARGE_SMOKE_SEQ +#define PTO_ATTENTION_LARGE_SMOKE_SEQ 16 +#endif + +#ifndef PTO_ATTENTION_SMOKE_QD +#define PTO_ATTENTION_SMOKE_QD 16 +#endif + +#ifndef PTO_ATTENTION_SMOKE_VD +#define PTO_ATTENTION_SMOKE_VD 16 +#endif + +#ifndef PTO_ATTENTION_SMALL_SMOKE_QD +#define PTO_ATTENTION_SMALL_SMOKE_QD 4 +#endif + +#ifndef PTO_ATTENTION_MASKED_SMOKE_SEQ +#define PTO_ATTENTION_MASKED_SMOKE_SEQ 18 +#endif + +#ifndef PTO_ATTENTION_MASKED_SMOKE_QD +#define PTO_ATTENTION_MASKED_SMOKE_QD 16 +#endif + +#ifndef PTO_ATTENTION_MASKED_SMOKE_VD +#define PTO_ATTENTION_MASKED_SMOKE_VD 16 +#endif + +namespace pto { +namespace kernels { +namespace shapes { + +inline constexpr int kMemoryRows = env::select(32, 1024); +inline constexpr int kMemoryCols = env::select(32, 1024); + +inline constexpr int kMatmulM = env::select(16, 256); +inline constexpr int kMatmulN = env::select(16, 256); +inline constexpr int kMatmulK = env::select(16, 256); +inline constexpr int kMatmulReuseExtent = env::select(16, 64); + +inline constexpr int kAttentionSeq = env::select(PTO_ATTENTION_SMOKE_SEQ, 128); +inline constexpr int kAttentionLargeSeq = + env::select(PTO_ATTENTION_LARGE_SMOKE_SEQ, 256); +inline constexpr int kAttentionQD = env::select(PTO_ATTENTION_SMOKE_QD, 16); +inline constexpr int kAttentionVD = env::select(PTO_ATTENTION_SMOKE_VD, 16); +inline constexpr int kAttentionSmallQD = + env::select(PTO_ATTENTION_SMALL_SMOKE_QD, 4); +inline constexpr int kAttentionMaskedSeq = + env::select(PTO_ATTENTION_MASKED_SMOKE_SEQ, 130); +inline constexpr int kAttentionMaskedQD = + env::select(PTO_ATTENTION_MASKED_SMOKE_QD, 16); +inline constexpr int kAttentionMaskedVD = + env::select(PTO_ATTENTION_MASKED_SMOKE_VD, 16); + +inline constexpr int kMlaInputDim = 16; +inline constexpr int kMlaLatentDim = 4; +inline constexpr int kMlaOutputDim = 16; + +inline constexpr int kNormTokens = env::select(16, 128); + +inline constexpr int kSmallVector = env::select(64, 1024); +inline constexpr int kSmallTable = env::select(97, 2048); + +} // namespace shapes +} // namespace kernels +} // namespace pto + +#endif // PTO_COMMON_RUNTIME_KERNEL_SHAPES_HPP diff --git a/benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_tiling.hpp b/benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_tiling.hpp new file mode 100644 index 00000000..d9c12d64 --- /dev/null +++ b/benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_tiling.hpp @@ -0,0 +1,71 @@ +#ifndef PTO_COMMON_RUNTIME_KERNEL_TILING_HPP +#define PTO_COMMON_RUNTIME_KERNEL_TILING_HPP + +#ifndef PTO_GEMM_TILE_M +#define PTO_GEMM_TILE_M 16 +#endif + +#ifndef PTO_GEMM_TILE_N +#define PTO_GEMM_TILE_N 16 +#endif + +#ifndef PTO_GEMM_TILE_K +#define PTO_GEMM_TILE_K 4 +#endif + +#ifndef PTO_FLASH_TILE_M +#define PTO_FLASH_TILE_M 16 +#endif + +#ifndef PTO_FLASH_TILE_K +#define PTO_FLASH_TILE_K 4 +#endif + +#ifndef PTO_FLASH_VEC_TILE_M +#define PTO_FLASH_VEC_TILE_M 8 +#endif + +#ifndef PTO_FLASH_VEC_TILE_K +#define PTO_FLASH_VEC_TILE_K 4 +#endif + +#ifndef PTO_FLASH_VEC_YDIM +#define PTO_FLASH_VEC_YDIM 4 +#endif + +#ifndef PTO_FLASH_CUBE_TILE_M +#define PTO_FLASH_CUBE_TILE_M 16 +#endif + +#ifndef PTO_FLASH_CUBE_TILE_K +#define PTO_FLASH_CUBE_TILE_K 16 +#endif + +#ifndef PTO_FLASH_CUBE_YDIM +#define PTO_FLASH_CUBE_YDIM 2 +#endif + +namespace pto { +namespace kernels { +namespace tiling { + +inline constexpr int kGemmTileM = PTO_GEMM_TILE_M; +inline constexpr int kGemmTileN = PTO_GEMM_TILE_N; +inline constexpr int kGemmTileK = PTO_GEMM_TILE_K; + +inline constexpr int kFlashTileM = PTO_FLASH_TILE_M; +inline constexpr int kFlashTileK = PTO_FLASH_TILE_K; + +inline constexpr int kFlashVecTileM = PTO_FLASH_VEC_TILE_M; +inline constexpr int kFlashVecTileK = PTO_FLASH_VEC_TILE_K; +inline constexpr int kFlashVecYDim = PTO_FLASH_VEC_YDIM; + +inline constexpr int kFlashCubeTileM = PTO_FLASH_CUBE_TILE_M; +inline constexpr int kFlashCubeTileK = PTO_FLASH_CUBE_TILE_K; +inline constexpr int kFlashCubeYDim = PTO_FLASH_CUBE_YDIM; + +} // namespace tiling +} // namespace kernels +} // namespace pto + +#endif // PTO_COMMON_RUNTIME_KERNEL_TILING_HPP diff --git a/benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp b/benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp new file mode 100644 index 00000000..19210d65 --- /dev/null +++ b/benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp @@ -0,0 +1,978 @@ +#ifndef PTO_LINX_IMPL_BACKEND_HPP +#define PTO_LINX_IMPL_BACKEND_HPP + +#include +#if defined(PTO_HOST_SIM) +#include +#include +#endif +#include + +namespace pto { +namespace linx { +namespace detail { + +template struct dependent_false { + static constexpr bool value = false; +}; + +template struct is_same { + static constexpr bool value = false; +}; + +template struct is_same { + static constexpr bool value = true; +}; + +template struct is_arithmetic { + static constexpr bool value = false; +}; + +template <> struct is_arithmetic { + static constexpr bool value = true; +}; + +template <> struct is_arithmetic { + static constexpr bool value = true; +}; + +template <> struct is_arithmetic { + static constexpr bool value = true; +}; + +template <> struct is_arithmetic { + static constexpr bool value = true; +}; + +template <> struct is_arithmetic { + static constexpr bool value = true; +}; + +template <> struct is_arithmetic { + static constexpr bool value = true; +}; + +template <> struct is_arithmetic { + static constexpr bool value = true; +}; + +template <> struct is_arithmetic { + static constexpr bool value = true; +}; + +template <> struct is_arithmetic { + static constexpr bool value = true; +}; + +template struct is_floating_point { + static constexpr bool value = false; +}; + +template <> struct is_floating_point { + static constexpr bool value = true; +}; + +template <> struct is_floating_point { + static constexpr bool value = true; +}; + +template struct DTypeCode { + static_assert(dependent_false::value, + "PTO Linx canonical v0.57: unsupported tile dtype"); +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 17u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 25u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 1u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 19u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 27u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 18u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 26u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 16u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 24u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 0u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 2u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 3u; +}; + +template <> struct DTypeCode { + static constexpr unsigned value = 11u; +}; + +constexpr unsigned kMinTileBytes = 512u; +constexpr unsigned kMaxTileBytes = 4096u; +constexpr unsigned kTileWords = kMaxTileBytes / sizeof(uint32_t); + +#if defined(PTO_HOST_SIM) +struct RawTile { + alignas(64) uint32_t words[kTileWords]; +}; +#else +using RawTile = int __attribute__((__vector_size__(4096), __aligned__(64))); +#endif + +constexpr unsigned clampTileBytes(unsigned bytes) { + return bytes < kMinTileBytes + ? kMinTileBytes + : (bytes > kMaxTileBytes ? kMaxTileBytes : bytes); +} + +constexpr unsigned nextPow2(unsigned value) { + unsigned p = 1u; + while (p < value && p < kMaxTileBytes) + p <<= 1u; + return p; +} + +constexpr unsigned sizeCodeFromBytes(unsigned bytes) { + const unsigned clipped = clampTileBytes(bytes); + const unsigned p2 = nextPow2(clipped); + unsigned code = 0u; + while ((1u << (code + 4u)) < p2) + ++code; + if (code < 5u) + code = 5u; + if (code > 8u) + code = 8u; + return code; +} + +constexpr unsigned dtypeElemBits(unsigned dtype) { + switch (dtype & 0x1fu) { + case 0u: // FP64 + case 16u: // INT64 + case 24u: // UINT64 + return 64u; + case 1u: // FP32 + case 17u: // INT32 + case 25u: // UINT32 + return 32u; + case 2u: // FP16 + case 6u: // BF16 + case 18u: // INT16 + case 26u: // UINT16 + return 16u; + case 3u: // FP8 + case 7u: // FPL8 + case 19u: // INT8 + case 27u: // UINT8 + return 8u; + case 11u: // FP4 + case 12u: // FPL4 + case 20u: // INT4 + case 28u: // UINT4 + return 4u; + default: + return 32u; + } +} + +constexpr unsigned dtypeElemBytesForStorage(unsigned dtype) { + const unsigned bits = dtypeElemBits(dtype); + return (bits + 7u) / 8u; +} + +constexpr unsigned dtypeElemCountForBytes(uint64_t bytes, unsigned dtype) { + const unsigned bits = dtypeElemBits(dtype); + if (bits == 0u) + return 0u; + const uint64_t total_bits = bytes * 8u; + return static_cast(total_bits / bits); +} + +template inline long long encodeScalar(Scalar value) { + static_assert(is_arithmetic::value, + "PTO Linx canonical v0.57: scalar operand must be arithmetic"); + if constexpr (is_same::value) { + return static_cast(value.bits); + } else if constexpr (is_same::value) { + return static_cast(value.bits); + } else if constexpr (is_same::value) { + return static_cast(value.bits & 0x0fu); + } + if constexpr (is_floating_point::value) { + if constexpr (sizeof(Scalar) == sizeof(uint32_t)) { + union { + Scalar f; + uint32_t u; + } cvt = {value}; + return static_cast(cvt.u); + } else if constexpr (sizeof(Scalar) == sizeof(uint64_t)) { + union { + Scalar f; + uint64_t u; + } cvt = {value}; + return static_cast(cvt.u); + } else { + return static_cast(value); + } + } + return static_cast(value); +} + +#if defined(PTO_HOST_SIM) + +inline uint64_t sizeBytesFromCode(unsigned size_code) { + return (size_code < 60u) ? (1ull << (size_code + 4u)) : 0ull; +} + +template inline uint32_t bitCastToU32(T value) { + static_assert(sizeof(T) == sizeof(uint32_t), + "bitCastToU32 requires 32-bit type"); + uint32_t out = 0; + memcpy(&out, &value, sizeof(uint32_t)); + return out; +} + +template inline T bitCastFromU32(uint32_t bits) { + static_assert(sizeof(T) == sizeof(uint32_t), + "bitCastFromU32 requires 32-bit type"); + T out{}; + memcpy(&out, &bits, sizeof(uint32_t)); + return out; +} + +inline float scalarAsF32(long long scalar_bits) { + uint32_t bits = static_cast(scalar_bits & 0xffffffffull); + return bitCastFromU32(bits); +} + +inline int32_t scalarAsI32(long long scalar_bits) { + return static_cast(scalar_bits & 0xffffffffull); +} + +inline uint32_t scalarToWordDType(long long scalar_bits, unsigned dtype) { + switch (dtype & 0x1fu) { + case 2u: + return static_cast(static_cast(scalar_bits & 0xffffu)); + case 3u: + return static_cast(static_cast(scalar_bits & 0xffu)); + case 11u: + return static_cast(static_cast(scalar_bits & 0x0fu)); + default: + return static_cast(scalar_bits & 0xffffffffu); + } +} + +inline uint32_t quantizeF32ToWord(float x, unsigned dtype) { + switch (dtype & 0x1fu) { + case 2u: + return pto::lowp_word_from_fp16(pto::float_to_fp16(x)); + case 3u: + return pto::lowp_word_from_fp8(pto::float_to_fp8_e4m3(x)); + case 11u: + return pto::lowp_word_from_fp4(pto::float_to_fp4_e2m1(x)); + case 17u: + return static_cast(static_cast(x)); + case 25u: + return static_cast(x < 0.0f ? 0.0f : x); + case 18u: + return static_cast(static_cast(x)); + case 26u: + return static_cast(x < 0.0f ? 0.0f : x); + case 19u: + return static_cast(static_cast( + x < -128.0f ? -128.0f : (x > 127.0f ? 127.0f : x))); + case 27u: + return static_cast( + x < 0.0f ? 0.0f : (x > 255.0f ? 255.0f : x)); + default: + return bitCastToU32(x); + } +} + +inline float dequantWordToF32(uint32_t word, unsigned dtype) { + switch (dtype & 0x1fu) { + case 2u: + return pto::fp16_to_float(pto::fp16_from_lowp_word(word)); + case 3u: + return pto::fp8_e4m3_to_float(pto::fp8_from_lowp_word(word)); + case 11u: + return pto::fp4_e2m1_to_float(pto::fp4_from_lowp_word(word)); + case 17u: + return static_cast(static_cast(word)); + case 25u: + return static_cast(word); + case 18u: + return static_cast(static_cast(word)); + case 26u: + return static_cast(static_cast(word)); + case 19u: + return static_cast(static_cast(word)); + case 27u: + return static_cast(static_cast(word)); + default: + return bitCastFromU32(word); + } +} + +template +inline RawTile teplUnaryHost(const RawTile &src, unsigned elems, unsigned rows, + unsigned cols) { + RawTile out{}; + for (unsigned i = 0; i < kTileWords; ++i) + out.words[i] = 0u; + + switch (TileOpcode & 0x3ffu) { + case 0x00du: // TCVT + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float f = dequantWordToF32(src.words[i], SrcDType); + out.words[i] = quantizeF32ToWord(f, DType); + } + break; + case 0x00bu: // TRELU + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float f = dequantWordToF32(src.words[i], DType); + out.words[i] = quantizeF32ToWord(f > 0.0f ? f : 0.0f, DType); + } + break; + case 0x00eu: // TEXP + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float f = dequantWordToF32(src.words[i], DType); + out.words[i] = quantizeF32ToWord(expf(f), DType); + } + break; + case 0x018u: // TRECIP + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float f = dequantWordToF32(src.words[i], DType); + float inv = (f == 0.0f) ? 0.0f : (1.0f / f); + out.words[i] = quantizeF32ToWord(inv, DType); + } + break; + case 0x00fu: // TLOG + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float f = dequantWordToF32(src.words[i], DType); + out.words[i] = quantizeF32ToWord(f > 0.0f ? logf(f) : -INFINITY, DType); + } + break; + case 0x010u: // TSQRT + case 0x011u: // TRSQRT + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float f = dequantWordToF32(src.words[i], DType); + const float root = f >= 0.0f ? sqrtf(f) : NAN; + out.words[i] = quantizeF32ToWord((TileOpcode & 0x3ffu) == 0x011u + ? (root == 0.0f ? 0.0f : 1.0f / root) + : root, + DType); + } + break; + case 0x02du: // TABS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float f = dequantWordToF32(src.words[i], DType); + out.words[i] = quantizeF32ToWord(fabsf(f), DType); + } + break; + case 0x02eu: // TNOT + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + out.words[i] = ~src.words[i]; + break; + case 0x012u: // TROWMAX + case 0x013u: // TROWMIN + case 0x014u: { // TROWSUM + for (unsigned r = 0; r < rows; ++r) { + float value = dequantWordToF32(src.words[r * cols], DType); + if ((TileOpcode & 0x3ffu) == 0x014u) + value = 0.0f; + for (unsigned c = 0; c < cols; ++c) { + const float cur = dequantWordToF32(src.words[r * cols + c], DType); + if ((TileOpcode & 0x3ffu) == 0x012u) + value = value > cur ? value : cur; + else if ((TileOpcode & 0x3ffu) == 0x013u) + value = value < cur ? value : cur; + else + value += cur; + } + out.words[r * cols] = quantizeF32ToWord(value, DType); + } + break; + } + case 0x015u: // TCOLMAX + case 0x016u: // TCOLMIN + case 0x017u: { // TCOLSUM + for (unsigned c = 0; c < cols; ++c) { + float value = dequantWordToF32(src.words[c], DType); + if ((TileOpcode & 0x3ffu) == 0x017u) + value = 0.0f; + for (unsigned r = 0; r < rows; ++r) { + const float cur = dequantWordToF32(src.words[r * cols + c], DType); + if ((TileOpcode & 0x3ffu) == 0x015u) + value = value > cur ? value : cur; + else if ((TileOpcode & 0x3ffu) == 0x016u) + value = value < cur ? value : cur; + else + value += cur; + } + out.words[c] = quantizeF32ToWord(value, DType); + } + break; + } + case 0x01du: { // TTRANSPOSE + for (unsigned r = 0; r < rows; ++r) + for (unsigned c = 0; c < cols; ++c) + out.words[c * rows + r] = src.words[r * cols + c]; + break; + } + case 0x01eu: // TCOLEXPAND + case 0x01fu: { // TROWEXPAND + for (unsigned r = 0; r < rows; ++r) + for (unsigned c = 0; c < cols; ++c) + out.words[r * cols + c] = (TileOpcode & 0x3ffu) == 0x01eu + ? src.words[c] + : src.words[r * cols]; + break; + } + case 0x0c0u: { // TSORT + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + out.words[i] = src.words[i]; + for (unsigned r = 0; r < rows; ++r) + for (unsigned i = 1; i < cols; ++i) { + const uint32_t key = out.words[r * cols + i]; + const float key_f = dequantWordToF32(key, DType); + unsigned j = i; + while (j > 0 && + dequantWordToF32(out.words[r * cols + j - 1], DType) > key_f) { + out.words[r * cols + j] = out.words[r * cols + j - 1]; + --j; + } + out.words[r * cols + j] = key; + } + break; + } + case 0x0c2u: // THISTOGRAM + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + out.words[src.words[i] % (cols == 0u ? 1u : cols)] += 1u; + break; + default: + // Unsupported op in host backend: keep destination zeroed. + break; + } + return out; +} + +template +inline RawTile teplBinaryHost(const RawTile &lhs, const RawTile &rhs, + unsigned elems) { + RawTile out{}; + for (unsigned i = 0; i < kTileWords; ++i) + out.words[i] = 0u; + + switch (TileOpcode & 0x3ffu) { + case 0x01au: // TGATHER + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + out.words[i] = lhs.words[rhs.words[i] % elems]; + break; + case 0x01bu: // TSCATTER + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + out.words[rhs.words[i] % elems] = lhs.words[i]; + break; + case 0x000u: // TADD + case 0x020u: // TADDS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float a = dequantWordToF32(lhs.words[i], DType); + const float b = dequantWordToF32(rhs.words[i], DType); + out.words[i] = quantizeF32ToWord(a + b, DType); + } + break; + case 0x001u: // TSUB + case 0x021u: // TSUBS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float a = dequantWordToF32(lhs.words[i], DType); + const float b = dequantWordToF32(rhs.words[i], DType); + out.words[i] = quantizeF32ToWord(a - b, DType); + } + break; + case 0x002u: // TMUL + case 0x022u: { // TMULS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float a = dequantWordToF32(lhs.words[i], DType); + const float b = dequantWordToF32(rhs.words[i], DType); + out.words[i] = quantizeF32ToWord(a * b, DType); + } + break; + } + case 0x003u: // TDIV + case 0x023u: { // TDIVS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float a = dequantWordToF32(lhs.words[i], DType); + const float b = dequantWordToF32(rhs.words[i], DType); + const float q = (b == 0.0f) ? 0.0f : (a / b); + out.words[i] = quantizeF32ToWord(q, DType); + } + break; + } + case 0x004u: // TMAX + case 0x024u: { // TMAXS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float a = dequantWordToF32(lhs.words[i], DType); + const float b = dequantWordToF32(rhs.words[i], DType); + out.words[i] = quantizeF32ToWord(a > b ? a : b, DType); + } + break; + } + case 0x005u: // TMIN + case 0x025u: { // TMINS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) { + const float a = dequantWordToF32(lhs.words[i], DType); + const float b = dequantWordToF32(rhs.words[i], DType); + out.words[i] = quantizeF32ToWord(a < b ? a : b, DType); + } + break; + } + case 0x006u: // TAND + case 0x026u: // TANDS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + out.words[i] = lhs.words[i] & rhs.words[i]; + break; + case 0x007u: // TOR + case 0x027u: // TORS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + out.words[i] = lhs.words[i] | rhs.words[i]; + break; + case 0x008u: // TXOR + case 0x028u: // TXORS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + out.words[i] = lhs.words[i] ^ rhs.words[i]; + break; + case 0x009u: // TSHL + case 0x029u: // TSHLS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + out.words[i] = lhs.words[i] << (rhs.words[i] & 31u); + break; + case 0x00au: // TSHR + case 0x02au: // TSHRS + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + out.words[i] = lhs.words[i] >> (rhs.words[i] & 31u); + break; + default: + break; + } + return out; +} + +#endif + +template +inline RawTile tileTLoad(const void *base, unsigned valid_col, + unsigned valid_row, unsigned physical_col, + uint64_t stride_bytes) { + static_assert(SizeCode >= 5u && SizeCode <= 8u, + "PTO Linx canonical v0.57: size_code must be in [5,8]"); +#if defined(PTO_HOST_SIM) + (void)Layout; + RawTile out{}; + for (unsigned i = 0; i < kTileWords; ++i) + out.words[i] = 0u; + + const uint64_t bytes64 = sizeBytesFromCode(SizeCode); + const unsigned elem_bytes = dtypeElemBytesForStorage(DType); + const unsigned elem_bits = dtypeElemBits(DType); + if (bytes64 == 0 || bytes64 > kMaxTileBytes || elem_bits == 0u || + (bytes64 % elem_bytes) != 0u) + return out; + + const unsigned max_elems = dtypeElemCountForBytes(bytes64, DType); + const uint64_t cols = valid_col; + const uint64_t rows = valid_row; + const uint64_t physical_cols = physical_col; + if (rows == 0u || cols == 0u || physical_cols < cols) + return out; + if (rows > (UINT64_MAX / physical_cols)) + return out; + if (rows * physical_cols > max_elems) + return out; + + const uint64_t row_span_bits = cols * elem_bits; + const uint64_t row_span_bytes = (row_span_bits + 7u) / 8u; + stride_bytes = stride_bytes > 0 ? stride_bytes : row_span_bytes; + if (stride_bytes < row_span_bytes || + (elem_bytes != 0u && (stride_bytes % elem_bytes) != 0u)) { + return out; + } + + const uint8_t *src = reinterpret_cast(base); + for (uint64_t r = 0; r < rows; ++r) { + const uint64_t row_base = r * stride_bytes; + for (uint64_t c = 0; c < cols; ++c) { + const uint64_t idx64 = r * physical_cols + c; + if (idx64 >= kTileWords) + return out; + const unsigned idx = static_cast(idx64); + + uint32_t value = 0u; + if (elem_bits == 4u) { + const uint64_t byte_addr = row_base + (c >> 1u); + const uint8_t packed = src[byte_addr]; + value = ((c & 1u) == 0u) ? (packed & 0x0fu) : ((packed >> 4u) & 0x0fu); + } else if (elem_bytes == 1u) { + value = static_cast(src[row_base + c]); + } else if (elem_bytes == 2u) { + uint16_t v = 0u; + memcpy(&v, src + row_base + c * 2u, sizeof(v)); + value = static_cast(v); + } else if (elem_bytes == 4u) { + uint32_t v = 0u; + memcpy(&v, src + row_base + c * 4u, sizeof(v)); + value = v; + } else if (elem_bytes == 8u) { + uint64_t v = 0u; + memcpy(&v, src + row_base + c * 8u, sizeof(v)); + value = static_cast(v & 0xffffffffu); + } else { + return out; + } + out.words[idx] = value; + } + } + return out; +#else + return __builtin_linx_tile_tload(base, SizeCode, DType, Layout, valid_col, + valid_row, physical_col, stride_bytes); +#endif +} + +template +inline void tileTStore(void *base, RawTile tile, unsigned valid_col, + unsigned valid_row, unsigned physical_col, + uint64_t stride_bytes) { + static_assert(SizeCode >= 5u && SizeCode <= 8u, + "PTO Linx canonical v0.57: size_code must be in [5,8]"); +#if defined(PTO_HOST_SIM) + (void)Layout; + const uint64_t bytes64 = sizeBytesFromCode(SizeCode); + const unsigned elem_bytes = dtypeElemBytesForStorage(DType); + const unsigned elem_bits = dtypeElemBits(DType); + if (bytes64 == 0 || bytes64 > kMaxTileBytes || elem_bits == 0u || + (bytes64 % elem_bytes) != 0u) + return; + + const unsigned max_elems = dtypeElemCountForBytes(bytes64, DType); + const uint64_t cols = valid_col; + const uint64_t rows = valid_row; + const uint64_t physical_cols = physical_col; + if (rows == 0u || cols == 0u || physical_cols < cols) + return; + if (rows > (UINT64_MAX / physical_cols)) + return; + if (rows * physical_cols > max_elems) + return; + + const uint64_t row_span_bits = cols * elem_bits; + const uint64_t row_span_bytes = (row_span_bits + 7u) / 8u; + stride_bytes = stride_bytes > 0 ? stride_bytes : row_span_bytes; + if (stride_bytes < row_span_bytes || + (elem_bytes != 0u && (stride_bytes % elem_bytes) != 0u)) { + return; + } + + uint8_t *dst = reinterpret_cast(base); + for (uint64_t r = 0; r < rows; ++r) { + const uint64_t row_base = r * stride_bytes; + for (uint64_t c = 0; c < cols; ++c) { + const uint64_t idx64 = r * physical_cols + c; + if (idx64 >= kTileWords) + return; + const uint32_t value = tile.words[static_cast(idx64)]; + + if (elem_bits == 4u) { + const uint64_t byte_addr = row_base + (c >> 1u); + uint8_t packed = dst[byte_addr]; + const uint8_t nibble = static_cast(value & 0x0fu); + if ((c & 1u) == 0u) + packed = static_cast((packed & 0xf0u) | nibble); + else + packed = static_cast((packed & 0x0fu) | (nibble << 4u)); + dst[byte_addr] = packed; + } else if (elem_bytes == 1u) { + dst[row_base + c] = static_cast(value & 0xffu); + } else if (elem_bytes == 2u) { + const uint16_t v = static_cast(value & 0xffffu); + memcpy(dst + row_base + c * 2u, &v, sizeof(v)); + } else if (elem_bytes == 4u) { + memcpy(dst + row_base + c * 4u, &value, sizeof(value)); + } else if (elem_bytes == 8u) { + const uint64_t v = static_cast(value); + memcpy(dst + row_base + c * 8u, &v, sizeof(v)); + } else { + return; + } + } + } +#else + __builtin_linx_tile_tstore(base, tile, SizeCode, DType, Layout, valid_col, + valid_row, physical_col, stride_bytes); +#endif +} + +template +inline RawTile cubeMamulb(RawTile lhs, RawTile rhs) { + static_assert(M <= 0xffu && N <= 0xffu && K <= 0xffu, + "PTO Linx canonical v0.57: cube dimensions must fit u8"); +#if defined(PTO_HOST_SIM) + RawTile out{}; + for (unsigned i = 0; i < kTileWords; ++i) + out.words[i] = 0u; + + for (unsigned i = 0; i < M; ++i) { + for (unsigned j = 0; j < N; ++j) { + int64_t acc = 0; + for (unsigned k = 0; k < K; ++k) { + const unsigned a_idx = i * K + k; + const unsigned b_idx = k * N + j; + if (a_idx >= kTileWords || b_idx >= kTileWords) + continue; + const int32_t a = static_cast(lhs.words[a_idx]); + const int32_t b = static_cast(rhs.words[b_idx]); + acc += static_cast(a) * static_cast(b); + } + const unsigned out_idx = i * N + j; + if (out_idx < kTileWords) + out.words[out_idx] = static_cast(static_cast(acc)); + } + } + return out; +#else + return __builtin_linx_cube_mamulb(lhs, rhs, M, N, K); +#endif +} + +template +inline RawTile cubeMamulbAcc(RawTile acc, RawTile lhs, RawTile rhs) { + static_assert(M <= 0xffu && N <= 0xffu && K <= 0xffu, + "PTO Linx canonical v0.57: cube dimensions must fit u8"); +#if defined(PTO_HOST_SIM) + RawTile out = acc; + for (unsigned i = 0; i < M; ++i) { + for (unsigned j = 0; j < N; ++j) { + const unsigned out_idx = i * N + j; + int64_t sum = + (out_idx < kTileWords) ? static_cast(out.words[out_idx]) : 0; + for (unsigned k = 0; k < K; ++k) { + const unsigned a_idx = i * K + k; + const unsigned b_idx = k * N + j; + if (a_idx >= kTileWords || b_idx >= kTileWords) + continue; + const int32_t a = static_cast(lhs.words[a_idx]); + const int32_t b = static_cast(rhs.words[b_idx]); + sum += static_cast(a) * static_cast(b); + } + if (out_idx < kTileWords) + out.words[out_idx] = static_cast(static_cast(sum)); + } + } + return out; +#else + return __builtin_linx_cube_mamulb_acc(acc, lhs, rhs, M, N, K); +#endif +} + +template +inline RawTile teplUnary(RawTile src, unsigned valid_col, unsigned valid_row, + unsigned physical_col) { + static_assert(TileOpcode <= 0x3ffu, + "PTO Linx canonical v0.57: TEPL tile opcode must fit u10"); + static_assert(SizeCode >= 5u && SizeCode <= 8u, + "PTO Linx canonical v0.57: size_code must be in [5,8]"); +#if defined(PTO_HOST_SIM) + const uint64_t bytes64 = sizeBytesFromCode(SizeCode); + const unsigned elem_bytes = dtypeElemBytesForStorage(DType); + const unsigned carrier_elems = + (bytes64 == 0 || bytes64 > kMaxTileBytes || elem_bytes == 0) + ? 0u + : dtypeElemCountForBytes(bytes64, DType); + if (valid_col == 0u || valid_row == 0u || physical_col < valid_col || + physical_col == 0u || valid_row > carrier_elems / physical_col) + return RawTile{}; + RawTile packed{}; + for (unsigned r = 0; r < valid_row; ++r) + for (unsigned c = 0; c < valid_col; ++c) + packed.words[r * valid_col + c] = src.words[r * physical_col + c]; + RawTile packed_out = teplUnaryHost( + packed, valid_row * valid_col, valid_row, valid_col); + RawTile out{}; + if constexpr ((TileOpcode & 0x3ffu) == 0x01du) { + for (unsigned r = 0; r < valid_col; ++r) + for (unsigned c = 0; c < valid_row; ++c) + out.words[r * physical_col + c] = packed_out.words[r * valid_row + c]; + } else { + for (unsigned r = 0; r < valid_row; ++r) + for (unsigned c = 0; c < valid_col; ++c) + out.words[r * physical_col + c] = packed_out.words[r * valid_col + c]; + } + return out; +#else + return __builtin_linx_tepl_unary(src, TileOpcode, SizeCode, DType, valid_col, + valid_row, physical_col); +#endif +} + +template +inline RawTile teplBinary(RawTile lhs, RawTile rhs, unsigned valid_col, + unsigned valid_row, unsigned physical_col) { + static_assert(TileOpcode <= 0x3ffu, + "PTO Linx canonical v0.57: TEPL tile opcode must fit u10"); + static_assert(SizeCode >= 5u && SizeCode <= 8u, + "PTO Linx canonical v0.57: size_code must be in [5,8]"); +#if defined(PTO_HOST_SIM) + const uint64_t bytes64 = sizeBytesFromCode(SizeCode); + const unsigned elem_bytes = dtypeElemBytesForStorage(DType); + const unsigned carrier_elems = + (bytes64 == 0 || bytes64 > kMaxTileBytes || elem_bytes == 0) + ? 0u + : dtypeElemCountForBytes(bytes64, DType); + if (valid_col == 0u || valid_row == 0u || physical_col < valid_col || + physical_col == 0u || valid_row > carrier_elems / physical_col) + return RawTile{}; + RawTile packed_lhs{}; + RawTile packed_rhs{}; + for (unsigned r = 0; r < valid_row; ++r) + for (unsigned c = 0; c < valid_col; ++c) { + packed_lhs.words[r * valid_col + c] = lhs.words[r * physical_col + c]; + packed_rhs.words[r * valid_col + c] = rhs.words[r * physical_col + c]; + } + RawTile packed_out = teplBinaryHost(packed_lhs, packed_rhs, + valid_row * valid_col); + RawTile out{}; + for (unsigned r = 0; r < valid_row; ++r) + for (unsigned c = 0; c < valid_col; ++c) + out.words[r * physical_col + c] = packed_out.words[r * valid_col + c]; + return out; +#else + return __builtin_linx_tepl_binary(lhs, rhs, TileOpcode, SizeCode, DType, + valid_col, valid_row, physical_col); +#endif +} + +template +inline RawTile teplBinaryScalar(RawTile lhs, Scalar scalar, unsigned valid_col, + unsigned valid_row, unsigned physical_col) { + static_assert(TileOpcode <= 0x3ffu, + "PTO Linx canonical v0.57: TEPL tile opcode must fit u10"); + static_assert(SizeCode >= 5u && SizeCode <= 8u, + "PTO Linx canonical v0.57: size_code must be in [5,8]"); + static_assert(Mode == 1u, "PTO Linx canonical v0.57: tepl.binary.scalar " + "requires operand mode=VS(1)"); +#if defined(PTO_HOST_SIM) + RawTile rhs{}; + const uint64_t bytes64 = sizeBytesFromCode(SizeCode); + const unsigned elem_bytes = dtypeElemBytesForStorage(DType); + const unsigned elems = + (bytes64 == 0 || bytes64 > kMaxTileBytes || elem_bytes == 0) + ? 0u + : dtypeElemCountForBytes(bytes64, DType); + const long long bits = encodeScalar(scalar); + const uint32_t scalar_word = scalarToWordDType(bits, DType); + for (unsigned i = 0; i < elems && i < kTileWords; ++i) + rhs.words[i] = scalar_word; + return teplBinary(lhs, rhs, valid_col, valid_row, + physical_col); +#else + return __builtin_linx_tepl_binary_scalar(lhs, encodeScalar(scalar), + TileOpcode, SizeCode, DType, Mode, + valid_col, valid_row, physical_col); +#endif +} + +template +inline RawTile teplSplat(Scalar scalar, unsigned valid_col, unsigned valid_row, + unsigned physical_col) { + static_assert(TileOpcode <= 0x3ffu, + "PTO Linx canonical v0.57: TEPL tile opcode must fit u10"); + static_assert(SizeCode >= 5u && SizeCode <= 8u, + "PTO Linx canonical v0.57: size_code must be in [5,8]"); + static_assert( + Mode == 2u, + "PTO Linx canonical v0.57: tepl.splat requires operand mode=SV(2)"); +#if defined(PTO_HOST_SIM) + RawTile out{}; + for (unsigned i = 0; i < kTileWords; ++i) + out.words[i] = 0u; + + const uint64_t bytes64 = sizeBytesFromCode(SizeCode); + const unsigned elem_bytes = dtypeElemBytesForStorage(DType); + const unsigned elems = + (bytes64 == 0 || bytes64 > kMaxTileBytes || elem_bytes == 0) + ? 0u + : dtypeElemCountForBytes(bytes64, DType); + + if ((TileOpcode & 0x3ffu) != 0x019u) + return out; + + const long long bits = encodeScalar(scalar); + const uint32_t scalar_word = scalarToWordDType(bits, DType); + if (valid_col == 0u || valid_row == 0u || physical_col < valid_col || + physical_col == 0u || valid_row > elems / physical_col) + return RawTile{}; + for (unsigned r = 0; r < valid_row; ++r) + for (unsigned c = 0; c < valid_col; ++c) + out.words[r * physical_col + c] = scalar_word; + return out; +#else + return __builtin_linx_tepl_splat(encodeScalar(scalar), TileOpcode, SizeCode, + DType, Mode, valid_col, valid_row, + physical_col); +#endif +} + +template +inline RawTile tileTMov(RawTile src) { + static_assert(SizeCode >= 5u && SizeCode <= 8u, + "PTO Linx canonical v0.57: size_code must be in [5,8]"); + static_assert(HasLayout <= 1u, + "PTO Linx canonical v0.57: has_layout must be bool"); + static_assert(Mode <= 1u, + "PTO Linx canonical v0.57: tmov mode must be 0(V2V) or 1(A2V)"); +#if defined(PTO_HOST_SIM) + (void)DType; + (void)Layout; + (void)HasLayout; + (void)Mode; + return src; +#else + return __builtin_linx_tile_tmov(src, Mode, SizeCode, DType, Layout, + HasLayout); +#endif +} + +} // namespace detail +} // namespace linx +} // namespace pto + +#endif // PTO_LINX_IMPL_BACKEND_HPP diff --git a/benchmark/one-level-arch/kernels/README.md b/benchmark/one-level-arch/kernels/README.md index b7e1390d..0fb45528 100644 --- a/benchmark/one-level-arch/kernels/README.md +++ b/benchmark/one-level-arch/kernels/README.md @@ -10,6 +10,10 @@ for type/dimension parameterization. > tile 版算子（engram/mhc/moe/quant/transpose 五模块），已通过 linx 工具链编译+链接验证。 > 详见 [`deepseek/TileKernels迁移说明.md`](deepseek/TileKernels迁移说明.md) 与各模块 README。 +> **PTO-Kernel imports**: `pto_kernels/` contains nine source-backed, tile-only +> memory, elementwise, GEMM, and attention kernels imported at a pinned +> `LinxISA/PTO-Kernel` revision. See [`pto_kernels/README.md`](pto_kernels/README.md). + ### 1. Matmul — `matmul/` - `matmul.hpp` — general matrix multiply; FP32/FP16/FP8; mask, dynamic, vec variants; A/B tile reuse. - `matmul_mx.hpp` — MX quantized matmul; FP4×FP4, BF16×FP4 mixed precision; microscaling factors. diff --git a/benchmark/one-level-arch/kernels/pto_kernels/README.md b/benchmark/one-level-arch/kernels/pto_kernels/README.md new file mode 100644 index 00000000..3c7a23bb --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/README.md @@ -0,0 +1,24 @@ +# PTO-Kernel Imports + +This directory contains tile-only kernels migrated from +[`LinxISA/PTO-Kernel`](https://github.com/LinxISA/PTO-Kernel) revision +`0eb72cb20b0de99326d984a9a27ddb815e6e4c24`. + +The migration intentionally includes only sources whose data paths can be +written with the LinxISA PTO 0.57 allowlist. Scalar QEMU fallback branches were +removed. Tile-grid loops remain because they select independent global tensor +tiles; all element and matrix work is performed by PTO intrinsics. + +| Family | Migrated kernels | PTO operations | +| --- | --- | --- | +| Memory | `tload_store` | `TLOAD`, `TSTORE` | +| Elementwise | `add_custom` | `TLOAD`, `TADD`, `TSTORE` | +| Matmul | `gemm`, `gemm_basic`, `gemm_demo`, `gemm_performance`, `mamulb`, `tmatmul_acc` | `TLOAD`, `TMATMUL`, `TMATMUL_ACC`, `TCVT`, `TADD`, `TMULS`, `TSTORE` | +| Attention | `flash_attention` | `TLOAD`, `TMATMUL`, `TCVT`, `TADD`, `TSTORE` | + +`migration.json` records the exact upstream source path for every import and +pins the isolated support-header copies by SHA-256. Host-simulation code remains +available inside the upstream backend header for syntax testing, but the active +Linx benchmark Makefile must not enable it. +Run `python3 scripts/verify_pto_kernel_migration.py` from the repository root +to verify provenance coverage, wrappers, and intrinsic allowlist compliance. diff --git a/benchmark/one-level-arch/kernels/pto_kernels/attention/flash_attention.cpp b/benchmark/one-level-arch/kernels/pto_kernels/attention/flash_attention.cpp new file mode 100644 index 00000000..7b142663 --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/attention/flash_attention.cpp @@ -0,0 +1,101 @@ +// Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. +#include +#include +#include + +using namespace pto; + +namespace { + +constexpr int kS = kernels::shapes::kAttentionLargeSeq; +constexpr int kQD = kernels::shapes::kAttentionSmallQD; +constexpr int kVD = kernels::shapes::kAttentionVD; +constexpr int kTm = kernels::tiling::kFlashTileM; +constexpr int kTk = kernels::tiling::kFlashTileK; + +static_assert(kTm * kTk * kQD * static_cast(sizeof(int)) <= 4096, + "QK matmul footprint must fit <=4KB"); +static_assert(kTm * kVD * kTk * static_cast(sizeof(int)) <= 4096, + "WV matmul footprint must fit <=4KB"); +static_assert(kS % kTm == 0 && kS % kTk == 0, + "global sequence shape must be divisible by tile shape"); + +using gmQ = global_tensor>; +using gmK = global_tensor>; +using gmV = global_tensor>; +using gmO = global_tensor>; + +using tileQ = TileLeft; +using tileK = TileRight; +using tileV = TileRight; +using tileScoreAcc = TileAcc; +using tileScoreVec = Tile; +using tileScoreLeft = TileLeft; +using tileOutAcc = TileAcc; +using tileOutVec = Tile; + +using itQ = global_iterator; +using itK = global_iterator; +using itV = global_iterator; +using itO = global_iterator; + +} // namespace + +extern "C" void flash_attention_i32(int *q_ptr, int *k_ptr, int *v_ptr, + int *out_ptr) { + itQ gQ(q_ptr); + itK gK(k_ptr); + itV gV(v_ptr); + itO gO(out_ptr); + + constexpr int kQTiles = kS / kTm; + constexpr int kKTiles = kS / kTk; + + for (int qi = 0; qi < kQTiles; ++qi) { + tileQ q; + TLOAD(q, gQ(qi, 0)); + + tileK k0; + tileV v0; + TLOAD(k0, gK(0, 0)); + TLOAD(v0, gV(0, 0)); + + tileScoreAcc sAcc0; + tileScoreVec sVec0; + tileScoreLeft sLeft0; + TMATMUL(sAcc0, q, k0); + TCVT(sVec0, sAcc0); + TCVT(sLeft0, sVec0); + + tileOutAcc outAcc; + TMATMUL(outAcc, sLeft0, v0); + + for (int kj = 1; kj < kKTiles; ++kj) { + tileK k; + tileV v; + TLOAD(k, gK(0, kj)); + TLOAD(v, gV(kj, 0)); + + tileScoreAcc sAcc; + tileScoreVec sVec; + tileScoreLeft sLeft; + TMATMUL(sAcc, q, k); + TCVT(sVec, sAcc); + TCVT(sLeft, sVec); + + tileOutAcc pieceAcc; + tileOutVec outVec; + tileOutVec pieceVec; + tileOutVec merged; + TMATMUL(pieceAcc, sLeft, v); + TCVT(outVec, outAcc); + TCVT(pieceVec, pieceAcc); + TADD(merged, outVec, pieceVec); + TCVT(outAcc, merged); + } + + tileOutVec out; + TCVT(out, outAcc); + TSTORE(gO(qi, 0), out); + } +} diff --git a/benchmark/one-level-arch/kernels/pto_kernels/elementwise/add_custom.cpp b/benchmark/one-level-arch/kernels/pto_kernels/elementwise/add_custom.cpp new file mode 100644 index 00000000..08149fee --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/elementwise/add_custom.cpp @@ -0,0 +1,50 @@ +// Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. +#include +#include + +using namespace pto; + +namespace { + +constexpr int kRows = kernels::shapes::kMemoryRows; +constexpr int kCols = kernels::shapes::kMemoryCols; +using tile_vec_f32 = Tile; + +static_assert(tile_vec_f32::Rows * tile_vec_f32::Cols * + static_cast(sizeof(float)) == + 4096, + "tile must be exactly 4KB"); +static_assert(kRows % tile_vec_f32::Rows == 0 && + kCols % tile_vec_f32::Cols == 0, + "global tensor must be divisible by tile shape"); + +using gmX = global_tensor>; +using gmY = global_tensor>; +using gmZ = global_tensor>; + +using itX = global_iterator; +using itY = global_iterator; +using itZ = global_iterator; + +} // namespace + +extern "C" void add_custom_f32(float *x_ptr, float *y_ptr, float *z_ptr) { + itX gX(x_ptr); + itY gY(y_ptr); + itZ gZ(z_ptr); + + constexpr int kRowTiles = kRows / tile_vec_f32::Rows; + constexpr int kColTiles = kCols / tile_vec_f32::Cols; + + for (int tr = 0; tr < kRowTiles; ++tr) { + for (int tc = 0; tc < kColTiles; ++tc) { + tile_vec_f32 tx; + tile_vec_f32 ty; + tile_vec_f32 tz; + TLOAD(tx, gX(tr, tc)); + TLOAD(ty, gY(tr, tc)); + TADD(tz, tx, ty); + TSTORE(gZ(tr, tc), tz); + } + } +} diff --git a/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp b/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp new file mode 100644 index 00000000..665a4d96 --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp @@ -0,0 +1,74 @@ +// Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. +#include +#include +#include + +using namespace pto; + +namespace { + +constexpr int kM = kernels::shapes::kMatmulM; +constexpr int kN = kernels::shapes::kMatmulN; +constexpr int kK = kernels::shapes::kMatmulK; + +constexpr int kTM = kernels::tiling::kGemmTileM; +constexpr int kTN = kernels::tiling::kGemmTileN; +constexpr int kTK = kernels::tiling::kGemmTileK; + +static_assert(kTM * kTN * kTK * static_cast(sizeof(int)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); +static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + +using tileA = TileLeft; +using tileB = TileRight; +using tileAcc = TileAcc; +using tileVec = Tile; + +using gmA = global_tensor>; +using gmB = global_tensor>; +using gmC = global_tensor>; + +using itA = global_iterator; +using itB = global_iterator; +using itC = global_iterator; + +} // namespace + +extern "C" void gemm_i32(int *lhs_ptr, int *rhs_ptr, int *dst_ptr) { + itA gA(lhs_ptr); + itB gB(rhs_ptr); + itC gC(dst_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileAcc prodAcc; + TMATMUL(prodAcc, a0, b0); + + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(prodAcc, prodAcc, a, b); + } + + tileVec prod; + tileVec bias; + tileVec sum; + TCVT(prod, prodAcc); + TLOAD(bias, gC(mi, nj)); + TADD(sum, prod, bias); + TSTORE(gC(mi, nj), sum); + } + } +} diff --git a/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_basic.cpp b/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_basic.cpp new file mode 100644 index 00000000..92757d73 --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_basic.cpp @@ -0,0 +1,71 @@ +// Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. +#include +#include +#include + +using namespace pto; + +namespace { + +constexpr int kM = kernels::shapes::kMatmulM; +constexpr int kN = kernels::shapes::kMatmulN; +constexpr int kK = kernels::shapes::kMatmulK; + +constexpr int kTM = kernels::tiling::kGemmTileM; +constexpr int kTN = kernels::tiling::kGemmTileN; +constexpr int kTK = kernels::tiling::kGemmTileK; + +static_assert(kTM * kTN * kTK * static_cast(sizeof(float)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); +static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + +using tileA = TileLeft; +using tileB = TileRight; +using tileAcc = TileAcc; +using tileVec = Tile; + +using gmA = global_tensor>; +using gmB = global_tensor>; +using gmC = global_tensor>; + +using itA = global_iterator; +using itB = global_iterator; +using itC = global_iterator; + +} // namespace + +extern "C" void gemm_basic_f32(float *lhs_ptr, float *rhs_ptr, + float *dst_ptr) { + itA gA(lhs_ptr); + itB gB(rhs_ptr); + itC gC(dst_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileAcc acc; + TMATMUL(acc, a0, b0); + + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(acc, acc, a, b); + } + + tileVec out; + TCVT(out, acc); + TSTORE(gC(mi, nj), out); + } + } +} diff --git a/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp b/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp new file mode 100644 index 00000000..6b1fc411 --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp @@ -0,0 +1,74 @@ +// Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. +#include +#include +#include + +using namespace pto; + +namespace { + +constexpr int kM = kernels::shapes::kMatmulM; +constexpr int kN = kernels::shapes::kMatmulN; +constexpr int kK = kernels::shapes::kMatmulK; + +constexpr int kTM = kernels::tiling::kGemmTileM; +constexpr int kTN = kernels::tiling::kGemmTileN; +constexpr int kTK = kernels::tiling::kGemmTileK; +constexpr float kAlpha = 0.125f; + +static_assert(kTM * kTN * kTK * static_cast(sizeof(float)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); +static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + +using tileA = TileLeft; +using tileB = TileRight; +using tileAcc = TileAcc; +using tileVec = Tile; + +using gmA = global_tensor>; +using gmB = global_tensor>; +using gmC = global_tensor>; + +using itA = global_iterator; +using itB = global_iterator; +using itC = global_iterator; + +} // namespace + +extern "C" void gemm_demo_f32(float *out_ptr, float *a_ptr, float *b_ptr) { + itA gA(a_ptr); + itB gB(b_ptr); + itC gC(out_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileAcc acc; + TMATMUL(acc, a0, b0); + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(acc, acc, a, b); + } + + tileVec out; + tileVec scaled; + tileVec merged; + TCVT(out, acc); + TMULS(scaled, out, kAlpha); + TADD(merged, out, scaled); + TSTORE(gC(mi, nj), merged); + } + } +} diff --git a/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp b/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp new file mode 100644 index 00000000..6691158b --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp @@ -0,0 +1,80 @@ +// Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. +#include +#include +#include + +using namespace pto; + +namespace { + +constexpr int kM = kernels::shapes::kMatmulM; +constexpr int kN = kernels::shapes::kMatmulN; +constexpr int kK = kernels::shapes::kMatmulK; + +constexpr int kTM = kernels::tiling::kGemmTileM; +constexpr int kTN = kernels::tiling::kGemmTileN; +constexpr int kTK = kernels::tiling::kGemmTileK; + +static_assert(kTM * kTN * kTK * static_cast(sizeof(float)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); +static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + +using tileA = TileLeft; +using tileB = TileRight; +using tileAcc = TileAcc; +using tileVec = Tile; + +using gmA = global_tensor>; +using gmB = global_tensor>; +using gmC = global_tensor>; + +using itA = global_iterator; +using itB = global_iterator; +using itC = global_iterator; + +} // namespace + +extern "C" void gemm_performance_f32(float *lhs_ptr, float *rhs_ptr, + float *dst_ptr, int repeat_tiles) { + if (repeat_tiles <= 0) + repeat_tiles = 1; + + itA gA(lhs_ptr); + itB gB(rhs_ptr); + itC gC(dst_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int rep = 0; rep < repeat_tiles; ++rep) { + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileAcc acc; + TMATMUL(acc, a0, b0); + + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(acc, acc, a, b); + } + + tileVec out; + tileVec prev; + tileVec merged; + TCVT(out, acc); + TLOAD(prev, gC(mi, nj)); + TADD(merged, prev, out); + TSTORE(gC(mi, nj), merged); + } + } + } +} diff --git a/benchmark/one-level-arch/kernels/pto_kernels/matmul/mamulb.cpp b/benchmark/one-level-arch/kernels/pto_kernels/matmul/mamulb.cpp new file mode 100644 index 00000000..381840bc --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/matmul/mamulb.cpp @@ -0,0 +1,70 @@ +// Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. +#include +#include +#include + +using namespace pto; + +namespace { + +constexpr int kM = kernels::shapes::kMatmulM; +constexpr int kN = kernels::shapes::kMatmulN; +constexpr int kK = kernels::shapes::kMatmulK; + +constexpr int kTM = kernels::tiling::kGemmTileM; +constexpr int kTN = kernels::tiling::kGemmTileN; +constexpr int kTK = kernels::tiling::kGemmTileK; + +static_assert(kTM * kTN * kTK * static_cast(sizeof(int)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); +static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + +using tileA = TileLeft; +using tileB = TileRight; +using tileCAcc = TileAcc; +using tileCVec = Tile; + +using gmA = global_tensor>; +using gmB = global_tensor>; +using gmC = global_tensor>; + +using itA = global_iterator; +using itB = global_iterator; +using itC = global_iterator; + +} // namespace + +extern "C" void mamulb_i32(int *lhs_ptr, int *rhs_ptr, int *dst_ptr) { + itA gA(lhs_ptr); + itB gB(rhs_ptr); + itC gC(dst_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileCAcc acc; + TMATMUL(acc, a0, b0); + + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(acc, acc, a, b); + } + + tileCVec out; + TCVT(out, acc); + TSTORE(gC(mi, nj), out); + } + } +} diff --git a/benchmark/one-level-arch/kernels/pto_kernels/matmul/tmatmul_acc.cpp b/benchmark/one-level-arch/kernels/pto_kernels/matmul/tmatmul_acc.cpp new file mode 100644 index 00000000..50ceac29 --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/matmul/tmatmul_acc.cpp @@ -0,0 +1,70 @@ +// Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. +#include +#include +#include + +using namespace pto; + +namespace { + +constexpr int kM = kernels::shapes::kMatmulM; +constexpr int kN = kernels::shapes::kMatmulN; +constexpr int kK = kernels::shapes::kMatmulK; + +constexpr int kTM = kernels::tiling::kGemmTileM; +constexpr int kTN = kernels::tiling::kGemmTileN; +constexpr int kTK = kernels::tiling::kGemmTileK; + +static_assert(kTM * kTN * kTK * static_cast(sizeof(int)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); +static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + +using tileA = TileLeft; +using tileB = TileRight; +using tileAcc = TileAcc; +using tileVec = Tile; + +using gmA = global_tensor>; +using gmB = global_tensor>; +using gmC = global_tensor>; + +using itA = global_iterator; +using itB = global_iterator; +using itC = global_iterator; + +} // namespace + +extern "C" void tmatmul_acc_i32(int *lhs_ptr, int *rhs_ptr, int *dst_ptr) { + itA gA(lhs_ptr); + itB gB(rhs_ptr); + itC gC(dst_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileAcc cAcc; + TMATMUL(cAcc, a0, b0); + + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(cAcc, cAcc, a, b); + } + + tileVec cVec; + TCVT(cVec, cAcc); + TSTORE(gC(mi, nj), cVec); + } + } +} diff --git a/benchmark/one-level-arch/kernels/pto_kernels/memory/tload_store.cpp b/benchmark/one-level-arch/kernels/pto_kernels/memory/tload_store.cpp new file mode 100644 index 00000000..03b616e3 --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/memory/tload_store.cpp @@ -0,0 +1,43 @@ +// Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. +#include +#include + +using namespace pto; + +namespace { + +constexpr int kRows = kernels::shapes::kMemoryRows; +constexpr int kCols = kernels::shapes::kMemoryCols; +using tile_vec_i32 = Tile; + +static_assert(tile_vec_i32::Rows * tile_vec_i32::Cols * + static_cast(sizeof(int)) == + 4096, + "tile must be exactly 4KB"); +static_assert(kRows % tile_vec_i32::Rows == 0 && + kCols % tile_vec_i32::Cols == 0, + "global tensor must be divisible by tile shape"); + +using gmSrc = global_tensor>; +using gmDst = global_tensor>; + +using itSrc = global_iterator; +using itDst = global_iterator; + +} // namespace + +extern "C" void tload_store_i32(int *src_ptr, int *dst_ptr) { + itSrc gSrc(src_ptr); + itDst gDst(dst_ptr); + + constexpr int kRowTiles = kRows / tile_vec_i32::Rows; + constexpr int kColTiles = kCols / tile_vec_i32::Cols; + + for (int tr = 0; tr < kRowTiles; ++tr) { + for (int tc = 0; tc < kColTiles; ++tc) { + tile_vec_i32 tile; + TLOAD(tile, gSrc(tr, tc)); + TSTORE(gDst(tr, tc), tile); + } + } +} diff --git a/benchmark/one-level-arch/kernels/pto_kernels/migration.json b/benchmark/one-level-arch/kernels/pto_kernels/migration.json new file mode 100644 index 00000000..9d6451f3 --- /dev/null +++ b/benchmark/one-level-arch/kernels/pto_kernels/migration.json @@ -0,0 +1,25 @@ +{ + "schema": 1, + "upstream": "https://github.com/LinxISA/PTO-Kernel", + "revision": "0eb72cb20b0de99326d984a9a27ddb815e6e4c24", + "policy": "PTO 0.57 allowlist only; scalar QEMU fallback paths removed", + "support_headers": [ + {"source": "common/linx_lowp_types.hpp", "upstream_source": "include/common/linx_lowp_types.hpp", "sha256": "dddda37f13a8fd6c35659afbf2075359fb7218bb709ee81851a5e8d525b5888d"}, + {"source": "common/pto_tileop.hpp", "upstream_source": "include/common/pto_tileop.hpp", "sha256": "5566c25f05e0a8c414b0cc34aa15b84356da6c6e3598c35a48a362c887b29229"}, + {"source": "common/runtime/kernel_env.hpp", "upstream_source": "include/common/runtime/kernel_env.hpp", "sha256": "377544f7b177104295a93becfa34a5f4417d3b59e889ce18574642c907e26ec3"}, + {"source": "common/runtime/kernel_shapes.hpp", "upstream_source": "include/common/runtime/kernel_shapes.hpp", "sha256": "5d8cfac46f16b06cff219dbf20aa51c536afc161389c19eea4396db2d196cda4"}, + {"source": "common/runtime/kernel_tiling.hpp", "upstream_source": "include/common/runtime/kernel_tiling.hpp", "sha256": "ebfea19f799a8af114db33bed8232bac1c96dd88af80d00ee33093fcc3577450"}, + {"source": "pto/linx/impl/backend.hpp", "upstream_source": "include/pto/linx/impl/backend.hpp", "sha256": "ae926652c0a706aedf2d30bd384570a090f72290d48a699ac269e1bb5912cc2c"} + ], + "kernels": [ + {"benchmark": "pto_tload_store", "source": "memory/tload_store.cpp", "upstream_source": "kernels/memory/tload_store.cpp"}, + {"benchmark": "pto_add", "source": "elementwise/add_custom.cpp", "upstream_source": "kernels/elementwise/add_custom.cpp"}, + {"benchmark": "pto_gemm", "source": "matmul/gemm.cpp", "upstream_source": "kernels/matmul/gemm.cpp"}, + {"benchmark": "pto_gemm_basic", "source": "matmul/gemm_basic.cpp", "upstream_source": "kernels/matmul/gemm_basic.cpp"}, + {"benchmark": "pto_gemm_demo", "source": "matmul/gemm_demo.cpp", "upstream_source": "kernels/matmul/gemm_demo.cpp"}, + {"benchmark": "pto_gemm_performance", "source": "matmul/gemm_performance.cpp", "upstream_source": "kernels/matmul/gemm_performance.cpp"}, + {"benchmark": "pto_mamulb", "source": "matmul/mamulb.cpp", "upstream_source": "kernels/matmul/mamulb.cpp"}, + {"benchmark": "pto_tmatmul_acc", "source": "matmul/tmatmul_acc.cpp", "upstream_source": "kernels/matmul/tmatmul_acc.cpp"}, + {"benchmark": "pto_flash_attention", "source": "attention/flash_attention.cpp", "upstream_source": "kernels/attention/flash_attention.cpp"} + ] +} diff --git a/benchmark/one-level-arch/test/kernel/README.md b/benchmark/one-level-arch/test/kernel/README.md index 9660bd85..87f50a5b 100644 --- a/benchmark/one-level-arch/test/kernel/README.md +++ b/benchmark/one-level-arch/test/kernel/README.md @@ -11,7 +11,8 @@ test/kernel/ ├── broadcast/ element_wise/ matmul/ ├── concat/ fa/ reduction/{reducemax_col,row,...} ├── control/ gather/ sort/ -└── transpose/ +├── pto_kernels/ transpose/ +└── reduction/ ``` ## Operator Test Status @@ -28,6 +29,7 @@ test/kernel/ | concat | 4 | ✓ | gather/scatter | | control | 1 | △ | pure tile-op; run gfsim with `-s core.singleTierMode=true`; `.data` via `gen_data.py` | | sort | 1 | △ | topk | +| pto_kernels | 9 | ✓ | Tile-only imports pinned to PTO-Kernel; strict 0.57 allowlist | (Configs reflect `compile.all` typical scenarios; `△` = compiles but needs special run flags / generated data.) diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/Makefile b/benchmark/one-level-arch/test/kernel/pto_kernels/Makefile new file mode 100644 index 00000000..d7b97ac8 --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/Makefile @@ -0,0 +1,5 @@ +DEFINES += -DPTO_QEMU_SMOKE=1 +TARGET = $(ELF_HEAD)_$(TESTCASE).elf +SRC_FILE = $(TEST_ROOT)/$(CASE_SRC_DIR)/$(TESTCASE).cpp + +include ../../common/Makefile.common diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/compile.all b/benchmark/one-level-arch/test/kernel/pto_kernels/compile.all new file mode 100755 index 00000000..92bcbac3 --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/compile.all @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${COMPILER_DIR:?Set COMPILER_DIR to the in-repo Linx compiler bin directory}" + +make TESTCASE=pto_tload_store COMPILER_DIR="$COMPILER_DIR" diss +make TESTCASE=pto_add COMPILER_DIR="$COMPILER_DIR" diss +make TESTCASE=pto_gemm COMPILER_DIR="$COMPILER_DIR" diss +make TESTCASE=pto_gemm_basic COMPILER_DIR="$COMPILER_DIR" diss +make TESTCASE=pto_gemm_demo COMPILER_DIR="$COMPILER_DIR" diss +make TESTCASE=pto_gemm_performance COMPILER_DIR="$COMPILER_DIR" diss +make TESTCASE=pto_mamulb COMPILER_DIR="$COMPILER_DIR" diss +make TESTCASE=pto_tmatmul_acc COMPILER_DIR="$COMPILER_DIR" diss +make TESTCASE=pto_flash_attention COMPILER_DIR="$COMPILER_DIR" diss diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_add.cpp b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_add.cpp new file mode 100644 index 00000000..fe488cd2 --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_add.cpp @@ -0,0 +1,11 @@ +#include "pto_kernels/elementwise/add_custom.cpp" + +int main() { + constexpr int elements = pto::kernels::shapes::kMemoryRows * + pto::kernels::shapes::kMemoryCols; + alignas(64) static float x[elements]{}; + alignas(64) static float y[elements]{}; + alignas(64) static float z[elements]{}; + add_custom_f32(x, y, z); + return 0; +} diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_flash_attention.cpp b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_flash_attention.cpp new file mode 100644 index 00000000..24467120 --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_flash_attention.cpp @@ -0,0 +1,13 @@ +#include "pto_kernels/attention/flash_attention.cpp" + +int main() { + constexpr int sequence = pto::kernels::shapes::kAttentionLargeSeq; + constexpr int query_depth = pto::kernels::shapes::kAttentionSmallQD; + constexpr int value_depth = pto::kernels::shapes::kAttentionVD; + alignas(64) static int query[sequence * query_depth]{}; + alignas(64) static int key[query_depth * sequence]{}; + alignas(64) static int value[sequence * value_depth]{}; + alignas(64) static int output[sequence * value_depth]{}; + flash_attention_i32(query, key, value, output); + return 0; +} diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm.cpp b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm.cpp new file mode 100644 index 00000000..ae3473b5 --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm.cpp @@ -0,0 +1,12 @@ +#include "pto_kernels/matmul/gemm.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static int lhs[m * k]{}; + alignas(64) static int rhs[k * n]{}; + alignas(64) static int dst[m * n]{}; + gemm_i32(lhs, rhs, dst); + return 0; +} diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_basic.cpp b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_basic.cpp new file mode 100644 index 00000000..c0893d6d --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_basic.cpp @@ -0,0 +1,12 @@ +#include "pto_kernels/matmul/gemm_basic.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static float lhs[m * k]{}; + alignas(64) static float rhs[k * n]{}; + alignas(64) static float dst[m * n]{}; + gemm_basic_f32(lhs, rhs, dst); + return 0; +} diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_demo.cpp b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_demo.cpp new file mode 100644 index 00000000..f36afa7e --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_demo.cpp @@ -0,0 +1,12 @@ +#include "pto_kernels/matmul/gemm_demo.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static float lhs[m * k]{}; + alignas(64) static float rhs[k * n]{}; + alignas(64) static float dst[m * n]{}; + gemm_demo_f32(dst, lhs, rhs); + return 0; +} diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_performance.cpp b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_performance.cpp new file mode 100644 index 00000000..14614ed1 --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_performance.cpp @@ -0,0 +1,12 @@ +#include "pto_kernels/matmul/gemm_performance.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static float lhs[m * k]{}; + alignas(64) static float rhs[k * n]{}; + alignas(64) static float dst[m * n]{}; + gemm_performance_f32(lhs, rhs, dst, 2); + return 0; +} diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_mamulb.cpp b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_mamulb.cpp new file mode 100644 index 00000000..759ec1f4 --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_mamulb.cpp @@ -0,0 +1,12 @@ +#include "pto_kernels/matmul/mamulb.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static int lhs[m * k]{}; + alignas(64) static int rhs[k * n]{}; + alignas(64) static int dst[m * n]{}; + mamulb_i32(lhs, rhs, dst); + return 0; +} diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tload_store.cpp b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tload_store.cpp new file mode 100644 index 00000000..a6c400ae --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tload_store.cpp @@ -0,0 +1,10 @@ +#include "pto_kernels/memory/tload_store.cpp" + +int main() { + constexpr int elements = pto::kernels::shapes::kMemoryRows * + pto::kernels::shapes::kMemoryCols; + alignas(64) static int source[elements]{}; + alignas(64) static int destination[elements]{}; + tload_store_i32(source, destination); + return 0; +} diff --git a/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tmatmul_acc.cpp b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tmatmul_acc.cpp new file mode 100644 index 00000000..2ffd6c60 --- /dev/null +++ b/benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tmatmul_acc.cpp @@ -0,0 +1,12 @@ +#include "pto_kernels/matmul/tmatmul_acc.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static int lhs[m * k]{}; + alignas(64) static int rhs[k * n]{}; + alignas(64) static int dst[m * n]{}; + tmatmul_acc_i32(lhs, rhs, dst); + return 0; +} diff --git a/docs/benchmarks/catalog.json b/docs/benchmarks/catalog.json index 8f06b24b..72f980c1 100644 --- a/docs/benchmarks/catalog.json +++ b/docs/benchmarks/catalog.json @@ -1,7 +1,7 @@ { "schema": 3, - "active_build_variants": 53, - "source_implementations": 20, + "active_build_variants": 62, + "source_implementations": 29, "families": [ { "backend": "One-level", @@ -151,6 +151,22 @@ "TSTORE" ] }, + { + "backend": "One-level", + "family": "pto_kernels", + "page": "benchmarks/catalog/one-level/pto-kernels/index.md", + "implementations": 9, + "variants": 9, + "intrinsics": [ + "TADD", + "TCVT", + "TLOAD", + "TMATMUL", + "TMATMUL_ACC", + "TMULS", + "TSTORE" + ] + }, { "backend": "One-level", "family": "reduction/reducemax_col", @@ -2850,6 +2866,326 @@ }, "helpers": [] }, + { + "backend": "One-level", + "family": "pto_kernels", + "manifest": "benchmark/one-level-arch/test/kernel/pto_kernels/compile.all", + "manifest_line": 6, + "command": "make TESTCASE=pto_tload_store COMPILER_DIR=\"$COMPILER_DIR\" diss", + "testcase": "pto_tload_store", + "source": "benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tload_store.cpp", + "page": "benchmarks/catalog/one-level/pto-kernels/pto-tload-store-1e107659.md", + "intrinsic_surface_scope": "source-union", + "source_intrinsics": [ + "TLOAD", + "TSTORE" + ], + "intrinsic_forms": { + "TLOAD": [ + "TLOAD" + ], + "TSTORE": [ + "TSTORE" + ] + }, + "helpers": [] + }, + { + "backend": "One-level", + "family": "pto_kernels", + "manifest": "benchmark/one-level-arch/test/kernel/pto_kernels/compile.all", + "manifest_line": 7, + "command": "make TESTCASE=pto_add COMPILER_DIR=\"$COMPILER_DIR\" diss", + "testcase": "pto_add", + "source": "benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_add.cpp", + "page": "benchmarks/catalog/one-level/pto-kernels/pto-add-6e2ae6dc.md", + "intrinsic_surface_scope": "source-union", + "source_intrinsics": [ + "TADD", + "TLOAD", + "TSTORE" + ], + "intrinsic_forms": { + "TLOAD": [ + "TLOAD" + ], + "TADD": [ + "TADD" + ], + "TSTORE": [ + "TSTORE" + ] + }, + "helpers": [] + }, + { + "backend": "One-level", + "family": "pto_kernels", + "manifest": "benchmark/one-level-arch/test/kernel/pto_kernels/compile.all", + "manifest_line": 8, + "command": "make TESTCASE=pto_gemm COMPILER_DIR=\"$COMPILER_DIR\" diss", + "testcase": "pto_gemm", + "source": "benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm.cpp", + "page": "benchmarks/catalog/one-level/pto-kernels/pto-gemm-eb591839.md", + "intrinsic_surface_scope": "source-union", + "source_intrinsics": [ + "TADD", + "TCVT", + "TLOAD", + "TMATMUL", + "TMATMUL_ACC", + "TSTORE" + ], + "intrinsic_forms": { + "TLOAD": [ + "TLOAD" + ], + "TMATMUL": [ + "TMATMUL" + ], + "TMATMUL_ACC": [ + "TMATMUL_ACC" + ], + "TCVT": [ + "TCVT" + ], + "TADD": [ + "TADD" + ], + "TSTORE": [ + "TSTORE" + ] + }, + "helpers": [] + }, + { + "backend": "One-level", + "family": "pto_kernels", + "manifest": "benchmark/one-level-arch/test/kernel/pto_kernels/compile.all", + "manifest_line": 9, + "command": "make TESTCASE=pto_gemm_basic COMPILER_DIR=\"$COMPILER_DIR\" diss", + "testcase": "pto_gemm_basic", + "source": "benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_basic.cpp", + "page": "benchmarks/catalog/one-level/pto-kernels/pto-gemm-basic-a2ba2c9c.md", + "intrinsic_surface_scope": "source-union", + "source_intrinsics": [ + "TCVT", + "TLOAD", + "TMATMUL", + "TMATMUL_ACC", + "TSTORE" + ], + "intrinsic_forms": { + "TLOAD": [ + "TLOAD" + ], + "TMATMUL": [ + "TMATMUL" + ], + "TMATMUL_ACC": [ + "TMATMUL_ACC" + ], + "TCVT": [ + "TCVT" + ], + "TSTORE": [ + "TSTORE" + ] + }, + "helpers": [] + }, + { + "backend": "One-level", + "family": "pto_kernels", + "manifest": "benchmark/one-level-arch/test/kernel/pto_kernels/compile.all", + "manifest_line": 10, + "command": "make TESTCASE=pto_gemm_demo COMPILER_DIR=\"$COMPILER_DIR\" diss", + "testcase": "pto_gemm_demo", + "source": "benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_demo.cpp", + "page": "benchmarks/catalog/one-level/pto-kernels/pto-gemm-demo-85ba9afc.md", + "intrinsic_surface_scope": "source-union", + "source_intrinsics": [ + "TADD", + "TCVT", + "TLOAD", + "TMATMUL", + "TMATMUL_ACC", + "TMULS", + "TSTORE" + ], + "intrinsic_forms": { + "TLOAD": [ + "TLOAD" + ], + "TMATMUL": [ + "TMATMUL" + ], + "TMATMUL_ACC": [ + "TMATMUL_ACC" + ], + "TCVT": [ + "TCVT" + ], + "TMULS": [ + "TMULS" + ], + "TADD": [ + "TADD" + ], + "TSTORE": [ + "TSTORE" + ] + }, + "helpers": [] + }, + { + "backend": "One-level", + "family": "pto_kernels", + "manifest": "benchmark/one-level-arch/test/kernel/pto_kernels/compile.all", + "manifest_line": 11, + "command": "make TESTCASE=pto_gemm_performance COMPILER_DIR=\"$COMPILER_DIR\" diss", + "testcase": "pto_gemm_performance", + "source": "benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_performance.cpp", + "page": "benchmarks/catalog/one-level/pto-kernels/pto-gemm-performance-ffe72cfc.md", + "intrinsic_surface_scope": "source-union", + "source_intrinsics": [ + "TADD", + "TCVT", + "TLOAD", + "TMATMUL", + "TMATMUL_ACC", + "TSTORE" + ], + "intrinsic_forms": { + "TLOAD": [ + "TLOAD" + ], + "TMATMUL": [ + "TMATMUL" + ], + "TMATMUL_ACC": [ + "TMATMUL_ACC" + ], + "TCVT": [ + "TCVT" + ], + "TADD": [ + "TADD" + ], + "TSTORE": [ + "TSTORE" + ] + }, + "helpers": [] + }, + { + "backend": "One-level", + "family": "pto_kernels", + "manifest": "benchmark/one-level-arch/test/kernel/pto_kernels/compile.all", + "manifest_line": 12, + "command": "make TESTCASE=pto_mamulb COMPILER_DIR=\"$COMPILER_DIR\" diss", + "testcase": "pto_mamulb", + "source": "benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_mamulb.cpp", + "page": "benchmarks/catalog/one-level/pto-kernels/pto-mamulb-e9f16fd1.md", + "intrinsic_surface_scope": "source-union", + "source_intrinsics": [ + "TCVT", + "TLOAD", + "TMATMUL", + "TMATMUL_ACC", + "TSTORE" + ], + "intrinsic_forms": { + "TLOAD": [ + "TLOAD" + ], + "TMATMUL": [ + "TMATMUL" + ], + "TMATMUL_ACC": [ + "TMATMUL_ACC" + ], + "TCVT": [ + "TCVT" + ], + "TSTORE": [ + "TSTORE" + ] + }, + "helpers": [] + }, + { + "backend": "One-level", + "family": "pto_kernels", + "manifest": "benchmark/one-level-arch/test/kernel/pto_kernels/compile.all", + "manifest_line": 13, + "command": "make TESTCASE=pto_tmatmul_acc COMPILER_DIR=\"$COMPILER_DIR\" diss", + "testcase": "pto_tmatmul_acc", + "source": "benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tmatmul_acc.cpp", + "page": "benchmarks/catalog/one-level/pto-kernels/pto-tmatmul-acc-e3e7d902.md", + "intrinsic_surface_scope": "source-union", + "source_intrinsics": [ + "TCVT", + "TLOAD", + "TMATMUL", + "TMATMUL_ACC", + "TSTORE" + ], + "intrinsic_forms": { + "TLOAD": [ + "TLOAD" + ], + "TMATMUL": [ + "TMATMUL" + ], + "TMATMUL_ACC": [ + "TMATMUL_ACC" + ], + "TCVT": [ + "TCVT" + ], + "TSTORE": [ + "TSTORE" + ] + }, + "helpers": [] + }, + { + "backend": "One-level", + "family": "pto_kernels", + "manifest": "benchmark/one-level-arch/test/kernel/pto_kernels/compile.all", + "manifest_line": 14, + "command": "make TESTCASE=pto_flash_attention COMPILER_DIR=\"$COMPILER_DIR\" diss", + "testcase": "pto_flash_attention", + "source": "benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_flash_attention.cpp", + "page": "benchmarks/catalog/one-level/pto-kernels/pto-flash-attention-e05ac981.md", + "intrinsic_surface_scope": "source-union", + "source_intrinsics": [ + "TADD", + "TCVT", + "TLOAD", + "TMATMUL", + "TSTORE" + ], + "intrinsic_forms": { + "TLOAD": [ + "TLOAD" + ], + "TMATMUL": [ + "TMATMUL" + ], + "TCVT": [ + "TCVT" + ], + "TADD": [ + "TADD" + ], + "TSTORE": [ + "TSTORE" + ] + }, + "helpers": [] + }, { "backend": "One-level", "family": "reduction/reducemax_col", diff --git a/docs/benchmarks/catalog/one-level/pto-kernels/index.md b/docs/benchmarks/catalog/one-level/pto-kernels/index.md new file mode 100644 index 00000000..2b1122b4 --- /dev/null +++ b/docs/benchmarks/catalog/one-level/pto-kernels/index.md @@ -0,0 +1,18 @@ +# One-level: pto_kernels + + + +This family contains **9** source implementations and +**9** active build variants. + +| Implementation | Source | Variants | PTO source-union surface | +| --- | --- | ---: | --- | +| [pto_add](pto-add-6e2ae6dc.md) | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_add.cpp` | 1 | [`TADD`](../../../../intrinsics/tadd.md), [`TLOAD`](../../../../intrinsics/tload.md), [`TSTORE`](../../../../intrinsics/tstore.md) | +| [pto_flash_attention](pto-flash-attention-e05ac981.md) | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_flash_attention.cpp` | 1 | [`TADD`](../../../../intrinsics/tadd.md), [`TCVT`](../../../../intrinsics/tcvt.md), [`TLOAD`](../../../../intrinsics/tload.md), [`TMATMUL`](../../../../intrinsics/tmatmul.md), [`TSTORE`](../../../../intrinsics/tstore.md) | +| [pto_gemm](pto-gemm-eb591839.md) | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm.cpp` | 1 | [`TADD`](../../../../intrinsics/tadd.md), [`TCVT`](../../../../intrinsics/tcvt.md), [`TLOAD`](../../../../intrinsics/tload.md), [`TMATMUL`](../../../../intrinsics/tmatmul.md), [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md), [`TSTORE`](../../../../intrinsics/tstore.md) | +| [pto_gemm_basic](pto-gemm-basic-a2ba2c9c.md) | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_basic.cpp` | 1 | [`TCVT`](../../../../intrinsics/tcvt.md), [`TLOAD`](../../../../intrinsics/tload.md), [`TMATMUL`](../../../../intrinsics/tmatmul.md), [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md), [`TSTORE`](../../../../intrinsics/tstore.md) | +| [pto_gemm_demo](pto-gemm-demo-85ba9afc.md) | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_demo.cpp` | 1 | [`TADD`](../../../../intrinsics/tadd.md), [`TCVT`](../../../../intrinsics/tcvt.md), [`TLOAD`](../../../../intrinsics/tload.md), [`TMATMUL`](../../../../intrinsics/tmatmul.md), [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md), [`TMULS`](../../../../intrinsics/tmuls.md), [`TSTORE`](../../../../intrinsics/tstore.md) | +| [pto_gemm_performance](pto-gemm-performance-ffe72cfc.md) | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_performance.cpp` | 1 | [`TADD`](../../../../intrinsics/tadd.md), [`TCVT`](../../../../intrinsics/tcvt.md), [`TLOAD`](../../../../intrinsics/tload.md), [`TMATMUL`](../../../../intrinsics/tmatmul.md), [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md), [`TSTORE`](../../../../intrinsics/tstore.md) | +| [pto_mamulb](pto-mamulb-e9f16fd1.md) | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_mamulb.cpp` | 1 | [`TCVT`](../../../../intrinsics/tcvt.md), [`TLOAD`](../../../../intrinsics/tload.md), [`TMATMUL`](../../../../intrinsics/tmatmul.md), [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md), [`TSTORE`](../../../../intrinsics/tstore.md) | +| [pto_tload_store](pto-tload-store-1e107659.md) | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tload_store.cpp` | 1 | [`TLOAD`](../../../../intrinsics/tload.md), [`TSTORE`](../../../../intrinsics/tstore.md) | +| [pto_tmatmul_acc](pto-tmatmul-acc-e3e7d902.md) | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tmatmul_acc.cpp` | 1 | [`TCVT`](../../../../intrinsics/tcvt.md), [`TLOAD`](../../../../intrinsics/tload.md), [`TMATMUL`](../../../../intrinsics/tmatmul.md), [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md), [`TSTORE`](../../../../intrinsics/tstore.md) | diff --git a/docs/benchmarks/catalog/one-level/pto-kernels/pto-add-6e2ae6dc.md b/docs/benchmarks/catalog/one-level/pto-kernels/pto-add-6e2ae6dc.md new file mode 100644 index 00000000..93484095 --- /dev/null +++ b/docs/benchmarks/catalog/one-level/pto-kernels/pto-add-6e2ae6dc.md @@ -0,0 +1,120 @@ +# pto_add: pto_add.cpp + + + +| Field | Value | +| --- | --- | +| Benchmark surface | One-level | +| Family | `pto_kernels` | +| Implementation source | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_add.cpp` | +| Active build variants | 1 | + +## How the Code Is Written + +The implementation loads or gathers global data into typed tiles, then applies vector elementwise arithmetic, then commits the result to global memory. + +```cpp title="pto_add.cpp" linenums="1" +#include "pto_kernels/elementwise/add_custom.cpp" + +int main() { + constexpr int elements = pto::kernels::shapes::kMemoryRows * + pto::kernels::shapes::kMemoryCols; + alignas(64) static float x[elements]{}; + alignas(64) static float y[elements]{}; + alignas(64) static float z[elements]{}; + add_custom_f32(x, y, z); + return 0; +} +``` + +## PTO-Bearing Local Implementations + +These local include files contain the PTO calls reached by this entrypoint. + +??? code "benchmark/one-level-arch/kernels/pto_kernels/elementwise/add_custom.cpp" + + ```cpp title="add_custom.cpp" linenums="1" + // Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. + #include + #include + + using namespace pto; + + namespace { + + constexpr int kRows = kernels::shapes::kMemoryRows; + constexpr int kCols = kernels::shapes::kMemoryCols; + using tile_vec_f32 = Tile; + + static_assert(tile_vec_f32::Rows * tile_vec_f32::Cols * + static_cast(sizeof(float)) == + 4096, + "tile must be exactly 4KB"); + static_assert(kRows % tile_vec_f32::Rows == 0 && + kCols % tile_vec_f32::Cols == 0, + "global tensor must be divisible by tile shape"); + + using gmX = global_tensor>; + using gmY = global_tensor>; + using gmZ = global_tensor>; + + using itX = global_iterator; + using itY = global_iterator; + using itZ = global_iterator; + + } // namespace + + extern "C" void add_custom_f32(float *x_ptr, float *y_ptr, float *z_ptr) { + itX gX(x_ptr); + itY gY(y_ptr); + itZ gZ(z_ptr); + + constexpr int kRowTiles = kRows / tile_vec_f32::Rows; + constexpr int kColTiles = kCols / tile_vec_f32::Cols; + + for (int tr = 0; tr < kRowTiles; ++tr) { + for (int tc = 0; tc < kColTiles; ++tc) { + tile_vec_f32 tx; + tile_vec_f32 ty; + tile_vec_f32 tz; + TLOAD(tx, gX(tr, tc)); + TLOAD(ty, gY(tr, tc)); + TADD(tz, tx, ty); + TSTORE(gZ(tr, tc), tz); + } + } + } + ``` + +## Supported PTO Intrinsics + +This is the source-level union across the compile-time paths in this +implementation. A build command may select a subset through its macros; +the exact source spellings below preserve aliases and masked forms. + +| Intrinsic contract | Source spelling | Called from | +| --- | --- | --- | +| [`TADD`](../../../../intrinsics/tadd.md) | `TADD` | `benchmark/one-level-arch/kernels/pto_kernels/elementwise/add_custom.cpp` | +| [`TLOAD`](../../../../intrinsics/tload.md) | `TLOAD` | `benchmark/one-level-arch/kernels/pto_kernels/elementwise/add_custom.cpp` | +| [`TSTORE`](../../../../intrinsics/tstore.md) | `TSTORE` | `benchmark/one-level-arch/kernels/pto_kernels/elementwise/add_custom.cpp` | + +## Active Build Commands + +Run these from `benchmark/one-level-arch/test/kernel/pto_kernels` after setting +`COMPILER_DIR` and `LINX_SYSROOT` as described in the build guide. + +| Manifest line | Command | +| ---: | --- | +| 7 | `make TESTCASE=pto_add COMPILER_DIR="$COMPILER_DIR" diss` | + +## Resolved One-Level Source Closure + +??? info "7 one-level source files reached through local includes" + + - `benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp` + - `benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp` + - `benchmark/one-level-arch/kernels/pto_kernels/elementwise/add_custom.cpp` + - `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_add.cpp` diff --git a/docs/benchmarks/catalog/one-level/pto-kernels/pto-flash-attention-e05ac981.md b/docs/benchmarks/catalog/one-level/pto-kernels/pto-flash-attention-e05ac981.md new file mode 100644 index 00000000..582b884b --- /dev/null +++ b/docs/benchmarks/catalog/one-level/pto-kernels/pto-flash-attention-e05ac981.md @@ -0,0 +1,176 @@ +# pto_flash_attention: pto_flash_attention.cpp + + + +| Field | Value | +| --- | --- | +| Benchmark surface | One-level | +| Family | `pto_kernels` | +| Implementation source | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_flash_attention.cpp` | +| Active build variants | 1 | + +## How the Code Is Written + +The implementation loads or gathers global data into typed tiles, then executes matrix or matrix-vector work on cube/accumulator tiles, then applies vector elementwise arithmetic, then commits the result to global memory. + +```cpp title="pto_flash_attention.cpp" linenums="1" +#include "pto_kernels/attention/flash_attention.cpp" + +int main() { + constexpr int sequence = pto::kernels::shapes::kAttentionLargeSeq; + constexpr int query_depth = pto::kernels::shapes::kAttentionSmallQD; + constexpr int value_depth = pto::kernels::shapes::kAttentionVD; + alignas(64) static int query[sequence * query_depth]{}; + alignas(64) static int key[query_depth * sequence]{}; + alignas(64) static int value[sequence * value_depth]{}; + alignas(64) static int output[sequence * value_depth]{}; + flash_attention_i32(query, key, value, output); + return 0; +} +``` + +## PTO-Bearing Local Implementations + +These local include files contain the PTO calls reached by this entrypoint. + +??? code "benchmark/one-level-arch/kernels/pto_kernels/attention/flash_attention.cpp" + + ```cpp title="flash_attention.cpp" linenums="1" + // Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. + #include + #include + #include + + using namespace pto; + + namespace { + + constexpr int kS = kernels::shapes::kAttentionLargeSeq; + constexpr int kQD = kernels::shapes::kAttentionSmallQD; + constexpr int kVD = kernels::shapes::kAttentionVD; + constexpr int kTm = kernels::tiling::kFlashTileM; + constexpr int kTk = kernels::tiling::kFlashTileK; + + static_assert(kTm * kTk * kQD * static_cast(sizeof(int)) <= 4096, + "QK matmul footprint must fit <=4KB"); + static_assert(kTm * kVD * kTk * static_cast(sizeof(int)) <= 4096, + "WV matmul footprint must fit <=4KB"); + static_assert(kS % kTm == 0 && kS % kTk == 0, + "global sequence shape must be divisible by tile shape"); + + using gmQ = global_tensor>; + using gmK = global_tensor>; + using gmV = global_tensor>; + using gmO = global_tensor>; + + using tileQ = TileLeft; + using tileK = TileRight; + using tileV = TileRight; + using tileScoreAcc = TileAcc; + using tileScoreVec = Tile; + using tileScoreLeft = TileLeft; + using tileOutAcc = TileAcc; + using tileOutVec = Tile; + + using itQ = global_iterator; + using itK = global_iterator; + using itV = global_iterator; + using itO = global_iterator; + + } // namespace + + extern "C" void flash_attention_i32(int *q_ptr, int *k_ptr, int *v_ptr, + int *out_ptr) { + itQ gQ(q_ptr); + itK gK(k_ptr); + itV gV(v_ptr); + itO gO(out_ptr); + + constexpr int kQTiles = kS / kTm; + constexpr int kKTiles = kS / kTk; + + for (int qi = 0; qi < kQTiles; ++qi) { + tileQ q; + TLOAD(q, gQ(qi, 0)); + + tileK k0; + tileV v0; + TLOAD(k0, gK(0, 0)); + TLOAD(v0, gV(0, 0)); + + tileScoreAcc sAcc0; + tileScoreVec sVec0; + tileScoreLeft sLeft0; + TMATMUL(sAcc0, q, k0); + TCVT(sVec0, sAcc0); + TCVT(sLeft0, sVec0); + + tileOutAcc outAcc; + TMATMUL(outAcc, sLeft0, v0); + + for (int kj = 1; kj < kKTiles; ++kj) { + tileK k; + tileV v; + TLOAD(k, gK(0, kj)); + TLOAD(v, gV(kj, 0)); + + tileScoreAcc sAcc; + tileScoreVec sVec; + tileScoreLeft sLeft; + TMATMUL(sAcc, q, k); + TCVT(sVec, sAcc); + TCVT(sLeft, sVec); + + tileOutAcc pieceAcc; + tileOutVec outVec; + tileOutVec pieceVec; + tileOutVec merged; + TMATMUL(pieceAcc, sLeft, v); + TCVT(outVec, outAcc); + TCVT(pieceVec, pieceAcc); + TADD(merged, outVec, pieceVec); + TCVT(outAcc, merged); + } + + tileOutVec out; + TCVT(out, outAcc); + TSTORE(gO(qi, 0), out); + } + } + ``` + +## Supported PTO Intrinsics + +This is the source-level union across the compile-time paths in this +implementation. A build command may select a subset through its macros; +the exact source spellings below preserve aliases and masked forms. + +| Intrinsic contract | Source spelling | Called from | +| --- | --- | --- | +| [`TADD`](../../../../intrinsics/tadd.md) | `TADD` | `benchmark/one-level-arch/kernels/pto_kernels/attention/flash_attention.cpp` | +| [`TCVT`](../../../../intrinsics/tcvt.md) | `TCVT` | `benchmark/one-level-arch/kernels/pto_kernels/attention/flash_attention.cpp` | +| [`TLOAD`](../../../../intrinsics/tload.md) | `TLOAD` | `benchmark/one-level-arch/kernels/pto_kernels/attention/flash_attention.cpp` | +| [`TMATMUL`](../../../../intrinsics/tmatmul.md) | `TMATMUL` | `benchmark/one-level-arch/kernels/pto_kernels/attention/flash_attention.cpp` | +| [`TSTORE`](../../../../intrinsics/tstore.md) | `TSTORE` | `benchmark/one-level-arch/kernels/pto_kernels/attention/flash_attention.cpp` | + +## Active Build Commands + +Run these from `benchmark/one-level-arch/test/kernel/pto_kernels` after setting +`COMPILER_DIR` and `LINX_SYSROOT` as described in the build guide. + +| Manifest line | Command | +| ---: | --- | +| 14 | `make TESTCASE=pto_flash_attention COMPILER_DIR="$COMPILER_DIR" diss` | + +## Resolved One-Level Source Closure + +??? info "8 one-level source files reached through local includes" + + - `benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_tiling.hpp` + - `benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp` + - `benchmark/one-level-arch/kernels/pto_kernels/attention/flash_attention.cpp` + - `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_flash_attention.cpp` diff --git a/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-basic-a2ba2c9c.md b/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-basic-a2ba2c9c.md new file mode 100644 index 00000000..46e46edc --- /dev/null +++ b/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-basic-a2ba2c9c.md @@ -0,0 +1,145 @@ +# pto_gemm_basic: pto_gemm_basic.cpp + + + +| Field | Value | +| --- | --- | +| Benchmark surface | One-level | +| Family | `pto_kernels` | +| Implementation source | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_basic.cpp` | +| Active build variants | 1 | + +## How the Code Is Written + +The implementation loads or gathers global data into typed tiles, then executes matrix or matrix-vector work on cube/accumulator tiles, then commits the result to global memory. + +```cpp title="pto_gemm_basic.cpp" linenums="1" +#include "pto_kernels/matmul/gemm_basic.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static float lhs[m * k]{}; + alignas(64) static float rhs[k * n]{}; + alignas(64) static float dst[m * n]{}; + gemm_basic_f32(lhs, rhs, dst); + return 0; +} +``` + +## PTO-Bearing Local Implementations + +These local include files contain the PTO calls reached by this entrypoint. + +??? code "benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_basic.cpp" + + ```cpp title="gemm_basic.cpp" linenums="1" + // Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. + #include + #include + #include + + using namespace pto; + + namespace { + + constexpr int kM = kernels::shapes::kMatmulM; + constexpr int kN = kernels::shapes::kMatmulN; + constexpr int kK = kernels::shapes::kMatmulK; + + constexpr int kTM = kernels::tiling::kGemmTileM; + constexpr int kTN = kernels::tiling::kGemmTileN; + constexpr int kTK = kernels::tiling::kGemmTileK; + + static_assert(kTM * kTN * kTK * static_cast(sizeof(float)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); + static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + + using tileA = TileLeft; + using tileB = TileRight; + using tileAcc = TileAcc; + using tileVec = Tile; + + using gmA = global_tensor>; + using gmB = global_tensor>; + using gmC = global_tensor>; + + using itA = global_iterator; + using itB = global_iterator; + using itC = global_iterator; + + } // namespace + + extern "C" void gemm_basic_f32(float *lhs_ptr, float *rhs_ptr, + float *dst_ptr) { + itA gA(lhs_ptr); + itB gB(rhs_ptr); + itC gC(dst_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileAcc acc; + TMATMUL(acc, a0, b0); + + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(acc, acc, a, b); + } + + tileVec out; + TCVT(out, acc); + TSTORE(gC(mi, nj), out); + } + } + } + ``` + +## Supported PTO Intrinsics + +This is the source-level union across the compile-time paths in this +implementation. A build command may select a subset through its macros; +the exact source spellings below preserve aliases and masked forms. + +| Intrinsic contract | Source spelling | Called from | +| --- | --- | --- | +| [`TCVT`](../../../../intrinsics/tcvt.md) | `TCVT` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_basic.cpp` | +| [`TLOAD`](../../../../intrinsics/tload.md) | `TLOAD` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_basic.cpp` | +| [`TMATMUL`](../../../../intrinsics/tmatmul.md) | `TMATMUL` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_basic.cpp` | +| [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md) | `TMATMUL_ACC` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_basic.cpp` | +| [`TSTORE`](../../../../intrinsics/tstore.md) | `TSTORE` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_basic.cpp` | + +## Active Build Commands + +Run these from `benchmark/one-level-arch/test/kernel/pto_kernels` after setting +`COMPILER_DIR` and `LINX_SYSROOT` as described in the build guide. + +| Manifest line | Command | +| ---: | --- | +| 9 | `make TESTCASE=pto_gemm_basic COMPILER_DIR="$COMPILER_DIR" diss` | + +## Resolved One-Level Source Closure + +??? info "8 one-level source files reached through local includes" + + - `benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_tiling.hpp` + - `benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp` + - `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_basic.cpp` + - `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_basic.cpp` diff --git a/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-demo-85ba9afc.md b/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-demo-85ba9afc.md new file mode 100644 index 00000000..3cc7278f --- /dev/null +++ b/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-demo-85ba9afc.md @@ -0,0 +1,150 @@ +# pto_gemm_demo: pto_gemm_demo.cpp + + + +| Field | Value | +| --- | --- | +| Benchmark surface | One-level | +| Family | `pto_kernels` | +| Implementation source | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_demo.cpp` | +| Active build variants | 1 | + +## How the Code Is Written + +The implementation loads or gathers global data into typed tiles, then executes matrix or matrix-vector work on cube/accumulator tiles, then applies vector elementwise arithmetic, then commits the result to global memory. + +```cpp title="pto_gemm_demo.cpp" linenums="1" +#include "pto_kernels/matmul/gemm_demo.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static float lhs[m * k]{}; + alignas(64) static float rhs[k * n]{}; + alignas(64) static float dst[m * n]{}; + gemm_demo_f32(dst, lhs, rhs); + return 0; +} +``` + +## PTO-Bearing Local Implementations + +These local include files contain the PTO calls reached by this entrypoint. + +??? code "benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp" + + ```cpp title="gemm_demo.cpp" linenums="1" + // Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. + #include + #include + #include + + using namespace pto; + + namespace { + + constexpr int kM = kernels::shapes::kMatmulM; + constexpr int kN = kernels::shapes::kMatmulN; + constexpr int kK = kernels::shapes::kMatmulK; + + constexpr int kTM = kernels::tiling::kGemmTileM; + constexpr int kTN = kernels::tiling::kGemmTileN; + constexpr int kTK = kernels::tiling::kGemmTileK; + constexpr float kAlpha = 0.125f; + + static_assert(kTM * kTN * kTK * static_cast(sizeof(float)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); + static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + + using tileA = TileLeft; + using tileB = TileRight; + using tileAcc = TileAcc; + using tileVec = Tile; + + using gmA = global_tensor>; + using gmB = global_tensor>; + using gmC = global_tensor>; + + using itA = global_iterator; + using itB = global_iterator; + using itC = global_iterator; + + } // namespace + + extern "C" void gemm_demo_f32(float *out_ptr, float *a_ptr, float *b_ptr) { + itA gA(a_ptr); + itB gB(b_ptr); + itC gC(out_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileAcc acc; + TMATMUL(acc, a0, b0); + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(acc, acc, a, b); + } + + tileVec out; + tileVec scaled; + tileVec merged; + TCVT(out, acc); + TMULS(scaled, out, kAlpha); + TADD(merged, out, scaled); + TSTORE(gC(mi, nj), merged); + } + } + } + ``` + +## Supported PTO Intrinsics + +This is the source-level union across the compile-time paths in this +implementation. A build command may select a subset through its macros; +the exact source spellings below preserve aliases and masked forms. + +| Intrinsic contract | Source spelling | Called from | +| --- | --- | --- | +| [`TADD`](../../../../intrinsics/tadd.md) | `TADD` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp` | +| [`TCVT`](../../../../intrinsics/tcvt.md) | `TCVT` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp` | +| [`TLOAD`](../../../../intrinsics/tload.md) | `TLOAD` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp` | +| [`TMATMUL`](../../../../intrinsics/tmatmul.md) | `TMATMUL` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp` | +| [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md) | `TMATMUL_ACC` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp` | +| [`TMULS`](../../../../intrinsics/tmuls.md) | `TMULS` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp` | +| [`TSTORE`](../../../../intrinsics/tstore.md) | `TSTORE` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp` | + +## Active Build Commands + +Run these from `benchmark/one-level-arch/test/kernel/pto_kernels` after setting +`COMPILER_DIR` and `LINX_SYSROOT` as described in the build guide. + +| Manifest line | Command | +| ---: | --- | +| 10 | `make TESTCASE=pto_gemm_demo COMPILER_DIR="$COMPILER_DIR" diss` | + +## Resolved One-Level Source Closure + +??? info "8 one-level source files reached through local includes" + + - `benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_tiling.hpp` + - `benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp` + - `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_demo.cpp` + - `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_demo.cpp` diff --git a/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-eb591839.md b/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-eb591839.md new file mode 100644 index 00000000..f9656901 --- /dev/null +++ b/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-eb591839.md @@ -0,0 +1,149 @@ +# pto_gemm: pto_gemm.cpp + + + +| Field | Value | +| --- | --- | +| Benchmark surface | One-level | +| Family | `pto_kernels` | +| Implementation source | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm.cpp` | +| Active build variants | 1 | + +## How the Code Is Written + +The implementation loads or gathers global data into typed tiles, then executes matrix or matrix-vector work on cube/accumulator tiles, then applies vector elementwise arithmetic, then commits the result to global memory. + +```cpp title="pto_gemm.cpp" linenums="1" +#include "pto_kernels/matmul/gemm.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static int lhs[m * k]{}; + alignas(64) static int rhs[k * n]{}; + alignas(64) static int dst[m * n]{}; + gemm_i32(lhs, rhs, dst); + return 0; +} +``` + +## PTO-Bearing Local Implementations + +These local include files contain the PTO calls reached by this entrypoint. + +??? code "benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp" + + ```cpp title="gemm.cpp" linenums="1" + // Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. + #include + #include + #include + + using namespace pto; + + namespace { + + constexpr int kM = kernels::shapes::kMatmulM; + constexpr int kN = kernels::shapes::kMatmulN; + constexpr int kK = kernels::shapes::kMatmulK; + + constexpr int kTM = kernels::tiling::kGemmTileM; + constexpr int kTN = kernels::tiling::kGemmTileN; + constexpr int kTK = kernels::tiling::kGemmTileK; + + static_assert(kTM * kTN * kTK * static_cast(sizeof(int)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); + static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + + using tileA = TileLeft; + using tileB = TileRight; + using tileAcc = TileAcc; + using tileVec = Tile; + + using gmA = global_tensor>; + using gmB = global_tensor>; + using gmC = global_tensor>; + + using itA = global_iterator; + using itB = global_iterator; + using itC = global_iterator; + + } // namespace + + extern "C" void gemm_i32(int *lhs_ptr, int *rhs_ptr, int *dst_ptr) { + itA gA(lhs_ptr); + itB gB(rhs_ptr); + itC gC(dst_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileAcc prodAcc; + TMATMUL(prodAcc, a0, b0); + + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(prodAcc, prodAcc, a, b); + } + + tileVec prod; + tileVec bias; + tileVec sum; + TCVT(prod, prodAcc); + TLOAD(bias, gC(mi, nj)); + TADD(sum, prod, bias); + TSTORE(gC(mi, nj), sum); + } + } + } + ``` + +## Supported PTO Intrinsics + +This is the source-level union across the compile-time paths in this +implementation. A build command may select a subset through its macros; +the exact source spellings below preserve aliases and masked forms. + +| Intrinsic contract | Source spelling | Called from | +| --- | --- | --- | +| [`TADD`](../../../../intrinsics/tadd.md) | `TADD` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp` | +| [`TCVT`](../../../../intrinsics/tcvt.md) | `TCVT` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp` | +| [`TLOAD`](../../../../intrinsics/tload.md) | `TLOAD` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp` | +| [`TMATMUL`](../../../../intrinsics/tmatmul.md) | `TMATMUL` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp` | +| [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md) | `TMATMUL_ACC` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp` | +| [`TSTORE`](../../../../intrinsics/tstore.md) | `TSTORE` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp` | + +## Active Build Commands + +Run these from `benchmark/one-level-arch/test/kernel/pto_kernels` after setting +`COMPILER_DIR` and `LINX_SYSROOT` as described in the build guide. + +| Manifest line | Command | +| ---: | --- | +| 8 | `make TESTCASE=pto_gemm COMPILER_DIR="$COMPILER_DIR" diss` | + +## Resolved One-Level Source Closure + +??? info "8 one-level source files reached through local includes" + + - `benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_tiling.hpp` + - `benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp` + - `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm.cpp` + - `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm.cpp` diff --git a/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-performance-ffe72cfc.md b/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-performance-ffe72cfc.md new file mode 100644 index 00000000..26ed3c64 --- /dev/null +++ b/docs/benchmarks/catalog/one-level/pto-kernels/pto-gemm-performance-ffe72cfc.md @@ -0,0 +1,155 @@ +# pto_gemm_performance: pto_gemm_performance.cpp + + + +| Field | Value | +| --- | --- | +| Benchmark surface | One-level | +| Family | `pto_kernels` | +| Implementation source | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_performance.cpp` | +| Active build variants | 1 | + +## How the Code Is Written + +The implementation loads or gathers global data into typed tiles, then executes matrix or matrix-vector work on cube/accumulator tiles, then applies vector elementwise arithmetic, then commits the result to global memory. + +```cpp title="pto_gemm_performance.cpp" linenums="1" +#include "pto_kernels/matmul/gemm_performance.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static float lhs[m * k]{}; + alignas(64) static float rhs[k * n]{}; + alignas(64) static float dst[m * n]{}; + gemm_performance_f32(lhs, rhs, dst, 2); + return 0; +} +``` + +## PTO-Bearing Local Implementations + +These local include files contain the PTO calls reached by this entrypoint. + +??? code "benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp" + + ```cpp title="gemm_performance.cpp" linenums="1" + // Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. + #include + #include + #include + + using namespace pto; + + namespace { + + constexpr int kM = kernels::shapes::kMatmulM; + constexpr int kN = kernels::shapes::kMatmulN; + constexpr int kK = kernels::shapes::kMatmulK; + + constexpr int kTM = kernels::tiling::kGemmTileM; + constexpr int kTN = kernels::tiling::kGemmTileN; + constexpr int kTK = kernels::tiling::kGemmTileK; + + static_assert(kTM * kTN * kTK * static_cast(sizeof(float)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); + static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + + using tileA = TileLeft; + using tileB = TileRight; + using tileAcc = TileAcc; + using tileVec = Tile; + + using gmA = global_tensor>; + using gmB = global_tensor>; + using gmC = global_tensor>; + + using itA = global_iterator; + using itB = global_iterator; + using itC = global_iterator; + + } // namespace + + extern "C" void gemm_performance_f32(float *lhs_ptr, float *rhs_ptr, + float *dst_ptr, int repeat_tiles) { + if (repeat_tiles <= 0) + repeat_tiles = 1; + + itA gA(lhs_ptr); + itB gB(rhs_ptr); + itC gC(dst_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int rep = 0; rep < repeat_tiles; ++rep) { + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileAcc acc; + TMATMUL(acc, a0, b0); + + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(acc, acc, a, b); + } + + tileVec out; + tileVec prev; + tileVec merged; + TCVT(out, acc); + TLOAD(prev, gC(mi, nj)); + TADD(merged, prev, out); + TSTORE(gC(mi, nj), merged); + } + } + } + } + ``` + +## Supported PTO Intrinsics + +This is the source-level union across the compile-time paths in this +implementation. A build command may select a subset through its macros; +the exact source spellings below preserve aliases and masked forms. + +| Intrinsic contract | Source spelling | Called from | +| --- | --- | --- | +| [`TADD`](../../../../intrinsics/tadd.md) | `TADD` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp` | +| [`TCVT`](../../../../intrinsics/tcvt.md) | `TCVT` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp` | +| [`TLOAD`](../../../../intrinsics/tload.md) | `TLOAD` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp` | +| [`TMATMUL`](../../../../intrinsics/tmatmul.md) | `TMATMUL` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp` | +| [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md) | `TMATMUL_ACC` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp` | +| [`TSTORE`](../../../../intrinsics/tstore.md) | `TSTORE` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp` | + +## Active Build Commands + +Run these from `benchmark/one-level-arch/test/kernel/pto_kernels` after setting +`COMPILER_DIR` and `LINX_SYSROOT` as described in the build guide. + +| Manifest line | Command | +| ---: | --- | +| 11 | `make TESTCASE=pto_gemm_performance COMPILER_DIR="$COMPILER_DIR" diss` | + +## Resolved One-Level Source Closure + +??? info "8 one-level source files reached through local includes" + + - `benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_tiling.hpp` + - `benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp` + - `benchmark/one-level-arch/kernels/pto_kernels/matmul/gemm_performance.cpp` + - `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_gemm_performance.cpp` diff --git a/docs/benchmarks/catalog/one-level/pto-kernels/pto-mamulb-e9f16fd1.md b/docs/benchmarks/catalog/one-level/pto-kernels/pto-mamulb-e9f16fd1.md new file mode 100644 index 00000000..bdbdb441 --- /dev/null +++ b/docs/benchmarks/catalog/one-level/pto-kernels/pto-mamulb-e9f16fd1.md @@ -0,0 +1,144 @@ +# pto_mamulb: pto_mamulb.cpp + + + +| Field | Value | +| --- | --- | +| Benchmark surface | One-level | +| Family | `pto_kernels` | +| Implementation source | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_mamulb.cpp` | +| Active build variants | 1 | + +## How the Code Is Written + +The implementation loads or gathers global data into typed tiles, then executes matrix or matrix-vector work on cube/accumulator tiles, then commits the result to global memory. + +```cpp title="pto_mamulb.cpp" linenums="1" +#include "pto_kernels/matmul/mamulb.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static int lhs[m * k]{}; + alignas(64) static int rhs[k * n]{}; + alignas(64) static int dst[m * n]{}; + mamulb_i32(lhs, rhs, dst); + return 0; +} +``` + +## PTO-Bearing Local Implementations + +These local include files contain the PTO calls reached by this entrypoint. + +??? code "benchmark/one-level-arch/kernels/pto_kernels/matmul/mamulb.cpp" + + ```cpp title="mamulb.cpp" linenums="1" + // Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. + #include + #include + #include + + using namespace pto; + + namespace { + + constexpr int kM = kernels::shapes::kMatmulM; + constexpr int kN = kernels::shapes::kMatmulN; + constexpr int kK = kernels::shapes::kMatmulK; + + constexpr int kTM = kernels::tiling::kGemmTileM; + constexpr int kTN = kernels::tiling::kGemmTileN; + constexpr int kTK = kernels::tiling::kGemmTileK; + + static_assert(kTM * kTN * kTK * static_cast(sizeof(int)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); + static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + + using tileA = TileLeft; + using tileB = TileRight; + using tileCAcc = TileAcc; + using tileCVec = Tile; + + using gmA = global_tensor>; + using gmB = global_tensor>; + using gmC = global_tensor>; + + using itA = global_iterator; + using itB = global_iterator; + using itC = global_iterator; + + } // namespace + + extern "C" void mamulb_i32(int *lhs_ptr, int *rhs_ptr, int *dst_ptr) { + itA gA(lhs_ptr); + itB gB(rhs_ptr); + itC gC(dst_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileCAcc acc; + TMATMUL(acc, a0, b0); + + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(acc, acc, a, b); + } + + tileCVec out; + TCVT(out, acc); + TSTORE(gC(mi, nj), out); + } + } + } + ``` + +## Supported PTO Intrinsics + +This is the source-level union across the compile-time paths in this +implementation. A build command may select a subset through its macros; +the exact source spellings below preserve aliases and masked forms. + +| Intrinsic contract | Source spelling | Called from | +| --- | --- | --- | +| [`TCVT`](../../../../intrinsics/tcvt.md) | `TCVT` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/mamulb.cpp` | +| [`TLOAD`](../../../../intrinsics/tload.md) | `TLOAD` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/mamulb.cpp` | +| [`TMATMUL`](../../../../intrinsics/tmatmul.md) | `TMATMUL` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/mamulb.cpp` | +| [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md) | `TMATMUL_ACC` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/mamulb.cpp` | +| [`TSTORE`](../../../../intrinsics/tstore.md) | `TSTORE` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/mamulb.cpp` | + +## Active Build Commands + +Run these from `benchmark/one-level-arch/test/kernel/pto_kernels` after setting +`COMPILER_DIR` and `LINX_SYSROOT` as described in the build guide. + +| Manifest line | Command | +| ---: | --- | +| 12 | `make TESTCASE=pto_mamulb COMPILER_DIR="$COMPILER_DIR" diss` | + +## Resolved One-Level Source Closure + +??? info "8 one-level source files reached through local includes" + + - `benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_tiling.hpp` + - `benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp` + - `benchmark/one-level-arch/kernels/pto_kernels/matmul/mamulb.cpp` + - `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_mamulb.cpp` diff --git a/docs/benchmarks/catalog/one-level/pto-kernels/pto-tload-store-1e107659.md b/docs/benchmarks/catalog/one-level/pto-kernels/pto-tload-store-1e107659.md new file mode 100644 index 00000000..c7e97d1a --- /dev/null +++ b/docs/benchmarks/catalog/one-level/pto-kernels/pto-tload-store-1e107659.md @@ -0,0 +1,111 @@ +# pto_tload_store: pto_tload_store.cpp + + + +| Field | Value | +| --- | --- | +| Benchmark surface | One-level | +| Family | `pto_kernels` | +| Implementation source | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tload_store.cpp` | +| Active build variants | 1 | + +## How the Code Is Written + +The implementation loads or gathers global data into typed tiles, then commits the result to global memory. + +```cpp title="pto_tload_store.cpp" linenums="1" +#include "pto_kernels/memory/tload_store.cpp" + +int main() { + constexpr int elements = pto::kernels::shapes::kMemoryRows * + pto::kernels::shapes::kMemoryCols; + alignas(64) static int source[elements]{}; + alignas(64) static int destination[elements]{}; + tload_store_i32(source, destination); + return 0; +} +``` + +## PTO-Bearing Local Implementations + +These local include files contain the PTO calls reached by this entrypoint. + +??? code "benchmark/one-level-arch/kernels/pto_kernels/memory/tload_store.cpp" + + ```cpp title="tload_store.cpp" linenums="1" + // Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. + #include + #include + + using namespace pto; + + namespace { + + constexpr int kRows = kernels::shapes::kMemoryRows; + constexpr int kCols = kernels::shapes::kMemoryCols; + using tile_vec_i32 = Tile; + + static_assert(tile_vec_i32::Rows * tile_vec_i32::Cols * + static_cast(sizeof(int)) == + 4096, + "tile must be exactly 4KB"); + static_assert(kRows % tile_vec_i32::Rows == 0 && + kCols % tile_vec_i32::Cols == 0, + "global tensor must be divisible by tile shape"); + + using gmSrc = global_tensor>; + using gmDst = global_tensor>; + + using itSrc = global_iterator; + using itDst = global_iterator; + + } // namespace + + extern "C" void tload_store_i32(int *src_ptr, int *dst_ptr) { + itSrc gSrc(src_ptr); + itDst gDst(dst_ptr); + + constexpr int kRowTiles = kRows / tile_vec_i32::Rows; + constexpr int kColTiles = kCols / tile_vec_i32::Cols; + + for (int tr = 0; tr < kRowTiles; ++tr) { + for (int tc = 0; tc < kColTiles; ++tc) { + tile_vec_i32 tile; + TLOAD(tile, gSrc(tr, tc)); + TSTORE(gDst(tr, tc), tile); + } + } + } + ``` + +## Supported PTO Intrinsics + +This is the source-level union across the compile-time paths in this +implementation. A build command may select a subset through its macros; +the exact source spellings below preserve aliases and masked forms. + +| Intrinsic contract | Source spelling | Called from | +| --- | --- | --- | +| [`TLOAD`](../../../../intrinsics/tload.md) | `TLOAD` | `benchmark/one-level-arch/kernels/pto_kernels/memory/tload_store.cpp` | +| [`TSTORE`](../../../../intrinsics/tstore.md) | `TSTORE` | `benchmark/one-level-arch/kernels/pto_kernels/memory/tload_store.cpp` | + +## Active Build Commands + +Run these from `benchmark/one-level-arch/test/kernel/pto_kernels` after setting +`COMPILER_DIR` and `LINX_SYSROOT` as described in the build guide. + +| Manifest line | Command | +| ---: | --- | +| 6 | `make TESTCASE=pto_tload_store COMPILER_DIR="$COMPILER_DIR" diss` | + +## Resolved One-Level Source Closure + +??? info "7 one-level source files reached through local includes" + + - `benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp` + - `benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp` + - `benchmark/one-level-arch/kernels/pto_kernels/memory/tload_store.cpp` + - `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tload_store.cpp` diff --git a/docs/benchmarks/catalog/one-level/pto-kernels/pto-tmatmul-acc-e3e7d902.md b/docs/benchmarks/catalog/one-level/pto-kernels/pto-tmatmul-acc-e3e7d902.md new file mode 100644 index 00000000..4313be24 --- /dev/null +++ b/docs/benchmarks/catalog/one-level/pto-kernels/pto-tmatmul-acc-e3e7d902.md @@ -0,0 +1,144 @@ +# pto_tmatmul_acc: pto_tmatmul_acc.cpp + + + +| Field | Value | +| --- | --- | +| Benchmark surface | One-level | +| Family | `pto_kernels` | +| Implementation source | `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tmatmul_acc.cpp` | +| Active build variants | 1 | + +## How the Code Is Written + +The implementation loads or gathers global data into typed tiles, then executes matrix or matrix-vector work on cube/accumulator tiles, then commits the result to global memory. + +```cpp title="pto_tmatmul_acc.cpp" linenums="1" +#include "pto_kernels/matmul/tmatmul_acc.cpp" + +int main() { + constexpr int m = pto::kernels::shapes::kMatmulM; + constexpr int n = pto::kernels::shapes::kMatmulN; + constexpr int k = pto::kernels::shapes::kMatmulK; + alignas(64) static int lhs[m * k]{}; + alignas(64) static int rhs[k * n]{}; + alignas(64) static int dst[m * n]{}; + tmatmul_acc_i32(lhs, rhs, dst); + return 0; +} +``` + +## PTO-Bearing Local Implementations + +These local include files contain the PTO calls reached by this entrypoint. + +??? code "benchmark/one-level-arch/kernels/pto_kernels/matmul/tmatmul_acc.cpp" + + ```cpp title="tmatmul_acc.cpp" linenums="1" + // Migrated from LinxISA/PTO-Kernel at 0eb72cb20b0d; scalar fallback removed. + #include + #include + #include + + using namespace pto; + + namespace { + + constexpr int kM = kernels::shapes::kMatmulM; + constexpr int kN = kernels::shapes::kMatmulN; + constexpr int kK = kernels::shapes::kMatmulK; + + constexpr int kTM = kernels::tiling::kGemmTileM; + constexpr int kTN = kernels::tiling::kGemmTileN; + constexpr int kTK = kernels::tiling::kGemmTileK; + + static_assert(kTM * kTN * kTK * static_cast(sizeof(int)) <= 4096, + "TMATMUL tile footprint must fit <=4KB"); + static_assert(kM % kTM == 0 && kN % kTN == 0 && kK % kTK == 0, + "global tensor shape must be divisible by tile shape"); + + using tileA = TileLeft; + using tileB = TileRight; + using tileAcc = TileAcc; + using tileVec = Tile; + + using gmA = global_tensor>; + using gmB = global_tensor>; + using gmC = global_tensor>; + + using itA = global_iterator; + using itB = global_iterator; + using itC = global_iterator; + + } // namespace + + extern "C" void tmatmul_acc_i32(int *lhs_ptr, int *rhs_ptr, int *dst_ptr) { + itA gA(lhs_ptr); + itB gB(rhs_ptr); + itC gC(dst_ptr); + + constexpr int kMTiles = kM / kTM; + constexpr int kNTiles = kN / kTN; + constexpr int kKTiles = kK / kTK; + + for (int mi = 0; mi < kMTiles; ++mi) { + for (int nj = 0; nj < kNTiles; ++nj) { + tileA a0; + tileB b0; + TLOAD(a0, gA(mi, 0)); + TLOAD(b0, gB(0, nj)); + + tileAcc cAcc; + TMATMUL(cAcc, a0, b0); + + for (int kk = 1; kk < kKTiles; ++kk) { + tileA a; + tileB b; + TLOAD(a, gA(mi, kk)); + TLOAD(b, gB(kk, nj)); + TMATMUL_ACC(cAcc, cAcc, a, b); + } + + tileVec cVec; + TCVT(cVec, cAcc); + TSTORE(gC(mi, nj), cVec); + } + } + } + ``` + +## Supported PTO Intrinsics + +This is the source-level union across the compile-time paths in this +implementation. A build command may select a subset through its macros; +the exact source spellings below preserve aliases and masked forms. + +| Intrinsic contract | Source spelling | Called from | +| --- | --- | --- | +| [`TCVT`](../../../../intrinsics/tcvt.md) | `TCVT` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/tmatmul_acc.cpp` | +| [`TLOAD`](../../../../intrinsics/tload.md) | `TLOAD` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/tmatmul_acc.cpp` | +| [`TMATMUL`](../../../../intrinsics/tmatmul.md) | `TMATMUL` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/tmatmul_acc.cpp` | +| [`TMATMUL_ACC`](../../../../intrinsics/tmatmul_acc.md) | `TMATMUL_ACC` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/tmatmul_acc.cpp` | +| [`TSTORE`](../../../../intrinsics/tstore.md) | `TSTORE` | `benchmark/one-level-arch/kernels/pto_kernels/matmul/tmatmul_acc.cpp` | + +## Active Build Commands + +Run these from `benchmark/one-level-arch/test/kernel/pto_kernels` after setting +`COMPILER_DIR` and `LINX_SYSROOT` as described in the build guide. + +| Manifest line | Command | +| ---: | --- | +| 13 | `make TESTCASE=pto_tmatmul_acc COMPILER_DIR="$COMPILER_DIR" diss` | + +## Resolved One-Level Source Closure + +??? info "8 one-level source files reached through local includes" + + - `benchmark/one-level-arch/include/pto_kernel/common/linx_lowp_types.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/pto_tileop.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_env.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_shapes.hpp` + - `benchmark/one-level-arch/include/pto_kernel/common/runtime/kernel_tiling.hpp` + - `benchmark/one-level-arch/include/pto_kernel/pto/linx/impl/backend.hpp` + - `benchmark/one-level-arch/kernels/pto_kernels/matmul/tmatmul_acc.cpp` + - `benchmark/one-level-arch/test/kernel/pto_kernels/src/pto_tmatmul_acc.cpp` diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md index c3f009a5..80cdcfea 100644 --- a/docs/benchmarks/index.md +++ b/docs/benchmarks/index.md @@ -7,8 +7,8 @@ implementation page presents the complete entry source, PTO-bearing local include implementations, the canonical PTO intrinsic surface, exact build commands, and required data objects. -**Coverage:** 53 active build variants, 20 source-backed -implementations, and 13 benchmark families. +**Coverage:** 62 active build variants, 29 source-backed +implementations, and 14 benchmark families. | Surface | Family | Implementations | Build variants | PTO source-union surface | | --- | --- | ---: | ---: | --- | @@ -19,6 +19,7 @@ implementations, and 13 benchmark families. | One-level | [fa](catalog/one-level/fa/index.md) | 2 | 9 | 23 intrinsics | | One-level | [gather](catalog/one-level/gather/index.md) | 1 | 1 | 3 intrinsics | | One-level | [matmul](catalog/one-level/matmul/index.md) | 3 | 16 | 11 intrinsics | +| One-level | [pto_kernels](catalog/one-level/pto-kernels/index.md) | 9 | 9 | 7 intrinsics | | One-level | [reduction/reducemax_col](catalog/one-level/reduction-reducemax-col/index.md) | 1 | 1 | 5 intrinsics | | One-level | [reduction/reducemax_row](catalog/one-level/reduction-reducemax-row/index.md) | 1 | 1 | 5 intrinsics | | One-level | [reduction/reducesum_col](catalog/one-level/reduction-reducesum-col/index.md) | 1 | 2 | 5 intrinsics | diff --git a/docs/tutorials/migrated-pto-kernels.md b/docs/tutorials/migrated-pto-kernels.md new file mode 100644 index 00000000..34cdafde --- /dev/null +++ b/docs/tutorials/migrated-pto-kernels.md @@ -0,0 +1,63 @@ +# Migrated PTO Tile Kernels + +SuperNPUBench includes nine tile-only kernels imported from +[`LinxISA/PTO-Kernel`](https://github.com/LinxISA/PTO-Kernel) at revision +`0eb72cb20b0de99326d984a9a27ddb815e6e4c24`. + +## Included Kernels + +| Benchmark | Upstream source | PTO 0.57 data path | +| --- | --- | --- | +| `pto_tload_store` | `kernels/memory/tload_store.cpp` | `TLOAD`, `TSTORE` | +| `pto_add` | `kernels/elementwise/add_custom.cpp` | `TLOAD`, `TADD`, `TSTORE` | +| `pto_gemm` | `kernels/matmul/gemm.cpp` | GEMM with destination accumulation | +| `pto_gemm_basic` | `kernels/matmul/gemm_basic.cpp` | Basic FP32 tiled GEMM | +| `pto_gemm_demo` | `kernels/matmul/gemm_demo.cpp` | GEMM followed by tile scaling and addition | +| `pto_gemm_performance` | `kernels/matmul/gemm_performance.cpp` | Repeated tiled GEMM accumulation | +| `pto_mamulb` | `kernels/matmul/mamulb.cpp` | Integer tiled matrix multiply | +| `pto_tmatmul_acc` | `kernels/matmul/tmatmul_acc.cpp` | Explicit `TMATMUL_ACC` coverage | +| `pto_flash_attention` | `kernels/attention/flash_attention.cpp` | Two-stage integer QK and score-V tile matmul | + +The [PTO-Kernel benchmark catalog](../benchmarks/catalog/one-level/pto-kernels/index.md) +shows the complete wrapper and migrated source for every case, together with +its exact intrinsic union and build command. + +## Tile-Only Policy + +The imported kernels use scalar C++ only for tile-grid control. Their data +paths do not index global pointers or provide a scalar execution branch. Every +load, elementwise operation, matrix multiply, conversion, and store is a named +PTO 0.57 intrinsic. + +The imported support headers are namespaced under `pto_kernel/`. They do not +shadow the compatibility headers used by existing SuperNPUBench suites. + +## Build + +```bash +cd benchmark/one-level-arch/test/kernel/pto_kernels +PLAT=linx COMPILER_DIR=/path/to/linx-isa/compiler/llvm/build-linxisa-clang/bin \ + bash compile.all +``` + +Build one case with: + +```bash +make TESTCASE=pto_flash_attention \ + PLAT=linx \ + COMPILER_DIR=/path/to/linx-isa/compiler/llvm/build-linxisa-clang/bin \ + diss +``` + +## Verify the Intrinsic Boundary + +```bash +python3 scripts/verify_pto_kernel_migration.py +python3 scripts/generate_benchmark_manual.py +python3 -m mkdocs build --strict --clean +python3 scripts/verify_golden_manual.py --site site +``` + +The migration verifier rejects any operation outside the 111-entry PTO 0.57 +allowlist, scalar pointer-indexed data paths, missing wrappers, and +manifest/build-list drift. diff --git a/docs/tutorials/tile-kernel.md b/docs/tutorials/tile-kernel.md index 2e19955e..45f3938e 100644 --- a/docs/tutorials/tile-kernel.md +++ b/docs/tutorials/tile-kernel.md @@ -31,6 +31,10 @@ The exact implementation page in the [benchmark catalog](../benchmarks/index.md) shows the complete source closure and the 0.57 intrinsics reached by the active build. +For additional source-backed examples covering memory movement, tile addition, +six GEMM forms, and a two-stage attention kernel, see +[Migrated PTO Tile Kernels](migrated-pto-kernels.md). + ## Compile ```bash diff --git a/mkdocs.yml b/mkdocs.yml index dc97b8bd..4ad1fa64 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -80,6 +80,7 @@ nav: - FlashAttention: benchmarks/catalog/one-level/fa/index.md - Gather: benchmarks/catalog/one-level/gather/index.md - Matmul: benchmarks/catalog/one-level/matmul/index.md + - PTO-Kernel imports: benchmarks/catalog/one-level/pto-kernels/index.md - Reduce-max column: benchmarks/catalog/one-level/reduction-reducemax-col/index.md - Reduce-max row: benchmarks/catalog/one-level/reduction-reducemax-row/index.md - Reduce-sum column: benchmarks/catalog/one-level/reduction-reducesum-col/index.md @@ -90,6 +91,7 @@ nav: - Group programming: tutorials/group-programming.md - C++ language: tutorials/cpp-language.md - Tile kernel: tutorials/tile-kernel.md + - Migrated PTO kernels: tutorials/migrated-pto-kernels.md - GEMM: tutorials/gemm.md - FlashAttention: tutorials/flash-attention.md - Build and Debug: diff --git a/scripts/verify_golden_manual.py b/scripts/verify_golden_manual.py index 5133fa4e..38d0e039 100644 --- a/scripts/verify_golden_manual.py +++ b/scripts/verify_golden_manual.py @@ -192,9 +192,9 @@ def check_catalog(root: Path, docs: Path, errors: list[str]) -> None: rows = catalog["builds"] sources = {item.source for item in actual} families = {item.family for item in actual} - if (len(actual), len(sources), len(families)) != (53, 20, 13): + if (len(actual), len(sources), len(families)) != (62, 29, 14): errors.append( - "one-level inventory changed: expected 53 builds/20 implementations/13 " + "one-level inventory changed: expected 62 builds/29 implementations/14 " f"families, found {len(actual)}/{len(sources)}/{len(families)}" ) if catalog.get("schema") != 3: @@ -360,6 +360,9 @@ def main() -> None: errors: list[str] = [] check_intrinsics(root, docs, errors) check_model(docs, errors) + from verify_pto_kernel_migration import check_migration + + check_migration(root, errors) check_catalog(root, docs, errors) check_removed_surfaces(root, docs, errors) check_site((args.site or root / "site").resolve(), errors) @@ -367,8 +370,8 @@ def main() -> None: for error in errors: print(f"ERROR: {error}") raise SystemExit(f"golden manual verification failed with {len(errors)} errors") - print("Golden manual verified: 113 public intrinsics, 53 one-level builds, " - "20 implementations, links clean.") + print("Golden manual verified: 113 public intrinsics, 62 one-level builds, " + "29 implementations, links clean.") if __name__ == "__main__": diff --git a/scripts/verify_pto_kernel_migration.py b/scripts/verify_pto_kernel_migration.py new file mode 100755 index 00000000..460ca477 --- /dev/null +++ b/scripts/verify_pto_kernel_migration.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Verify that imported PTO-Kernel sources stay on the PTO 0.57 surface.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path + + +PTO_CALL = re.compile(r"\b((?:T|M)[A-Z][A-Z0-9_]*)\s*\(") +SCALAR_POINTER_ACCESS = re.compile(r"\b[A-Za-z_]\w*_ptr\s*\[") +REMOVED_API = re.compile(r"\b(?:TSYNC|RecordEvent|WaitEvents|event_t)\b") + + +def read_allowlist(root: Path) -> set[str]: + path = root / "scripts" / "data" / "linxisa-0.57-intrinsics.txt" + return { + line.strip() + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + + +def check_migration(root: Path, errors: list[str]) -> None: + kernel_root = root / "benchmark" / "one-level-arch" / "kernels" / "pto_kernels" + support_root = root / "benchmark" / "one-level-arch" / "include" / "pto_kernel" + test_root = root / "benchmark" / "one-level-arch" / "test" / "kernel" / "pto_kernels" + manifest_path = kernel_root / "migration.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + allowlist = read_allowlist(root) + + if manifest.get("schema") != 1: + errors.append(f"{manifest_path}: expected schema 1") + revision = manifest.get("revision", "") + if not re.fullmatch(r"[0-9a-f]{40}", revision): + errors.append(f"{manifest_path}: revision is not a full Git commit") + + kernels = manifest.get("kernels", []) + support_headers = manifest.get("support_headers", []) + benchmarks = [item.get("benchmark", "") for item in kernels] + if len(kernels) != 9 or len(set(benchmarks)) != 9: + errors.append(f"{manifest_path}: expected nine unique migrated kernels") + + declared_sources = {item.get("source", "") for item in kernels} + actual_sources = { + path.relative_to(kernel_root).as_posix() + for path in kernel_root.rglob("*.cpp") + } + if actual_sources != declared_sources: + errors.append( + "PTO migration source set differs from migration.json: " + f"missing={sorted(declared_sources - actual_sources)}, " + f"extra={sorted(actual_sources - declared_sources)}" + ) + + declared_headers = {item.get("source", "") for item in support_headers} + actual_headers = { + path.relative_to(support_root).as_posix() + for path in support_root.rglob("*.hpp") + } + if len(support_headers) != 6 or actual_headers != declared_headers: + errors.append( + "PTO support-header set differs from migration.json: " + f"missing={sorted(declared_headers - actual_headers)}, " + f"extra={sorted(actual_headers - declared_headers)}" + ) + for item in support_headers: + path = support_root / item["source"] + if not path.is_file(): + continue + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if digest != item.get("sha256"): + errors.append(f"{path}: content differs from the pinned migration digest") + text = path.read_text(encoding="utf-8") + for include in re.findall(r'^\s*#\s*include\s*[<\"]([^>\"]+)[>\"]', text, re.MULTILINE): + if include.startswith(("common/", "pto/")): + errors.append(f"{path}: support include is not isolated under pto_kernel/: {include}") + + build_surface = "\n".join( + (test_root / name).read_text(encoding="utf-8") + for name in ("Makefile", "compile.all") + ) + if "PTO_HOST_SIM" in build_surface: + errors.append(f"{test_root}: active Linx build enables the host-simulation backend") + + compile_text = (test_root / "compile.all").read_text(encoding="utf-8") + compile_cases = set(re.findall(r"\bpto_[a-z0-9_]+\b", compile_text)) + if compile_cases != set(benchmarks): + errors.append( + "PTO migration manifest differs from compile.all: " + f"missing={sorted(set(benchmarks) - compile_cases)}, " + f"extra={sorted(compile_cases - set(benchmarks))}" + ) + + for item in kernels: + benchmark = item["benchmark"] + source = kernel_root / item["source"] + wrapper = test_root / "src" / f"{benchmark}.cpp" + if not source.is_file(): + errors.append(f"missing migrated kernel: {source}") + continue + if not wrapper.is_file(): + errors.append(f"missing migrated benchmark wrapper: {wrapper}") + elif f'"pto_kernels/{item["source"]}"' not in wrapper.read_text(encoding="utf-8"): + errors.append(f"{wrapper}: does not include the declared migrated source") + + text = source.read_text(encoding="utf-8") + calls = set(PTO_CALL.findall(text)) + unsupported = calls - allowlist + if not calls: + errors.append(f"{source}: no PTO intrinsic calls found") + if unsupported: + errors.append(f"{source}: unsupported PTO calls {sorted(unsupported)}") + if "PTO_QEMU_SMOKE" in text or SCALAR_POINTER_ACCESS.search(text): + errors.append(f"{source}: contains a scalar data-path fallback") + if REMOVED_API.search(text): + errors.append(f"{source}: contains a removed event/synchronization API") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1]) + args = parser.parse_args() + errors: list[str] = [] + check_migration(args.repo_root.resolve(), errors) + if errors: + for error in errors: + print(f"ERROR: {error}") + raise SystemExit(f"PTO kernel migration verification failed with {len(errors)} errors") + print("PTO kernel migration verified: 9 tile-only kernels, PTO 0.57 allowlist clean.") + + +if __name__ == "__main__": + main()