From 8dbea2c6548521147027cbe9b7759643a0239fb2 Mon Sep 17 00:00:00 2001 From: Alex Collini Date: Fri, 9 Oct 2026 17:58:17 +0200 Subject: [PATCH] fix(recompiler): emit a GPU-selected formatted buffer access through one shared function A formatted buffer load or store through a GPU-selected descriptor still emitted the switch over every buffer format inline, at each access. A Ghost of Yotei compute pass with ~300 such accesses compiled to 37M SPIR-V words, and its pipeline took 40 s or more to create. The switch is now emitted once per component count, direction, d16, coherence, kind and stop mode, as a SPIR-V function that takes the descriptor fields, the offsets and the access's PC. When faults stop the invocation, the helper returns whether one did and the caller stops there. BDA faults record the PC from BdaInstructionPc, the access's own PC or the one a shared helper takes. Co-Authored-By: Claude Opus 5.5 --- .../tests/FormattedBufferOnce.cpp | 10 ++- .../include/SpirvBackend/SpirvBda.hpp | 1 + .../SpirvBackend/SpirvEmitterState.hpp | 2 + .../SpirvBackend/src/SpirvBdaRead.cpp | 18 ++-- .../SpirvBackend/src/SpirvRuntimeBuffer.cpp | 83 ++++++++++++++++++- 5 files changed, 101 insertions(+), 13 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp b/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp index b011accdef5..d4ae978ef0e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp @@ -55,8 +55,9 @@ Counts Count(std::span words) { return counts; } -Counts Compile(std::size_t loads, bool formattedStores = false) { +Counts Compile(std::size_t loads, bool formattedStores = false, bool gpu = false) { std::vector code; + if (gpu) code.push_back(0x7e160500u); for (std::uint32_t load = 0; load < loads; ++load) code.insert(code.end(), {0xe0000000u | (load * 4u), 0x80020000u | ((10u + load) << 8u)}); code.push_back(0xbf8c3f70u); for (std::uint32_t load = 0; load < loads; ++load) code.insert(code.end(), {(formattedStores ? 0xe0100000u : 0xe0700000u) | (load * 4u), 0x80010000u | ((10u + load) << 8u)}); @@ -93,7 +94,12 @@ int main() { Check(storeOne.functions == none.functions + 2 && storeOne.calls == none.calls + 2, "formatted buffer once: a formatted load and a formatted store to another resource emitted " + std::to_string(storeOne.functions - none.functions) + " functions and " + std::to_string(storeOne.calls - none.calls) + " calls more than none, expected one function and one call each"); Check(storeThree.functions == storeOne.functions && storeThree.calls == storeOne.calls + 4 && storeThree.switches == storeOne.switches, "formatted buffer once: two more formatted loads and stores emitted " + std::to_string(storeThree.functions - storeOne.functions) + " functions, " + std::to_string(storeThree.calls - storeOne.calls) + " calls and " + std::to_string(storeThree.switches - storeOne.switches) + " switches more, expected four calls only"); Check(three.words - one.words < one.words / 10u, "formatted buffer once: 2 more formatted loads added " + std::to_string(three.words - one.words) + " SPIR-V words to a " + std::to_string(one.words) + "-word module"); - std::printf("formatted buffer once tests passed (%zu words for 1 load, %zu for 3)\n", one.words, three.words); + const auto gpuOne = Compile(1, false, true); + const auto gpuThree = Compile(3, false, true); + Check(gpuThree.functions == gpuOne.functions && gpuThree.calls > gpuOne.calls, "formatted buffer once: two more formatted loads from a GPU-selected descriptor emitted " + std::to_string(gpuThree.functions - gpuOne.functions) + " more functions, expected calls to the shared one"); + Check(gpuThree.switches == gpuOne.switches, "formatted buffer once: two more formatted loads from a GPU-selected descriptor emitted " + std::to_string(gpuThree.switches - gpuOne.switches) + " more format switches"); + Check(gpuThree.words - gpuOne.words < gpuOne.words / 10u, "formatted buffer once: 2 more formatted loads from a GPU-selected descriptor added " + std::to_string(gpuThree.words - gpuOne.words) + " SPIR-V words to a " + std::to_string(gpuOne.words) + "-word module"); + std::printf("formatted buffer once tests passed (%zu words for 1 load, %zu for 3; GPU-selected %zu and %zu)\n", one.words, three.words, gpuOne.words, gpuThree.words); return 0; } catch (const std::exception& error) { std::fprintf(stderr, "%s\n", error.what()); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index 3d2b1cb5bff..0b308f556ce 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -24,6 +24,7 @@ void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& targe // every invocation running, so their faulting reads return zero instead. bool BdaInvocationsMayStop(const IrProgram& program); void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition); +std::uint32_t BdaInstructionPc(SpirvEmitterState& state, const IrValue& inst); std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits); void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u); void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index 6fdd75bd35c..7471636440e 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -104,6 +104,7 @@ struct SpirvEmitterState { std::uint32_t storageBufferU64Variable = 0; std::vector memoryByteOffsets; std::map, std::uint32_t> formattedBufferFunctions; + std::map, std::uint32_t> formattedGpuBufferFunctions; std::uint32_t bdaPagetableVariable = 0; std::uint32_t faultBufferVariable = 0; std::uint32_t bdaPointerFunction = 0; @@ -120,6 +121,7 @@ struct SpirvEmitterState { std::array, 2> bdaDwordReadFunctions {}; std::array, 2> bdaSpanReadFunctions {}; std::uint32_t bdaStopValue = 0; + std::uint32_t bdaPcOverride = 0; // False for programs with workgroup barriers: faulting BDA accesses then continue (see BdaInvocationsMayStop). bool bdaStopsInvocations = true; bool continueTarget = false; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index d580d06478b..371d7fc06bc 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -271,11 +271,15 @@ bool BdaByteReadsForced() { return forced; } +std::uint32_t BdaInstructionPc(SpirvEmitterState& state, const IrValue& inst) { + return state.bdaPcOverride != 0u ? state.bdaPcOverride : ConstantU32(state, inst.Flags().pc); +} + std::uint32_t AddBdaAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, bool subtract) { auto& state = ctx.state; const auto result = Binary(state, subtract ? spv::OpISub : spv::OpIAdd, TypeScalarU64(state), address, offset); const auto overflow = subtract ? Binary(state, spv::OpUGreaterThan, TypeBool(state), offset, address) : Binary(state, spv::OpULessThan, TypeBool(state), result, address); - EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, 0u), ConstantU32(state, inst.Flags().pc), BdaAbi::FaultReason::Overflow); }); + EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, 0u), BdaInstructionPc(state, inst), BdaAbi::FaultReason::Overflow); }); StopBdaInvocationIf(state, overflow); return result; } @@ -315,7 +319,7 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std:: if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA read width"); if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing"); if (bits == 32u) return EmitBdaDwordReads(ctx, inst, address, 0u, 1u)[0]; - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); EmitBdaOverflowCheck(state, address, bits / 8u, instruction); return EmitBdaBytes(state, address, bits / 8u, instruction, BdaAccessMask(state, inst)); } @@ -323,7 +327,7 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std:: void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits) { auto& state = ctx.state; if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA write width"); if (bits != 32u) { EmitBdaOverflowCheck(state, address, bits / 8u, instruction); @@ -405,7 +409,7 @@ void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t auto& state = ctx.state; if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA store width"); if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); const auto storeBytes = [&] { const auto function = state.bdaByteWriteFunctions[BdaCoherent(state, inst)]; if (function == 0) ctx.Fail(inst, "BDA byte write function is missing"); @@ -443,7 +447,7 @@ void EmitBdaDwordWrites(SpirvValueEmitContext& ctx, const IrValue& inst, std::ui storeDwords(); return; } - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); const auto accessMask = BdaAccessMask(state, inst); const auto physical = state.module.AllocateId(); state.module.AddFunction(spv::OpFunctionCall, u64, physical, state.bdaWriteProbeFunction, addresses[0], ConstantU32(state, dwords * 4u), instruction); @@ -498,7 +502,7 @@ std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std auto& state = ctx.state; if (bytes != 4u && bytes != 8u) ctx.Fail(inst, "unsupported BDA atomic width"); if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); const auto type = bytes == 8u ? TypeScalarU64(state) : TypeU32(state); const auto zero = bytes == 8u ? BdaConstant(state, 0u) : ConstantU32(state, 0u); const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), address, BdaConstant(state, bytes - 1u)), BdaConstant(state, 0u)); @@ -527,7 +531,7 @@ std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const auto& state = ctx.state; if (dwords == 0u || dwords > 4u) ctx.Fail(inst, "unsupported BDA read width"); if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); const auto used = everyDword ? (1u << dwords) - 1u : UsedBdaDwords(inst, dwords); const auto isUsed = [&](std::uint32_t dword) { return (used & (1u << dword)) != 0u; }; std::array values{}; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp index 42c036e7987..f841ffd0b06 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -7,6 +7,7 @@ #include "PipelineSpecialization.hpp" #include #include +#include #include namespace ShaderRecompiler { @@ -47,7 +48,7 @@ std::uint32_t nonzero(SpirvEmitterState& state, std::uint32_t value) { void faultIf(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, BdaAbi::FaultReason reason) { auto& state = context.state; - EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), ConstantU32(state, instruction.Flags().pc), reason); }); + EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), BdaInstructionPc(state, instruction), reason); }); StopBdaInvocationIf(state, condition); } @@ -319,6 +320,7 @@ std::uint32_t formatted(SpirvValueEmitContext& context, const IrValue& instructi std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data); std::uint32_t formattedCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data); +std::uint32_t formattedGpuCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data); std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) { auto& state = context.state; @@ -343,7 +345,7 @@ std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& i return formatted(context, instruction, resource, GetFormatInfo(format), components, store, data); } if (!memory.gpuDescriptor) return formattedCall(context, instruction, resource, components, store, data); - return formatSwitch(context, instruction, resource, components, store, data); + return formattedGpuCall(context, instruction, resource, components, store, data); } std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) { @@ -374,7 +376,7 @@ std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instru phi.insert(phi.end(), {value, exit}); } EmitLabel(state, invalid); - if (memory.gpuDescriptor) RecordBdaFault(state, resource.base, ConstantU32(state, 0u), ConstantU32(state, instruction.Flags().pc), BdaAbi::FaultReason::InvalidDescriptor); + if (memory.gpuDescriptor) RecordBdaFault(state, resource.base, ConstantU32(state, 0u), BdaInstructionPc(state, instruction), BdaAbi::FaultReason::InvalidDescriptor); const auto zero = store || components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); const auto invalidExit = state.module.AllocateId(); state.module.AddFunction(spv::OpBranch, invalidExit); @@ -443,6 +445,79 @@ std::uint32_t formattedCall(SpirvValueEmitContext& context, const IrValue& instr return store ? ConstantU32(state, 0u) : result; } +std::uint32_t formattedGpuCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + const auto u32 = TypeU32(state); + const auto u64 = TypeScalarU64(state); + const auto boolean = TypeBool(state); + const bool stops = state.bdaStopsInvocations; + const auto valueType = components == 1u ? u32 : TypeU32Composite(state, components); + const auto resultType = stops ? (store ? u32 : state.module.Type(spv::OpTypeStruct, valueType, u32)) : (store ? state.module.Type(spv::OpTypeVoid) : valueType); + const std::array key{components, store ? 1u : 0u, memory.d16 ? 1u : 0u, memory.coherent ? 1u : 0u, static_cast(memory.kind), stops ? 1u : 0u}; + auto function = state.formattedGpuBufferFunctions.find(key); + if (function == state.formattedGpuBufferFunctions.end()) { + const auto id = state.module.AllocateId(); + const auto functionType = store ? state.module.Type(spv::OpTypeFunction, resultType, u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32, valueType) + : state.module.Type(spv::OpTypeFunction, resultType, u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32); + struct Restore { + SpirvEmitterState& state; + std::uint32_t label; + std::uint32_t stopValue; + ~Restore() { + state.module.EndHelperFunction(); + state.currentLabel = label; + state.bdaStopValue = stopValue; + state.bdaPcOverride = 0u; + } + }; + state.module.BeginHelperFunction(); + const Restore restore{state, state.currentLabel, state.bdaStopValue}; + state.module.AddName(id, std::string(store ? "store" : "load") + "_formatted_gpu_buffer_x" + std::to_string(components)); + state.module.AddFunction(spv::OpFunction, resultType, id, spv::FunctionControlDontInlineMask, functionType); + const std::array types{u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32, valueType}; + std::array parameters{}; + for (std::uint32_t index = 0; index < (store ? 11u : 10u); ++index) { + parameters[index] = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionParameter, types[index], parameters[index]); + } + EmitLabel(state, state.module.AllocateId()); + const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); + if (stops) state.bdaStopValue = store ? ConstantU32(state, 1u) : state.module.Constant(spv::OpConstantComposite, resultType, zero, ConstantU32(state, 1u)); + state.bdaPcOverride = parameters[9]; + const RuntimeDescriptor shared{parameters[0], parameters[1], parameters[2], parameters[3], parameters[4], parameters[5], parameters[6], parameters[7], parameters[8]}; + const auto value = formatSwitch(context, instruction, shared, components, store, store ? parameters[10] : 0u); + if (!stops && store) { + state.module.AddFunction(spv::OpReturn); + } else { + auto returned = stops && store ? ConstantU32(state, 0u) : value; + if (stops && !store) { + returned = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, resultType, returned, value, ConstantU32(state, 0u)); + } + state.module.AddFunction(spv::OpReturnValue, returned); + } + state.module.AddFunction(spv::OpFunctionEnd); + function = state.formattedGpuBufferFunctions.emplace(key, id).first; + } + const auto offset = Binary(state, spv::OpIAdd, u32, context.Arg(instruction, 2u), ConstantU32(state, memory.offset)); + std::vector call{spv::OpFunctionCall, resultType, state.module.AllocateId(), function->second, resource.base, resource.records, resource.stride, resource.word3, resource.swizzle, resource.index, resource.valid, offset, context.Arg(instruction, 3u), BdaInstructionPc(state, instruction)}; + if (store) call.push_back(data); + state.module.AddFunction(call); + const auto result = call[2]; + if (!stops) return store ? ConstantU32(state, 0u) : result; + if (store) { + StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, result, ConstantU32(state, 0u))); + return ConstantU32(state, 0u); + } + const auto stopped = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, stopped, result, 1u); + StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, stopped, ConstantU32(state, 0u))); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, valueType, value, result, 0u); + return value; +} + } std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) { @@ -538,7 +613,7 @@ std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrVa faultIf(context, instruction, unaligned, access.guest, bits / 8u, BdaAbi::FaultReason::Unaligned); return EmitValueOrDefaultIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), type, zero, [&] { const auto physical = state.module.AllocateId(); - state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), ConstantU32(state, instruction.Flags().pc)); + state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), BdaInstructionPc(state, instruction)); const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)); return EmitValueOrDefaultIfCondition(state, mapped, type, zero, [&] { const auto pointer = state.module.AllocateId();