Skip to content

fix(recompiler): emit a GPU-selected formatted buffer access through one shared function #2349

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adriadam10 merged 1 commit into from
Oct 10, 2026
Merged

fix(recompiler): emit a GPU-selected formatted buffer access through one shared function #2349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,9 @@ Counts Count(std::span<const std::uint32_t> words) {
return counts;
}

Counts Compile(std::size_t loads, bool formattedStores = false) {
Counts Compile(std::size_t loads, bool formattedStores = false, bool gpu = false) {
std::vector<std::uint32_t> code;
if (gpu) code.push_back(0x7e160500u);
for (std::uint32_t load = 0; load < loads; ++load) code.insert(code.end(), {0xe0000000u | (load * 4u), 0x80020000u | ((10u + load) << 8u)});
code.push_back(0xbf8c3f70u);
for (std::uint32_t load = 0; load < loads; ++load) code.insert(code.end(), {(formattedStores ? 0xe0100000u : 0xe0700000u) | (load * 4u), 0x80010000u | ((10u + load) << 8u)});
Expand Down Expand Up @@ -93,7 +94,12 @@ int main() {
Check(storeOne.functions == none.functions + 2 && storeOne.calls == none.calls + 2, "formatted buffer once: a formatted load and a formatted store to another resource emitted " + std::to_string(storeOne.functions - none.functions) + " functions and " + std::to_string(storeOne.calls - none.calls) + " calls more than none, expected one function and one call each");
Check(storeThree.functions == storeOne.functions && storeThree.calls == storeOne.calls + 4 && storeThree.switches == storeOne.switches, "formatted buffer once: two more formatted loads and stores emitted " + std::to_string(storeThree.functions - storeOne.functions) + " functions, " + std::to_string(storeThree.calls - storeOne.calls) + " calls and " + std::to_string(storeThree.switches - storeOne.switches) + " switches more, expected four calls only");
Check(three.words - one.words < one.words / 10u, "formatted buffer once: 2 more formatted loads added " + std::to_string(three.words - one.words) + " SPIR-V words to a " + std::to_string(one.words) + "-word module");
std::printf("formatted buffer once tests passed (%zu words for 1 load, %zu for 3)\n", one.words, three.words);
const auto gpuOne = Compile(1, false, true);
const auto gpuThree = Compile(3, false, true);
Check(gpuThree.functions == gpuOne.functions && gpuThree.calls > gpuOne.calls, "formatted buffer once: two more formatted loads from a GPU-selected descriptor emitted " + std::to_string(gpuThree.functions - gpuOne.functions) + " more functions, expected calls to the shared one");
Check(gpuThree.switches == gpuOne.switches, "formatted buffer once: two more formatted loads from a GPU-selected descriptor emitted " + std::to_string(gpuThree.switches - gpuOne.switches) + " more format switches");
Check(gpuThree.words - gpuOne.words < gpuOne.words / 10u, "formatted buffer once: 2 more formatted loads from a GPU-selected descriptor added " + std::to_string(gpuThree.words - gpuOne.words) + " SPIR-V words to a " + std::to_string(gpuOne.words) + "-word module");
std::printf("formatted buffer once tests passed (%zu words for 1 load, %zu for 3; GPU-selected %zu and %zu)\n", one.words, three.words, gpuOne.words, gpuThree.words);
return 0;
} catch (const std::exception& error) {
std::fprintf(stderr, "%s\n", error.what());
Expand Down
1 change: 1 addition & 0 deletions core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& targe
// every invocation running, so their faulting reads return zero instead.
bool BdaInvocationsMayStop(const IrProgram& program);
void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition);
std::uint32_t BdaInstructionPc(SpirvEmitterState& state, const IrValue& inst);
std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits);
void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u);
void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits);
Expand Down
2 changes: 2 additions & 0 deletions core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ struct SpirvEmitterState {
std::uint32_t storageBufferU64Variable = 0;
std::vector<std::uint32_t> memoryByteOffsets;
std::map<std::array<std::uint32_t, 6>, std::uint32_t> formattedBufferFunctions;
std::map<std::array<std::uint32_t, 6>, std::uint32_t> formattedGpuBufferFunctions;
std::uint32_t bdaPagetableVariable = 0;
std::uint32_t faultBufferVariable = 0;
std::uint32_t bdaPointerFunction = 0;
Expand All @@ -120,6 +121,7 @@ struct SpirvEmitterState {
std::array<std::array<std::uint32_t, 2>, 2> bdaDwordReadFunctions {};
std::array<std::array<std::uint32_t, 2>, 2> bdaSpanReadFunctions {};
std::uint32_t bdaStopValue = 0;
std::uint32_t bdaPcOverride = 0;
// False for programs with workgroup barriers: faulting BDA accesses then continue (see BdaInvocationsMayStop).
bool bdaStopsInvocations = true;
bool continueTarget = false;
Expand Down
18 changes: 11 additions & 7 deletions core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,11 +271,15 @@ bool BdaByteReadsForced() {
return forced;
}

std::uint32_t BdaInstructionPc(SpirvEmitterState& state, const IrValue& inst) {
return state.bdaPcOverride != 0u ? state.bdaPcOverride : ConstantU32(state, inst.Flags<MemoryFlags>().pc);
}

std::uint32_t AddBdaAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, bool subtract) {
auto& state = ctx.state;
const auto result = Binary(state, subtract ? spv::OpISub : spv::OpIAdd, TypeScalarU64(state), address, offset);
const auto overflow = subtract ? Binary(state, spv::OpUGreaterThan, TypeBool(state), offset, address) : Binary(state, spv::OpULessThan, TypeBool(state), result, address);
EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, 0u), ConstantU32(state, inst.Flags<MemoryFlags>().pc), BdaAbi::FaultReason::Overflow); });
EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, 0u), BdaInstructionPc(state, inst), BdaAbi::FaultReason::Overflow); });
StopBdaInvocationIf(state, overflow);
return result;
}
Expand Down Expand Up @@ -315,15 +319,15 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::
if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA read width");
if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing");
if (bits == 32u) return EmitBdaDwordReads(ctx, inst, address, 0u, 1u)[0];
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
const auto instruction = BdaInstructionPc(state, inst);
EmitBdaOverflowCheck(state, address, bits / 8u, instruction);
return EmitBdaBytes(state, address, bits / 8u, instruction, BdaAccessMask(state, inst));
}

void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits) {
auto& state = ctx.state;
if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing");
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
const auto instruction = BdaInstructionPc(state, inst);
if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA write width");
if (bits != 32u) {
EmitBdaOverflowCheck(state, address, bits / 8u, instruction);
Expand Down Expand Up @@ -405,7 +409,7 @@ void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t
auto& state = ctx.state;
if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA store width");
if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing");
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
const auto instruction = BdaInstructionPc(state, inst);
const auto storeBytes = [&] {
const auto function = state.bdaByteWriteFunctions[BdaCoherent(state, inst)];
if (function == 0) ctx.Fail(inst, "BDA byte write function is missing");
Expand Down Expand Up @@ -443,7 +447,7 @@ void EmitBdaDwordWrites(SpirvValueEmitContext& ctx, const IrValue& inst, std::ui
storeDwords();
return;
}
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
const auto instruction = BdaInstructionPc(state, inst);
const auto accessMask = BdaAccessMask(state, inst);
const auto physical = state.module.AllocateId();
state.module.AddFunction(spv::OpFunctionCall, u64, physical, state.bdaWriteProbeFunction, addresses[0], ConstantU32(state, dwords * 4u), instruction);
Expand Down Expand Up @@ -498,7 +502,7 @@ std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std
auto& state = ctx.state;
if (bytes != 4u && bytes != 8u) ctx.Fail(inst, "unsupported BDA atomic width");
if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing");
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
const auto instruction = BdaInstructionPc(state, inst);
const auto type = bytes == 8u ? TypeScalarU64(state) : TypeU32(state);
const auto zero = bytes == 8u ? BdaConstant(state, 0u) : ConstantU32(state, 0u);
const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), address, BdaConstant(state, bytes - 1u)), BdaConstant(state, 0u));
Expand Down Expand Up @@ -527,7 +531,7 @@ std::array<std::uint32_t, 4> EmitBdaDwordReads(SpirvValueEmitContext& ctx, const
auto& state = ctx.state;
if (dwords == 0u || dwords > 4u) ctx.Fail(inst, "unsupported BDA read width");
if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing");
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
const auto instruction = BdaInstructionPc(state, inst);
const auto used = everyDword ? (1u << dwords) - 1u : UsedBdaDwords(inst, dwords);
const auto isUsed = [&](std::uint32_t dword) { return (used & (1u << dword)) != 0u; };
std::array<std::uint32_t, 4> values{};
Expand Down
83 changes: 79 additions & 4 deletions core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "PipelineSpecialization.hpp"
#include <algorithm>
#include <array>
#include <string>
#include <vector>

namespace ShaderRecompiler {
Expand Down Expand Up @@ -47,7 +48,7 @@ std::uint32_t nonzero(SpirvEmitterState& state, std::uint32_t value) {

void faultIf(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, BdaAbi::FaultReason reason) {
auto& state = context.state;
EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), ConstantU32(state, instruction.Flags<MemoryFlags>().pc), reason); });
EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), BdaInstructionPc(state, instruction), reason); });
StopBdaInvocationIf(state, condition);
}

Expand Down Expand Up @@ -319,6 +320,7 @@ std::uint32_t formatted(SpirvValueEmitContext& context, const IrValue& instructi

std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data);
std::uint32_t formattedCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data);
std::uint32_t formattedGpuCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data);

std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) {
auto& state = context.state;
Expand All @@ -343,7 +345,7 @@ std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& i
return formatted(context, instruction, resource, GetFormatInfo(format), components, store, data);
}
if (!memory.gpuDescriptor) return formattedCall(context, instruction, resource, components, store, data);
return formatSwitch(context, instruction, resource, components, store, data);
return formattedGpuCall(context, instruction, resource, components, store, data);
}

std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) {
Expand Down Expand Up @@ -374,7 +376,7 @@ std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instru
phi.insert(phi.end(), {value, exit});
}
EmitLabel(state, invalid);
if (memory.gpuDescriptor) RecordBdaFault(state, resource.base, ConstantU32(state, 0u), ConstantU32(state, instruction.Flags<MemoryFlags>().pc), BdaAbi::FaultReason::InvalidDescriptor);
if (memory.gpuDescriptor) RecordBdaFault(state, resource.base, ConstantU32(state, 0u), BdaInstructionPc(state, instruction), BdaAbi::FaultReason::InvalidDescriptor);
const auto zero = store || components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components);
const auto invalidExit = state.module.AllocateId();
state.module.AddFunction(spv::OpBranch, invalidExit);
Expand Down Expand Up @@ -443,6 +445,79 @@ std::uint32_t formattedCall(SpirvValueEmitContext& context, const IrValue& instr
return store ? ConstantU32(state, 0u) : result;
}

std::uint32_t formattedGpuCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) {
auto& state = context.state;
const auto& memory = context.Memory(instruction);
const auto u32 = TypeU32(state);
const auto u64 = TypeScalarU64(state);
const auto boolean = TypeBool(state);
const bool stops = state.bdaStopsInvocations;
const auto valueType = components == 1u ? u32 : TypeU32Composite(state, components);
const auto resultType = stops ? (store ? u32 : state.module.Type(spv::OpTypeStruct, valueType, u32)) : (store ? state.module.Type(spv::OpTypeVoid) : valueType);
const std::array<std::uint32_t, 6> key{components, store ? 1u : 0u, memory.d16 ? 1u : 0u, memory.coherent ? 1u : 0u, static_cast<std::uint32_t>(memory.kind), stops ? 1u : 0u};
auto function = state.formattedGpuBufferFunctions.find(key);
if (function == state.formattedGpuBufferFunctions.end()) {
const auto id = state.module.AllocateId();
const auto functionType = store ? state.module.Type(spv::OpTypeFunction, resultType, u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32, valueType)
: state.module.Type(spv::OpTypeFunction, resultType, u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32);
struct Restore {
SpirvEmitterState& state;
std::uint32_t label;
std::uint32_t stopValue;
~Restore() {
state.module.EndHelperFunction();
state.currentLabel = label;
state.bdaStopValue = stopValue;
state.bdaPcOverride = 0u;
}
};
state.module.BeginHelperFunction();
const Restore restore{state, state.currentLabel, state.bdaStopValue};
state.module.AddName(id, std::string(store ? "store" : "load") + "_formatted_gpu_buffer_x" + std::to_string(components));
state.module.AddFunction(spv::OpFunction, resultType, id, spv::FunctionControlDontInlineMask, functionType);
const std::array<std::uint32_t, 11> types{u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32, valueType};
std::array<std::uint32_t, 11> parameters{};
for (std::uint32_t index = 0; index < (store ? 11u : 10u); ++index) {
parameters[index] = state.module.AllocateId();
state.module.AddFunction(spv::OpFunctionParameter, types[index], parameters[index]);
}
EmitLabel(state, state.module.AllocateId());
const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components);
if (stops) state.bdaStopValue = store ? ConstantU32(state, 1u) : state.module.Constant(spv::OpConstantComposite, resultType, zero, ConstantU32(state, 1u));
state.bdaPcOverride = parameters[9];
const RuntimeDescriptor shared{parameters[0], parameters[1], parameters[2], parameters[3], parameters[4], parameters[5], parameters[6], parameters[7], parameters[8]};
const auto value = formatSwitch(context, instruction, shared, components, store, store ? parameters[10] : 0u);
if (!stops && store) {
state.module.AddFunction(spv::OpReturn);
} else {
auto returned = stops && store ? ConstantU32(state, 0u) : value;
if (stops && !store) {
returned = state.module.AllocateId();
state.module.AddFunction(spv::OpCompositeConstruct, resultType, returned, value, ConstantU32(state, 0u));
}
state.module.AddFunction(spv::OpReturnValue, returned);
}
state.module.AddFunction(spv::OpFunctionEnd);
function = state.formattedGpuBufferFunctions.emplace(key, id).first;
}
const auto offset = Binary(state, spv::OpIAdd, u32, context.Arg(instruction, 2u), ConstantU32(state, memory.offset));
std::vector<std::uint32_t> call{spv::OpFunctionCall, resultType, state.module.AllocateId(), function->second, resource.base, resource.records, resource.stride, resource.word3, resource.swizzle, resource.index, resource.valid, offset, context.Arg(instruction, 3u), BdaInstructionPc(state, instruction)};
if (store) call.push_back(data);
state.module.AddFunction(call);
const auto result = call[2];
if (!stops) return store ? ConstantU32(state, 0u) : result;
if (store) {
StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, result, ConstantU32(state, 0u)));
return ConstantU32(state, 0u);
}
const auto stopped = state.module.AllocateId();
state.module.AddFunction(spv::OpCompositeExtract, u32, stopped, result, 1u);
StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, stopped, ConstantU32(state, 0u)));
const auto value = state.module.AllocateId();
state.module.AddFunction(spv::OpCompositeExtract, valueType, value, result, 0u);
return value;
}

}

std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) {
Expand Down Expand Up @@ -538,7 +613,7 @@ std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrVa
faultIf(context, instruction, unaligned, access.guest, bits / 8u, BdaAbi::FaultReason::Unaligned);
return EmitValueOrDefaultIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), type, zero, [&] {
const auto physical = state.module.AllocateId();
state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), ConstantU32(state, instruction.Flags<MemoryFlags>().pc));
state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), BdaInstructionPc(state, instruction));
const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u));
return EmitValueOrDefaultIfCondition(state, mapped, type, zero, [&] {
const auto pointer = state.module.AllocateId();
Expand Down
Loading