diff --git a/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp b/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp index b011accdef5..d4ae978ef0e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp @@ -55,8 +55,9 @@ Counts Count(std::span words) { return counts; } -Counts Compile(std::size_t loads, bool formattedStores = false) { +Counts Compile(std::size_t loads, bool formattedStores = false, bool gpu = false) { std::vector code; + if (gpu) code.push_back(0x7e160500u); for (std::uint32_t load = 0; load < loads; ++load) code.insert(code.end(), {0xe0000000u | (load * 4u), 0x80020000u | ((10u + load) << 8u)}); code.push_back(0xbf8c3f70u); for (std::uint32_t load = 0; load < loads; ++load) code.insert(code.end(), {(formattedStores ? 0xe0100000u : 0xe0700000u) | (load * 4u), 0x80010000u | ((10u + load) << 8u)}); @@ -93,7 +94,12 @@ int main() { Check(storeOne.functions == none.functions + 2 && storeOne.calls == none.calls + 2, "formatted buffer once: a formatted load and a formatted store to another resource emitted " + std::to_string(storeOne.functions - none.functions) + " functions and " + std::to_string(storeOne.calls - none.calls) + " calls more than none, expected one function and one call each"); Check(storeThree.functions == storeOne.functions && storeThree.calls == storeOne.calls + 4 && storeThree.switches == storeOne.switches, "formatted buffer once: two more formatted loads and stores emitted " + std::to_string(storeThree.functions - storeOne.functions) + " functions, " + std::to_string(storeThree.calls - storeOne.calls) + " calls and " + std::to_string(storeThree.switches - storeOne.switches) + " switches more, expected four calls only"); Check(three.words - one.words < one.words / 10u, "formatted buffer once: 2 more formatted loads added " + std::to_string(three.words - one.words) + " SPIR-V words to a " + std::to_string(one.words) + "-word module"); - std::printf("formatted buffer once tests passed (%zu words for 1 load, %zu for 3)\n", one.words, three.words); + const auto gpuOne = Compile(1, false, true); + const auto gpuThree = Compile(3, false, true); + Check(gpuThree.functions == gpuOne.functions && gpuThree.calls > gpuOne.calls, "formatted buffer once: two more formatted loads from a GPU-selected descriptor emitted " + std::to_string(gpuThree.functions - gpuOne.functions) + " more functions, expected calls to the shared one"); + Check(gpuThree.switches == gpuOne.switches, "formatted buffer once: two more formatted loads from a GPU-selected descriptor emitted " + std::to_string(gpuThree.switches - gpuOne.switches) + " more format switches"); + Check(gpuThree.words - gpuOne.words < gpuOne.words / 10u, "formatted buffer once: 2 more formatted loads from a GPU-selected descriptor added " + std::to_string(gpuThree.words - gpuOne.words) + " SPIR-V words to a " + std::to_string(gpuOne.words) + "-word module"); + std::printf("formatted buffer once tests passed (%zu words for 1 load, %zu for 3; GPU-selected %zu and %zu)\n", one.words, three.words, gpuOne.words, gpuThree.words); return 0; } catch (const std::exception& error) { std::fprintf(stderr, "%s\n", error.what()); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index 3d2b1cb5bff..0b308f556ce 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -24,6 +24,7 @@ void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& targe // every invocation running, so their faulting reads return zero instead. bool BdaInvocationsMayStop(const IrProgram& program); void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition); +std::uint32_t BdaInstructionPc(SpirvEmitterState& state, const IrValue& inst); std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits); void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u); void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index 6fdd75bd35c..7471636440e 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -104,6 +104,7 @@ struct SpirvEmitterState { std::uint32_t storageBufferU64Variable = 0; std::vector memoryByteOffsets; std::map, std::uint32_t> formattedBufferFunctions; + std::map, std::uint32_t> formattedGpuBufferFunctions; std::uint32_t bdaPagetableVariable = 0; std::uint32_t faultBufferVariable = 0; std::uint32_t bdaPointerFunction = 0; @@ -120,6 +121,7 @@ struct SpirvEmitterState { std::array, 2> bdaDwordReadFunctions {}; std::array, 2> bdaSpanReadFunctions {}; std::uint32_t bdaStopValue = 0; + std::uint32_t bdaPcOverride = 0; // False for programs with workgroup barriers: faulting BDA accesses then continue (see BdaInvocationsMayStop). bool bdaStopsInvocations = true; bool continueTarget = false; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index d580d06478b..371d7fc06bc 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -271,11 +271,15 @@ bool BdaByteReadsForced() { return forced; } +std::uint32_t BdaInstructionPc(SpirvEmitterState& state, const IrValue& inst) { + return state.bdaPcOverride != 0u ? state.bdaPcOverride : ConstantU32(state, inst.Flags().pc); +} + std::uint32_t AddBdaAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, bool subtract) { auto& state = ctx.state; const auto result = Binary(state, subtract ? spv::OpISub : spv::OpIAdd, TypeScalarU64(state), address, offset); const auto overflow = subtract ? Binary(state, spv::OpUGreaterThan, TypeBool(state), offset, address) : Binary(state, spv::OpULessThan, TypeBool(state), result, address); - EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, 0u), ConstantU32(state, inst.Flags().pc), BdaAbi::FaultReason::Overflow); }); + EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, 0u), BdaInstructionPc(state, inst), BdaAbi::FaultReason::Overflow); }); StopBdaInvocationIf(state, overflow); return result; } @@ -315,7 +319,7 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std:: if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA read width"); if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing"); if (bits == 32u) return EmitBdaDwordReads(ctx, inst, address, 0u, 1u)[0]; - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); EmitBdaOverflowCheck(state, address, bits / 8u, instruction); return EmitBdaBytes(state, address, bits / 8u, instruction, BdaAccessMask(state, inst)); } @@ -323,7 +327,7 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std:: void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits) { auto& state = ctx.state; if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA write width"); if (bits != 32u) { EmitBdaOverflowCheck(state, address, bits / 8u, instruction); @@ -405,7 +409,7 @@ void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t auto& state = ctx.state; if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA store width"); if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); const auto storeBytes = [&] { const auto function = state.bdaByteWriteFunctions[BdaCoherent(state, inst)]; if (function == 0) ctx.Fail(inst, "BDA byte write function is missing"); @@ -443,7 +447,7 @@ void EmitBdaDwordWrites(SpirvValueEmitContext& ctx, const IrValue& inst, std::ui storeDwords(); return; } - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); const auto accessMask = BdaAccessMask(state, inst); const auto physical = state.module.AllocateId(); state.module.AddFunction(spv::OpFunctionCall, u64, physical, state.bdaWriteProbeFunction, addresses[0], ConstantU32(state, dwords * 4u), instruction); @@ -498,7 +502,7 @@ std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std auto& state = ctx.state; if (bytes != 4u && bytes != 8u) ctx.Fail(inst, "unsupported BDA atomic width"); if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); const auto type = bytes == 8u ? TypeScalarU64(state) : TypeU32(state); const auto zero = bytes == 8u ? BdaConstant(state, 0u) : ConstantU32(state, 0u); const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), address, BdaConstant(state, bytes - 1u)), BdaConstant(state, 0u)); @@ -527,7 +531,7 @@ std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const auto& state = ctx.state; if (dwords == 0u || dwords > 4u) ctx.Fail(inst, "unsupported BDA read width"); if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); const auto used = everyDword ? (1u << dwords) - 1u : UsedBdaDwords(inst, dwords); const auto isUsed = [&](std::uint32_t dword) { return (used & (1u << dword)) != 0u; }; std::array values{}; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp index 42c036e7987..f841ffd0b06 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -7,6 +7,7 @@ #include "PipelineSpecialization.hpp" #include #include +#include #include namespace ShaderRecompiler { @@ -47,7 +48,7 @@ std::uint32_t nonzero(SpirvEmitterState& state, std::uint32_t value) { void faultIf(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, BdaAbi::FaultReason reason) { auto& state = context.state; - EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), ConstantU32(state, instruction.Flags().pc), reason); }); + EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), BdaInstructionPc(state, instruction), reason); }); StopBdaInvocationIf(state, condition); } @@ -319,6 +320,7 @@ std::uint32_t formatted(SpirvValueEmitContext& context, const IrValue& instructi std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data); std::uint32_t formattedCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data); +std::uint32_t formattedGpuCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data); std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) { auto& state = context.state; @@ -343,7 +345,7 @@ std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& i return formatted(context, instruction, resource, GetFormatInfo(format), components, store, data); } if (!memory.gpuDescriptor) return formattedCall(context, instruction, resource, components, store, data); - return formatSwitch(context, instruction, resource, components, store, data); + return formattedGpuCall(context, instruction, resource, components, store, data); } std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) { @@ -374,7 +376,7 @@ std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instru phi.insert(phi.end(), {value, exit}); } EmitLabel(state, invalid); - if (memory.gpuDescriptor) RecordBdaFault(state, resource.base, ConstantU32(state, 0u), ConstantU32(state, instruction.Flags().pc), BdaAbi::FaultReason::InvalidDescriptor); + if (memory.gpuDescriptor) RecordBdaFault(state, resource.base, ConstantU32(state, 0u), BdaInstructionPc(state, instruction), BdaAbi::FaultReason::InvalidDescriptor); const auto zero = store || components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); const auto invalidExit = state.module.AllocateId(); state.module.AddFunction(spv::OpBranch, invalidExit); @@ -443,6 +445,79 @@ std::uint32_t formattedCall(SpirvValueEmitContext& context, const IrValue& instr return store ? ConstantU32(state, 0u) : result; } +std::uint32_t formattedGpuCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + const auto u32 = TypeU32(state); + const auto u64 = TypeScalarU64(state); + const auto boolean = TypeBool(state); + const bool stops = state.bdaStopsInvocations; + const auto valueType = components == 1u ? u32 : TypeU32Composite(state, components); + const auto resultType = stops ? (store ? u32 : state.module.Type(spv::OpTypeStruct, valueType, u32)) : (store ? state.module.Type(spv::OpTypeVoid) : valueType); + const std::array key{components, store ? 1u : 0u, memory.d16 ? 1u : 0u, memory.coherent ? 1u : 0u, static_cast(memory.kind), stops ? 1u : 0u}; + auto function = state.formattedGpuBufferFunctions.find(key); + if (function == state.formattedGpuBufferFunctions.end()) { + const auto id = state.module.AllocateId(); + const auto functionType = store ? state.module.Type(spv::OpTypeFunction, resultType, u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32, valueType) + : state.module.Type(spv::OpTypeFunction, resultType, u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32); + struct Restore { + SpirvEmitterState& state; + std::uint32_t label; + std::uint32_t stopValue; + ~Restore() { + state.module.EndHelperFunction(); + state.currentLabel = label; + state.bdaStopValue = stopValue; + state.bdaPcOverride = 0u; + } + }; + state.module.BeginHelperFunction(); + const Restore restore{state, state.currentLabel, state.bdaStopValue}; + state.module.AddName(id, std::string(store ? "store" : "load") + "_formatted_gpu_buffer_x" + std::to_string(components)); + state.module.AddFunction(spv::OpFunction, resultType, id, spv::FunctionControlDontInlineMask, functionType); + const std::array types{u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32, valueType}; + std::array parameters{}; + for (std::uint32_t index = 0; index < (store ? 11u : 10u); ++index) { + parameters[index] = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionParameter, types[index], parameters[index]); + } + EmitLabel(state, state.module.AllocateId()); + const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); + if (stops) state.bdaStopValue = store ? ConstantU32(state, 1u) : state.module.Constant(spv::OpConstantComposite, resultType, zero, ConstantU32(state, 1u)); + state.bdaPcOverride = parameters[9]; + const RuntimeDescriptor shared{parameters[0], parameters[1], parameters[2], parameters[3], parameters[4], parameters[5], parameters[6], parameters[7], parameters[8]}; + const auto value = formatSwitch(context, instruction, shared, components, store, store ? parameters[10] : 0u); + if (!stops && store) { + state.module.AddFunction(spv::OpReturn); + } else { + auto returned = stops && store ? ConstantU32(state, 0u) : value; + if (stops && !store) { + returned = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, resultType, returned, value, ConstantU32(state, 0u)); + } + state.module.AddFunction(spv::OpReturnValue, returned); + } + state.module.AddFunction(spv::OpFunctionEnd); + function = state.formattedGpuBufferFunctions.emplace(key, id).first; + } + const auto offset = Binary(state, spv::OpIAdd, u32, context.Arg(instruction, 2u), ConstantU32(state, memory.offset)); + std::vector call{spv::OpFunctionCall, resultType, state.module.AllocateId(), function->second, resource.base, resource.records, resource.stride, resource.word3, resource.swizzle, resource.index, resource.valid, offset, context.Arg(instruction, 3u), BdaInstructionPc(state, instruction)}; + if (store) call.push_back(data); + state.module.AddFunction(call); + const auto result = call[2]; + if (!stops) return store ? ConstantU32(state, 0u) : result; + if (store) { + StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, result, ConstantU32(state, 0u))); + return ConstantU32(state, 0u); + } + const auto stopped = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, stopped, result, 1u); + StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, stopped, ConstantU32(state, 0u))); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, valueType, value, result, 0u); + return value; +} + } std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) { @@ -538,7 +613,7 @@ std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrVa faultIf(context, instruction, unaligned, access.guest, bits / 8u, BdaAbi::FaultReason::Unaligned); return EmitValueOrDefaultIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), type, zero, [&] { const auto physical = state.module.AllocateId(); - state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), ConstantU32(state, instruction.Flags().pc)); + state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), BdaInstructionPc(state, instruction)); const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)); return EmitValueOrDefaultIfCondition(state, mapped, type, zero, [&] { const auto pointer = state.module.AllocateId();