diff --git a/core/libs/prx/libSceAgcDriver/tests/TessellationLowering.cpp b/core/libs/prx/libSceAgcDriver/tests/TessellationLowering.cpp index 591d5cddd19..b978b4c0186 100644 --- a/core/libs/prx/libSceAgcDriver/tests/TessellationLowering.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/TessellationLowering.cpp @@ -4,11 +4,14 @@ #include "Optimization/TessellationLowering.hpp" #include "SpirvBackend/SpirvEmitter.hpp" #include +#include +#include #include #include #include #include #include +#include #include #include @@ -20,6 +23,12 @@ constexpr std::uint32_t InputControlPoints = 3u; constexpr std::uint32_t LocalStride = 32u; constexpr std::uint32_t PatchStride = InputControlPoints * LocalStride; constexpr std::uint32_t ReadPc = 0x40u; +constexpr std::uint32_t OutputControlPoints = 4u; +constexpr std::uint32_t ControlStride = 64u; +constexpr std::uint32_t ControlPatchStride = OutputControlPoints * ControlStride; +constexpr std::uint32_t RingMemory = 1u; +constexpr std::uint32_t RingWritePc = 0x80u; +constexpr std::uint32_t RingReadPc = 0xc0u; bool IsKind(const IrValue* value, TessellationAttribute kind) { return value->HasImmediate() && value->Type() == IrType::U32 && value->ImmediateU32() == static_cast(kind); @@ -33,6 +42,7 @@ struct Hull { Hull() { program.Resources().stage = IrShaderStage::TessellationControl; program.Resources().memoryInfo.push_back(MemoryInfo{.kind = ResourceKind::Lds}); + program.Resources().memoryInfo.push_back(MemoryInfo{.kind = ResourceKind::Buffer, .offen = true}); static_cast(newBlock()); } @@ -96,6 +106,35 @@ struct Hull { return builder.Emit(IrOpcode::INotEqual32, IrType::Bool, {&bit, &builder.Constant(0u)}); } + IrValue& ring(IrValue& offset) { + return builder.IAdd(builder.IMul(relativePatch(), builder.Constant(ControlPatchStride)), offset); + } + + IrValue& ownControlPoint(std::uint32_t field) { + return ring(builder.IAdd(builder.IMul(invocation(), builder.Constant(ControlStride)), builder.Constant(field))); + } + + IrValue& controlPoint(std::uint32_t point, std::uint32_t field) { + return ring(builder.Constant(point * ControlStride + field)); + } + + IrValue& ringResource() { + return builder.Emit(IrOpcode::GetBufferResource, IrType::BufferResource, {&builder.Constant(0u), &builder.Constant(0u), &builder.Constant(0u), &builder.Constant(0u)}); + } + + IrValue& offChip() { + return builder.Emit(IrOpcode::TessellationBase, IrType::U32, {&builder.Constant(static_cast(TessellationBaseKind::OffChip))}); + } + + void writeRing(IrValue& offset) { + static_cast(builder.Emit(IrOpcode::StoreBufferU32, IrType::Void, {&ringResource(), &builder.Constant(0u), &offset, &offChip(), &builder.Constant(1u), &builder.ConstantBool(true)}, MemoryFlags{RingMemory, RingWritePc})); + } + + void readRing(IrValue& offset, IrValue& predicate) { + auto& loaded = builder.Emit(IrOpcode::LoadBufferU32, IrType::U32, {&ringResource(), &builder.Constant(0u), &offset, &offChip(), &predicate}, MemoryFlags{RingMemory, RingReadPc}); + static_cast(builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {&loaded})); + } + IrValue& merge(std::uint32_t firstMultiple, std::uint32_t secondMultiple) { IrBlock& entry = *block; IrBlock& first = newBlock(); @@ -124,6 +163,7 @@ struct Outcome { }; const ShaderTessellationInputInfo Tessellation{.inputControlPoints = InputControlPoints, .outputControlPoints = InputControlPoints, .patchesPerGroup = 1u}; +const ShaderTessellationInputInfo RingTessellation{.inputControlPoints = InputControlPoints, .outputControlPoints = OutputControlPoints, .patchesPerGroup = 1u}; const ShaderTessellationInputInfo GroupedTessellation{.inputControlPoints = InputControlPoints, .outputControlPoints = InputControlPoints, .patchesPerGroup = 1u, .primitiveGroupSize = 1u}; Outcome Lower(IrProgram& program, const ShaderTessellationInputInfo& tessellation = Tessellation) { @@ -157,12 +197,87 @@ bool RefusedBecause(IrProgram& program, const char* reason) { return !outcome.lowered && outcome.reason.find(reason) != std::string::npos; } -std::uint32_t EmittedEvaluationControlPoint(std::uint32_t address) { +bool LowersRing(IrProgram& program) { + const Outcome outcome = Lower(program, RingTessellation); + if (!outcome.lowered) std::fprintf(stderr, "refused: %s\n", outcome.reason.c_str()); + return outcome.lowered; +} + +bool RingRefusedBecause(IrProgram& program, const char* reason) { + const Outcome outcome = Lower(program, RingTessellation); + if (outcome.lowered) std::fprintf(stderr, "lowered instead of refusing: %s\n", reason); + if (!outcome.lowered && outcome.reason.find(reason) == std::string::npos) std::fprintf(stderr, "refused for another reason: %s\n", outcome.reason.c_str()); + return !outcome.lowered && outcome.reason.find(reason) != std::string::npos; +} + +bool IsBarrier(const IrValue* inst) { + return inst->Opcode() == IrOpcode::Barrier; +} + +bool IsBranch(const IrValue* inst) { + return inst->Opcode() == IrOpcode::Branch || inst->Opcode() == IrOpcode::BranchConditional; +} + +bool IsControlOutputWrite(const IrValue* inst) { + return inst->Opcode() == IrOpcode::SetTessellationAttribute && IsKind(inst->Argument(0), TessellationAttribute::ControlOutput); +} + +bool IsControlOutputRead(const IrValue* inst) { + return inst->Opcode() == IrOpcode::GetTessellationAttribute && IsKind(inst->Argument(0), TessellationAttribute::ControlOutput); +} + +std::ptrdiff_t FirstPosition(const IrBlock& block, bool (*match)(const IrValue*)) { + const auto& instructions = block.Instructions(); + const auto found = std::find_if(instructions.begin(), instructions.end(), match); + return found == instructions.end() ? -1 : std::distance(instructions.begin(), found); +} + +std::ptrdiff_t LastPosition(const IrBlock& block, bool (*match)(const IrValue*)) { + const auto& instructions = block.Instructions(); + const auto found = std::find_if(instructions.rbegin(), instructions.rend(), match); + return found == instructions.rend() ? -1 : std::distance(instructions.begin(), found.base()) - 1; +} + +std::size_t Barriers(const IrProgram& program) { + std::size_t count = 0; + for (const IrBlock* block : program.BlockOrder()) count += static_cast(std::count_if(block->Instructions().begin(), block->Instructions().end(), IsBarrier)); + return count; +} + +std::uint32_t ReadAddress(const IrBlock& block) { + for (const IrValue* inst : block.Instructions()) { + if (!IsControlOutputRead(inst)) continue; + const IrValue* address = inst->Argument(1)->Resolve(); + return address->HasImmediate() ? address->ImmediateU32() : ~0u; + } + return ~0u; +} + +bool BarrierBetweenWritesAndBranch(const IrBlock& block) { + const auto barrier = FirstPosition(block, IsBarrier); + return barrier > LastPosition(block, IsControlOutputWrite) && barrier < FirstPosition(block, IsBranch); +} + +void SkipHullProgram(Hull& hull, IrBlock& entry, std::uint32_t lanes, IrBlock& skip, IrBlock& body) { + hull.builder.SetInsertionPoint(entry); + auto& lane = hull.builder.Emit(IrOpcode::LaneId, IrType::U32, {}); + auto& outside = hull.builder.Emit(IrOpcode::LogicalNot, IrType::Bool, {&hull.builder.Emit(IrOpcode::ULessThan32, IrType::Bool, {&lane, &hull.builder.Constant(lanes)})}); + hull.builder.BranchConditional(outside, skip, body); + constexpr std::uint32_t Renumbered = 100u; + for (auto& info : hull.program.Metadata().blockInfo) info.id += Renumbered; + auto& info = hull.program.Metadata().blockInfo.front(); + info.terminator.kind = TerminatorKind::ConditionalBranch; + info.terminator.trueBlock = skip.Id() + Renumbered; + info.terminator.falseBlock = body.Id() + Renumbered; + info.condition = &outside; +} + +std::uint32_t EmittedControlPoint(IrShaderStage stage, TessellationAttribute kind, std::uint32_t address) { constexpr std::uint32_t ControlStride = 16u; IrProgram program; IrBuilder builder{program}; program.SetWaveSize(64); - program.Resources().stage = IrShaderStage::TessellationEvaluation; + program.Resources().stage = stage; program.Resources().srtPlanComplete = true; program.Resources().resourceTrackingComplete = true; program.Metadata().shaderInfoComplete = true; @@ -175,7 +290,7 @@ std::uint32_t EmittedEvaluationControlPoint(std::uint32_t address) { info.id = block.Id(); program.Metadata().blockInfo.push_back(info); builder.SetInsertionPoint(block); - auto& read = builder.Emit(IrOpcode::GetTessellationAttribute, IrType::U32, {&builder.Constant(static_cast(TessellationAttribute::EvaluationInput)), &builder.Constant(address), &builder.ConstantBool(true)}); + auto& read = builder.Emit(IrOpcode::GetTessellationAttribute, IrType::U32, {&builder.Constant(static_cast(kind)), &builder.Constant(address), &builder.ConstantBool(true)}); static_cast(builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {&read})); ShaderVertexInputInfo vertex; vertex.tess.inputControlPoints = InputControlPoints; @@ -195,12 +310,16 @@ std::uint32_t EmittedEvaluationControlPoint(std::uint32_t address) { std::unordered_map constants; for (std::size_t at = 5; at < words.size(); at += words[at] >> 16u) { const auto count = words[at] >> 16u; - if (count == 0u || at + count > words.size()) throw std::runtime_error("the domain program compiled to a malformed SPIR-V instruction"); + if (count == 0u || at + count > words.size()) throw std::runtime_error("the tessellation program compiled to a malformed SPIR-V instruction"); const auto opcode = words[at] & 0xffffu; if (opcode == spv::OpConstant && count == 4u) constants[words[at + 2u]] = words[at + 3u]; if (opcode == spv::OpAccessChain && count == 7u) return constants.at(words[at + 4u]); } - throw std::runtime_error("the domain program reads no control point attribute"); + throw std::runtime_error("the tessellation program reads no control point attribute"); +} + +std::uint32_t EmittedEvaluationControlPoint(std::uint32_t address) { + return EmittedControlPoint(IrShaderStage::TessellationEvaluation, TessellationAttribute::EvaluationInput, address); } enum class Conjunction { None, Before, After }; @@ -309,5 +428,127 @@ int main() { return EmittedEvaluationControlPoint(InputControlPoints * 16u) == InputControlPoints - 1u; }); + passed &= run("a hull read of another control point's off-chip output behind a branch gets a barrier after every control point's writes", [] { + Hull hull; + hull.writeRing(hull.ownControlPoint(0u)); + hull.writeRing(hull.ownControlPoint(16u)); + IrBlock& entry = *hull.block; + IrBlock& reader = hull.newBlock(); + IrBlock& done = hull.newBlock(); + hull.builder.SetInsertionPoint(entry); + hull.builder.BranchConditional(hull.firstInvocation(), reader, done); + hull.builder.SetInsertionPoint(reader); + hull.readRing(hull.controlPoint(1u, 16u), hull.firstInvocation()); + hull.builder.Branch(done); + return LowersRing(hull.program) && Barriers(hull.program) == 1u && BarrierBetweenWritesAndBranch(entry) && ReadAddress(reader) == ControlStride + 16u; + }); + + passed &= run("a hull read of another control point's off-chip output in the block that writes it gets a barrier between them", [] { + Hull hull; + hull.writeRing(hull.ownControlPoint(0u)); + hull.readRing(hull.controlPoint(2u, 0u), hull.builder.ConstantBool(true)); + if (!LowersRing(hull.program) || Barriers(hull.program) != 1u) return false; + const auto barrier = FirstPosition(*hull.block, IsBarrier); + return barrier > LastPosition(*hull.block, IsControlOutputWrite) && barrier < FirstPosition(*hull.block, IsControlOutputRead) && ReadAddress(*hull.block) == 2u * ControlStride; + }); + + passed &= run("a hull read of its own control point's off-chip output needs no barrier", [] { + Hull hull; + hull.writeRing(hull.ownControlPoint(0u)); + hull.readRing(hull.ownControlPoint(0u), hull.builder.ConstantBool(true)); + if (!LowersRing(hull.program) || Barriers(hull.program) != 0u) return false; + const auto read = FirstPosition(*hull.block, IsControlOutputRead); + if (read < 0) return false; + const IrValue* address = (*std::next(hull.block->Instructions().begin(), read))->Argument(1)->Resolve(); + if (address->Opcode() != IrOpcode::IAdd32) return false; + const IrValue* scaled = address->Argument(0)->Resolve(); + return scaled->Opcode() == IrOpcode::IMul32 && scaled->Argument(1)->Resolve()->ImmediateU32() == ControlStride; + }); + + passed &= run("a branch around the hull program that no control point takes does not keep the barrier from its writes", [] { + Hull hull; + IrBlock& entry = *hull.block; + IrBlock& body = hull.newBlock(); + hull.writeRing(hull.ownControlPoint(0u)); + IrBlock& reader = hull.newBlock(); + hull.readRing(hull.controlPoint(1u, 0u), hull.firstInvocation()); + IrBlock& done = hull.newBlock(); + SkipHullProgram(hull, entry, OutputControlPoints, done, body); + hull.builder.SetInsertionPoint(body); + hull.builder.BranchConditional(hull.firstInvocation(), reader, done); + hull.builder.SetInsertionPoint(reader); + hull.builder.Branch(done); + return LowersRing(hull.program) && Barriers(hull.program) == 1u && BarrierBetweenWritesAndBranch(body); + }); + + passed &= run("a branch around the hull program that some control points take is refused", [] { + Hull hull; + IrBlock& entry = *hull.block; + IrBlock& body = hull.newBlock(); + hull.writeRing(hull.ownControlPoint(0u)); + IrBlock& reader = hull.newBlock(); + hull.readRing(hull.controlPoint(1u, 0u), hull.firstInvocation()); + IrBlock& done = hull.newBlock(); + SkipHullProgram(hull, entry, OutputControlPoints - 1u, done, body); + hull.builder.SetInsertionPoint(body); + hull.builder.BranchConditional(hull.firstInvocation(), reader, done); + hull.builder.SetInsertionPoint(reader); + hull.builder.Branch(done); + return RingRefusedBecause(hull.program, "one of its writes there comes after the last block every invocation reaches before the read"); + }); + + passed &= run("a hull read of another control point's off-chip output that no control point reaches is refused", [] { + Hull hull; + IrBlock& entry = *hull.block; + IrBlock& body = hull.newBlock(); + hull.writeRing(hull.ownControlPoint(0u)); + hull.readRing(hull.controlPoint(1u, 0u), hull.builder.ConstantBool(true)); + IrBlock& done = hull.newBlock(); + SkipHullProgram(hull, entry, 0u, done, body); + hull.builder.SetInsertionPoint(body); + hull.builder.Branch(done); + return RingRefusedBecause(hull.program, "in a block no control point reaches"); + }); + + passed &= run("a hull write of its off-chip output after another control point's read is refused", [] { + Hull hull; + hull.writeRing(hull.ownControlPoint(0u)); + hull.readRing(hull.controlPoint(1u, 0u), hull.builder.ConstantBool(true)); + hull.writeRing(hull.ownControlPoint(16u)); + return RingRefusedBecause(hull.program, "one of its writes there comes after the last block every invocation reaches before the read"); + }); + + passed &= run("a hull read of another control point's off-chip output in a loop through the program's entry is refused", [] { + Hull hull; + IrBlock& entry = *hull.block; + hull.writeRing(hull.ownControlPoint(0u)); + hull.readRing(hull.controlPoint(1u, 0u), hull.builder.ConstantBool(true)); + IrBlock& done = hull.newBlock(); + hull.builder.SetInsertionPoint(entry); + hull.builder.BranchConditional(hull.firstInvocation(), entry, done); + return RingRefusedBecause(hull.program, "no block before the read is reached once by every invocation"); + }); + + passed &= run("a hull read of off-chip memory where no control point writes is refused", [] { + Hull hull; + hull.readRing(hull.controlPoint(1u, 0u), hull.builder.ConstantBool(true)); + return RingRefusedBecause(hull.program, "writes no control point there"); + }); + + passed &= run("a hull read past its patch's off-chip control points is refused", [] { + Hull hull; + hull.writeRing(hull.ownControlPoint(0u)); + hull.readRing(hull.controlPoint(OutputControlPoints, 0u), hull.builder.ConstantBool(true)); + return RingRefusedBecause(hull.program, "reads off-chip memory outside its patch's control points"); + }); + + passed &= run("a hull read of another control point's output at an immediate address reads that control point", [] { + return EmittedControlPoint(IrShaderStage::TessellationControl, TessellationAttribute::ControlOutput, 2u * 16u + 4u) == 2u; + }); + + passed &= run("a hull read of another control point's output at an immediate address past the last control point reads the last control point", [] { + return EmittedControlPoint(IrShaderStage::TessellationControl, TessellationAttribute::ControlOutput, InputControlPoints * 16u) == InputControlPoints - 1u; + }); + return passed ? 0 : 1; } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Tessellation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Tessellation.cpp index 3beb74a3e23..a8611724803 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Tessellation.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Tessellation.cpp @@ -476,6 +476,21 @@ alignas(256) constexpr std::array MergedHullReadLane{ 0x4a162481, 0x16161690, 0x4c1616ff, 0x00008000, 0xe0741000, 0x0202070b, 0xbf810000, }; +alignas(256) constexpr std::array MergedHullRingReadBack{ + 0xbefe04c1, 0xd7650000, 0x000100c1, 0xd7660000, 0x000200c1, 0x7da206f9, 0x80060000, 0xbf88001c, + 0xd5480004, 0x02090502, 0xd5480005, 0x02090102, 0xd5480006, 0x02110902, 0xd5480007, 0x02111102, + 0x7e080d04, 0x7e0a0d05, 0x7e0c0d06, 0x7e0e0d07, 0x10100cff, 0x3f19999a, 0x061010ff, 0xbf666666, + 0x3e1008ff, 0x3e800000, 0xd54b0009, 0x03c5e105, 0x161406a0, 0xd9340000, 0x0000080a, 0xd8340008, + 0x0000070a, 0xd8340010, 0x0000070a, 0xbf8cc07f, 0xbefe04c1, 0xbf8cc07f, 0xd7650000, 0x000100c1, + 0xd7660000, 0x000200c1, 0xbf8a0000, 0x7da206f9, 0x81060000, 0xbf88002a, 0xd5480014, 0x02151101, + 0x362402ff, 0x000000ff, 0x160028a0, 0xd5430015, 0x040224ff, 0x00000060, 0xdbfc0000, 0x02000015, + 0x162c2890, 0xd5430016, 0x045a24ff, 0x00000030, 0xbf8cc07f, 0xe0781000, 0x02020216, 0x7da82881, + 0xbf880017, 0xbe94047e, 0xb06a0030, 0x162602f9, 0x0086066a, 0xe0301028, 0x02020613, 0xe0341010, + 0x02020713, 0xbf8c3f70, 0x7e300306, 0x7e320306, 0x7e340306, 0x7e360306, 0x34142484, 0xbefe0414, + 0xe0781000, 0x0403180a, 0x4a162481, 0x16161690, 0x4c1616ff, 0x00008000, 0xe0741000, 0x0202070b, + 0xbf810000, +}; + alignas(256) constexpr std::array PixelCode{ 0xc8020002, 0xc8060102, 0xc80a0202, 0xc80e0302, 0xf800180f, 0x03020100, 0xbf810000, }; @@ -490,6 +505,16 @@ constexpr std::uint16_t Corner(std::uint32_t patch, std::uint32_t x, std::uint32 return static_cast((PatchLevels[patch] << 8u) | (patch << 4u) | (x << 2u) | y); } +constexpr std::uint16_t LevelCorner(std::uint32_t level, std::uint32_t patch, std::uint32_t x, std::uint32_t y) { + return static_cast((level << 8u) | (patch << 4u) | (x << 2u) | y); +} + +alignas(256) constexpr std::array LastPointLevelIndices{ + LevelCorner(5u, 0u, 0u, 0u), LevelCorner(5u, 0u, 2u, 0u), Corner(0, 1, 2), + LevelCorner(1u, 1u, 0u, 0u), LevelCorner(1u, 1u, 2u, 0u), Corner(1, 1, 2), + LevelCorner(0u, 2u, 0u, 0u), LevelCorner(0u, 2u, 2u, 0u), Corner(2, 1, 2), +}; + alignas(256) constexpr std::array Indices{ Corner(0, 0, 0), Corner(0, 2, 0), Corner(0, 1, 2), Corner(1, 0, 0), Corner(1, 2, 0), Corner(1, 1, 2), @@ -531,6 +556,7 @@ alignas(256) constexpr std::array NextInputIndices{ const PatchDraw Triangles{Patches, Indices}; const PatchDraw NextInputTriangles{{4u, 3u, 1u, 2u, 2u}, NextInputIndices}; const PatchDraw MergedTriangles{MergedPatches, Indices}; +const PatchDraw MergedLastPointLevels{MergedPatches, LastPointLevelIndices}; struct Stages { ShaderRecompiler::RecompileResult local; @@ -1026,6 +1052,32 @@ void RequireMergedHullConstantIndices(const ShaderRecompiler::SpirvTarget& targe } } +void RequireBarrierBeforeRingReadBack(const ShaderRecompiler::SpirvTarget& target) { + std::uint32_t cursor = 0; + const auto hull = Compile(target, ShaderStage::TessellationControl, MergedHullRingReadBack, cursor, MergedLastPointLevels); + const auto words = hull.spirv.Words(); + std::vector outputs; + std::vector barriers; + std::vector stores; + std::size_t lastLoad = 0; + for (std::size_t at = 5; at < words.size(); at += words[at] >> 16u) { + const auto count = words[at] >> 16u; + Require(count != 0u && at + count <= words.size(), "the merged hull program compiled to a malformed SPIR-V instruction"); + const auto opcode = words[at] & 0xffffu; + const auto output = [&](std::uint32_t id) { return std::find(outputs.begin(), outputs.end(), id) != outputs.end(); }; + if (opcode == spv::OpVariable && words[at + 3u] == spv::StorageClassOutput) outputs.push_back(words[at + 2u]); + if (opcode == spv::OpAccessChain && output(words[at + 3u])) outputs.push_back(words[at + 2u]); + if (opcode == spv::OpStore && output(words[at + 1u])) stores.push_back(at); + if (opcode == spv::OpLoad && output(words[at + 3u])) lastLoad = at; + if (opcode == spv::OpControlBarrier) barriers.push_back(at); + } + const auto before = std::lower_bound(barriers.begin(), barriers.end(), lastLoad); + Require(lastLoad != 0u && before - barriers.begin() >= 2, "the merged hull program reading control point 2's output back has no barrier besides its s_barrier before the read"); + const auto first = *(before - 2); + const auto second = *(before - 1); + Require(std::any_of(stores.begin(), stores.end(), [&](std::size_t store) { return store > first && store < second; }), "the merged hull program reading control point 2's output back waits at no barrier after its control-point writes"); +} + void RequireRegisteredMergedStages() { const auto low = [](const auto& code) { return static_cast(reinterpret_cast(code.data()) >> 8u); }; const auto high = [](const auto& code) { return static_cast(reinterpret_cast(code.data()) >> 40u); }; @@ -1095,6 +1147,8 @@ int main() { RequirePatches(*device, target, MergedHullRegisters, "merged hull program storing its local part's registers", MergedTriangles, MergedHullRegisters); RequirePatches(*device, target, MergedHullHalfWords, "merged hull program passing a value through 16-bit LDS writes and reads", MergedTriangles, MergedHullHalfWords); RequirePatches(*device, target, MergedHullHalfRecord, "merged hull program writing its own record's position in 16-bit halves", MergedTriangles, MergedHullHalfRecord); + RequireBarrierBeforeRingReadBack(target); + RequirePatches(*device, target, MergedHullRingReadBack, "merged hull program reading control point 2's level back from the off-chip ring", MergedLastPointLevels, MergedHullRingReadBack); RequireCounterClockwise(*device, target); RequireLevelClamps(*device, target); RequireIntegerPartitioning(*device, target); diff --git a/core/shader/recompiler/Optimization/src/TessellationLowering.cpp b/core/shader/recompiler/Optimization/src/TessellationLowering.cpp index 6c3271b070a..7a6a4bec4fe 100644 --- a/core/shader/recompiler/Optimization/src/TessellationLowering.cpp +++ b/core/shader/recompiler/Optimization/src/TessellationLowering.cpp @@ -650,6 +650,83 @@ bool FirstInvocationOnly(IrValue* raw, AddressEvaluator& evaluator, std::unorder return result; } +std::optional InvocationValue(const IrValue* raw, std::uint32_t invocation, std::unordered_map>& memo) { + const IrValue* value = raw->Resolve(); + if (value->HasImmediate()) { + if (value->Type() == IrType::Bool) return value->ImmediateBool() ? 1u : 0u; + if (value->Type() == IrType::U32) return value->ImmediateU32(); + return std::nullopt; + } + if (const auto found = memo.find(value); found != memo.end()) return found->second; + memo[value] = std::nullopt; + const auto argument = [&](std::size_t index) { return InvocationValue(value->Argument(index), invocation, memo); }; + const auto binary = [&](auto operation) -> std::optional { + const auto lhs = argument(0); + const auto rhs = argument(1); + if (!lhs || !rhs) return std::nullopt; + return operation(*lhs, *rhs); + }; + const auto result = [&]() -> std::optional { + switch (value->Opcode()) { + case IrOpcode::LaneId: + return invocation; + case IrOpcode::GetBuiltin: + if (IsImmediate(value->Argument(0), static_cast(StageInputKind::InvocationId))) return invocation; + return std::nullopt; + case IrOpcode::LogicalNot: { + const auto operand = argument(0); + if (!operand) return std::nullopt; + return *operand == 0u ? 1u : 0u; + } + case IrOpcode::BitCount32: { + const auto operand = argument(0); + if (!operand) return std::nullopt; + return static_cast(std::popcount(*operand)); + } + case IrOpcode::SelectU32: + case IrOpcode::SelectU1: { + const auto condition = argument(0); + if (!condition) return std::nullopt; + return argument(*condition != 0u ? 1u : 2u); + } + case IrOpcode::BitwiseAnd32: + case IrOpcode::LogicalAnd: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs & rhs; }); + case IrOpcode::BitwiseOr32: + case IrOpcode::LogicalOr: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs | rhs; }); + case IrOpcode::BitwiseXor32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs ^ rhs; }); + case IrOpcode::ShiftLeftLogical32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) -> std::optional { return rhs < 32u ? std::optional(lhs << rhs) : std::nullopt; }); + case IrOpcode::ShiftRightLogical32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) -> std::optional { return rhs < 32u ? std::optional(lhs >> rhs) : std::nullopt; }); + case IrOpcode::IAdd32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs + rhs; }); + case IrOpcode::ISub32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs - rhs; }); + case IrOpcode::IMul32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs * rhs; }); + case IrOpcode::ULessThan32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs < rhs ? 1u : 0u; }); + case IrOpcode::ULessThanEqual32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs <= rhs ? 1u : 0u; }); + case IrOpcode::UGreaterThan32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs > rhs ? 1u : 0u; }); + case IrOpcode::UGreaterThanEqual32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs >= rhs ? 1u : 0u; }); + case IrOpcode::IEqual32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs == rhs ? 1u : 0u; }); + case IrOpcode::INotEqual32: + return binary([](std::uint32_t lhs, std::uint32_t rhs) { return lhs != rhs ? 1u : 0u; }); + default: + return std::nullopt; + } + }(); + memo[value] = result; + return result; +} + struct Access { IrBlock* block = nullptr; IrValue* inst = nullptr; @@ -707,6 +784,7 @@ class Lowering { if (stage == IrShaderStage::TessellationControl || capture) lowerOwnLaneBits(); collect(); derive(); + orderControlOutputReads(); for (const auto& access : accesses) rewrite(access); if (stage == IrShaderStage::TessellationEvaluation) removePrimitiveExports(); constexpr DeadCodeEliminator deadCodeEliminator; @@ -789,6 +867,8 @@ class Lowering { const bool factor = IsBase(inst->Argument(3), TessellationBaseKind::Factor); if (stage == IrShaderStage::TessellationControl && access.write) { access.kind = factor ? TessellationAttribute::Factor : TessellationAttribute::ControlOutput; + } else if (stage == IrShaderStage::TessellationControl && !factor) { + access.kind = TessellationAttribute::ControlOutput; } else if (stage == IrShaderStage::TessellationEvaluation && !access.write && !factor) { access.kind = TessellationAttribute::EvaluationInput; } else { @@ -826,8 +906,8 @@ class Lowering { if (access.kind == TessellationAttribute::LocalOutput) agree(localStride, access.address.relative, access.pc); if (access.kind == TessellationAttribute::ControlInput && access.address.vertex != 0) agree(localStride, access.address.vertex, access.pc); if (access.kind == TessellationAttribute::ControlInput && (access.address.dynamic || access.address.vertex == 0) && access.address.relative > 0 && access.address.relative % inputs == 0) agree(localStride, access.address.relative / inputs, access.pc); - if (access.kind == TessellationAttribute::ControlOutput && access.address.vertex != 0) agree(controlStride, access.address.vertex, access.pc); - if (access.kind == TessellationAttribute::ControlOutput && soleControlPoint(access)) agree(controlStride, access.address.relative, access.pc); + if (access.kind == TessellationAttribute::ControlOutput && access.write && access.address.vertex != 0) agree(controlStride, access.address.vertex, access.pc); + if (access.kind == TessellationAttribute::ControlOutput && access.write && soleControlPoint(access)) agree(controlStride, access.address.relative, access.pc); if (access.kind == TessellationAttribute::ControlShared) { if (access.address.relative <= 0 || access.address.relative % inputs != 0) Fail("the merged hull program accesses LDS outside its patch", access.pc); agree(sharedStride, access.address.relative / inputs, access.pc); @@ -865,6 +945,11 @@ class Lowering { if (address.dynamic && (address.dynamicScale == 0 || address.dynamicScale % localStride != 0)) Fail("the hull shader reads LDS at a computed offset that is not a whole number of control points", access.pc); break; case TessellationAttribute::ControlOutput: + if (!access.write) { + if (controlStride == 0u) Fail("the hull shader reads off-chip memory but writes no control point there, so the control point stride is unknown", access.pc); + if ((address.vertex != 0 && address.vertex != controlStride) || address.relative != outputs * controlStride || address.constant < 0 || end + address.vertex * (outputs - 1) > outputs * controlStride) Fail("the hull shader reads off-chip memory outside its patch's control points", access.pc); + break; + } if (address.vertex == 0 && !soleControlPoint(access)) { if (controlStride == 0u || evaluator.Lowest(address) < static_cast(tessellation.patchesPerGroup) * outputs * controlStride) Fail("the hull shader writes off-chip memory outside its control points and patch constants", access.pc); record(access, end); @@ -1092,6 +1177,111 @@ class Lowering { } } + using Edges = std::unordered_map>; + + std::optional uniformCondition(const IrValue* condition) const { + std::optional result; + for (std::uint32_t invocation = 0; invocation < tessellation.outputControlPoints; ++invocation) { + std::unordered_map> memo; + const auto value = InvocationValue(condition, invocation, memo); + if (!value || (result && *result != (*value != 0u))) return std::nullopt; + result = *value != 0u; + } + return result; + } + + Edges takenEdges() const { + const auto& order = program.BlockOrder(); + const auto& blockInfo = program.Metadata().blockInfo; + std::unordered_map byId; + if (order.size() == blockInfo.size()) { + for (std::size_t index = 0; index < order.size(); ++index) byId.emplace(blockInfo[index].id, order[index]); + } + Edges edges; + for (std::size_t index = 0; index < order.size(); ++index) { + auto& successors = edges[order[index]]; + successors.assign(order[index]->Successors().begin(), order[index]->Successors().end()); + if (byId.empty()) continue; + const auto& info = blockInfo[index]; + if (info.terminator.kind != TerminatorKind::ConditionalBranch || info.condition == nullptr || info.condition->Resolve()->Type() != IrType::U1) continue; + const auto uniform = uniformCondition(info.condition); + if (!uniform) continue; + const auto taken = byId.find(*uniform ? info.terminator.trueBlock : info.terminator.falseBlock); + if (taken == byId.end() || std::find(successors.begin(), successors.end(), taken->second) == successors.end()) continue; + successors.assign(1u, taken->second); + } + return edges; + } + + static std::unordered_set reachedFrom(const Edges& edges, std::vector pending, const IrBlock* avoided) { + std::unordered_set seen; + while (!pending.empty()) { + IrBlock* current = pending.back(); + pending.pop_back(); + if (current == avoided || !seen.insert(current).second) continue; + const auto& next = edges.at(current); + pending.insert(pending.end(), next.begin(), next.end()); + } + return seen; + } + + void orderControlOutputReads() { + if (tessellation.outputControlPoints == 1u) return; + std::unordered_set reads; + std::unordered_set readBlocks; + std::uint32_t readPc = 0; + for (const auto& access : accesses) { + if (access.kind != TessellationAttribute::ControlOutput || access.write || access.address.vertex != 0) continue; + if (reads.empty()) readPc = access.pc; + reads.insert(access.inst); + readBlocks.insert(access.block); + } + if (reads.empty()) return; + const auto edges = takenEdges(); + IrBlock* entry = program.BlockOrder().front(); + const auto live = reachedFrom(edges, {entry}, nullptr); + if (std::any_of(readBlocks.begin(), readBlocks.end(), [&](const IrBlock* block) { return !live.contains(block); })) Fail("the hull shader reads another control point's off-chip output in a block no control point reaches", readPc); + IrBlock* barrier = nullptr; + for (IrBlock* candidate : program.BlockOrder()) { + if (!live.contains(candidate)) continue; + const auto before = candidate == entry ? std::unordered_set{} : reachedFrom(edges, {entry}, candidate); + const bool everyPath = std::none_of(before.begin(), before.end(), [&](const IrBlock* block) { return edges.at(block).empty(); }); + const bool dominatesReads = std::none_of(readBlocks.begin(), readBlocks.end(), [&](const IrBlock* block) { return before.contains(block); }); + if (!everyPath || !dominatesReads || reachedFrom(edges, edges.at(candidate), nullptr).contains(candidate)) continue; + if (barrier == nullptr || reachedFrom(edges, edges.at(barrier), nullptr).contains(candidate)) barrier = candidate; + } + if (barrier == nullptr) Fail("the hull shader reads another control point's off-chip output, but no block before the read is reached once by every invocation, so no barrier can order it after the writes", readPc); + const auto after = reachedFrom(edges, edges.at(barrier), nullptr); + IrValue* firstRead = nullptr; + for (IrValue* inst : barrier->Instructions()) { + if (reads.contains(inst)) { + firstRead = inst; + break; + } + } + for (const auto& access : accesses) { + if (access.kind != TessellationAttribute::ControlOutput || !access.write || !live.contains(access.block)) continue; + bool late = after.contains(access.block); + if (access.block == barrier && firstRead != nullptr) { + const auto& instructions = barrier->Instructions(); + late = std::find(std::find(instructions.begin(), instructions.end(), firstRead), instructions.end(), access.inst) != instructions.end(); + } + if (late) Fail("the hull shader reads another control point's off-chip output, but one of its writes there comes after the last block every invocation reaches before the read", access.pc); + } + IrValue* position = firstRead; + if (position == nullptr) { + const auto& instructions = barrier->Instructions(); + const auto branch = std::find_if(instructions.begin(), instructions.end(), [](const IrValue* inst) { return inst->Opcode() == IrOpcode::Branch || inst->Opcode() == IrOpcode::BranchConditional; }); + if (branch != instructions.end()) position = *branch; + } + IrValue& value = program.CreateValue(IrOpcode::Barrier, IrType::Void); + if (position != nullptr) { + barrier->InsertInstructionBefore(position, &value); + } else { + barrier->AppendInstruction(&value); + } + } + void requireOwnLanes() const { for (IrBlock* block : program.BlockOrder()) { for (const IrValue* inst : block->Instructions()) { @@ -1190,7 +1380,7 @@ class Lowering { auto address = access.address.constant + 4 * static_cast(component); if (access.kind == TessellationAttribute::PatchOutput || access.kind == TessellationAttribute::PatchInput) address -= FloorDiv(address, patchStride) * patchStride; IrValue& offset = constant(static_cast(address)); - const bool indexed = access.kind == TessellationAttribute::ControlInput || (access.kind == TessellationAttribute::ControlShared && !access.write); + const bool indexed = access.kind == TessellationAttribute::ControlInput || ((access.kind == TessellationAttribute::ControlShared || access.kind == TessellationAttribute::ControlOutput) && !access.write); if (!indexed || access.address.vertex == 0) return offset; IrValue& invocation = insert(access, IrOpcode::GetBuiltin, IrType::U32, {&constant(static_cast(StageInputKind::InvocationId)), &constant(0u)}); IrValue& scaled = insert(access, IrOpcode::IMul32, IrType::U32, {&invocation, &constant(static_cast(access.address.vertex))}); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index dbee6f90ecb..8bbe4a18af1 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -908,8 +908,9 @@ std::uint32_t TessellationPointer(SpirvValueEmitContext& ctx, const IrValue& ins const bool local = kind == TessellationAttribute::LocalOutput || kind == TessellationAttribute::ControlInput; const auto stride = local ? state.program.Metadata().tessellationLocalStride : kind == TessellationAttribute::ControlShared ? state.program.Metadata().tessellationSharedStride : state.program.Metadata().tessellationControlStride; const auto type = TypePointer(state, storage, TypeU32(state)); + const bool controlRead = kind == TessellationAttribute::ControlOutput && inst.Opcode() == IrOpcode::GetTessellationAttribute; const auto inPatch = [&](std::uint32_t vertex) { - if (kind != TessellationAttribute::ControlInput && kind != TessellationAttribute::EvaluationInput) return vertex; + if (kind != TessellationAttribute::ControlInput && kind != TessellationAttribute::EvaluationInput && !controlRead) return vertex; const auto& tess = state.inputInfo.vertex->tess; const auto clamped = state.module.AllocateId(); state.module.AddFunction(spv::OpExtInst, TypeU32(state), clamped, GlslStd450(state), GLSLstd450UMin, vertex, ConstantU32(state, (kind == TessellationAttribute::ControlInput ? tess.inputControlPoints : tess.outputControlPoints) - 1u)); @@ -918,7 +919,7 @@ std::uint32_t TessellationPointer(SpirvValueEmitContext& ctx, const IrValue& ins const bool sharedRead = kind == TessellationAttribute::ControlShared && inst.Opcode() == IrOpcode::GetTessellationAttribute; const auto* sum = inst.Argument(1); const auto* scaled = sum->Opcode() == IrOpcode::IAdd32 ? sum->Argument(0) : nullptr; - if ((kind == TessellationAttribute::ControlInput || sharedRead) && scaled != nullptr && sum->Argument(1)->HasImmediate() && scaled->Opcode() == IrOpcode::IMul32 && scaled->Argument(1)->HasImmediate() && scaled->Argument(1)->ImmediateU32() == stride) { + if ((kind == TessellationAttribute::ControlInput || sharedRead || controlRead) && scaled != nullptr && sum->Argument(1)->HasImmediate() && scaled->Opcode() == IrOpcode::IMul32 && scaled->Argument(1)->HasImmediate() && scaled->Argument(1)->ImmediateU32() == stride) { const auto constant = sum->Argument(1)->ImmediateU32(); const auto vertex = inPatch(EmitBinaryU32(state, spv::OpIAdd, ctx.Arg(*scaled, 0), ConstantU32(state, constant / stride))); state.module.AddFunction(spv::OpAccessChain, type, pointer, variable, vertex, ConstantU32(state, constant % stride / 16u), ConstantU32(state, constant % stride % 16u / 4u)); @@ -933,8 +934,8 @@ std::uint32_t TessellationPointer(SpirvValueEmitContext& ctx, const IrValue& ins state.module.AddFunction(spv::OpAccessChain, type, pointer, variable, attribute, component); return pointer; } - const bool own = kind == TessellationAttribute::ControlOutput || (kind == TessellationAttribute::ControlShared && !sharedRead); - const auto controlPoint = kind == TessellationAttribute::ControlInput ? std::min(value / stride, state.inputInfo.vertex->tess.inputControlPoints - 1u) : kind == TessellationAttribute::EvaluationInput ? std::min(value / stride, state.inputInfo.vertex->tess.outputControlPoints - 1u) : value / stride; + const bool own = (kind == TessellationAttribute::ControlOutput && !controlRead) || (kind == TessellationAttribute::ControlShared && !sharedRead); + const auto controlPoint = kind == TessellationAttribute::ControlInput ? std::min(value / stride, state.inputInfo.vertex->tess.inputControlPoints - 1u) : kind == TessellationAttribute::EvaluationInput || controlRead ? std::min(value / stride, state.inputInfo.vertex->tess.outputControlPoints - 1u) : value / stride; state.module.AddFunction(spv::OpAccessChain, type, pointer, variable, own ? EmitLaneId(state) : ConstantU32(state, controlPoint), attribute, component); return pointer; } @@ -944,7 +945,7 @@ std::uint32_t TessellationPointer(SpirvValueEmitContext& ctx, const IrValue& ins if (kind == TessellationAttribute::LocalOutput || patch) { state.module.AddFunction(spv::OpAccessChain, type, pointer, variable, attribute, component); } else { - const bool own = kind == TessellationAttribute::ControlOutput || (kind == TessellationAttribute::ControlShared && !sharedRead); + const bool own = (kind == TessellationAttribute::ControlOutput && !controlRead) || (kind == TessellationAttribute::ControlShared && !sharedRead); const auto vertex = own ? EmitLaneId(state) : inPatch(EmitBinaryU32(state, spv::OpUDiv, address, ConstantU32(state, stride))); state.module.AddFunction(spv::OpAccessChain, type, pointer, variable, vertex, attribute, component); } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 4f5fde8d1d7..91a0dce30b0 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -144,7 +144,7 @@ - [DB_SHADER_CONTROL.PRIMITIVE_ORDERED_PIXEL_SHADER](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - the host pixel interlock spans the whole shader, not only up to `s_sendmsg` ORDERED_PS_DONE. - [POPS collision SGPRs](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - the collision wave ID and intrawave collision mask read 0, so a title skips its POPS wait and runs its per-collision loop once. - [Image loads](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) (shader recompiler) - an image_load reads a sampled binding, also of an image the shader stores to, so in a primitive-ordered pixel shader it misses an earlier overlapping primitive's image_store in the same draw. -- [Tessellation rings](../../../core/shader/recompiler/Optimization/src/TessellationLowering.cpp) of local, hull and domain programs become Vulkan tessellation interfaces and are never written to guest memory; a title that reads the off-chip or factor ring back sees no data. +- [Tessellation rings](../../../core/shader/recompiler/Optimization/src/TessellationLowering.cpp) of local, hull and domain programs become Vulkan tessellation interfaces and are never written to guest memory; a title that reads the off-chip or factor ring back sees no data. A hull program that reads its own patch's control points from the off-chip ring reads `gl_out` instead: before a read of another control point's record the lowering inserts a barrier in the last block every invocation reaches, and a field no control point writes reads an undefined value instead of the ring's earlier contents. - [Hull input control points](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - a hull LDS read at a control-point index computed from data reads `gl_in` at that index clamped to the patch's last input control point, and a domain off-chip read at a computed index reads `gl_in` clamped to the patch's last output control point. On the console an index past the patch reads the LDS slot or off-chip record of a neighbouring patch or of no patch; the host has no such memory, and Vulkan leaves an out-of-range `gl_in` index undefined. The lowering refuses a computed offset that is not a whole number of control points, but the index itself is only known when the program runs. - [Inner tessellation factors](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) of integer, fractional-even or fractional-odd triangle patches that are NaN tessellate as 1 (fractional-even then raises them to 2, as Vulkan does), as in the D3D11 reference tessellator; the console is not measured. - [Hull programs](../../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) start from the hull launch registers (user data, ring bases, v0, v1); on the console they start with what the local program leaves, and that it leaves them unchanged is not checked.