diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 25e74dc6065..10f75f762e2 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -150,6 +150,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawCapture.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawDecode.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawLookup.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawPipeline.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawPrecheck.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawRegisterKey.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawTiming.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp new file mode 100644 index 00000000000..8f290e4caae --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp @@ -0,0 +1,74 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAW_DRAWPIPELINE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAW_DRAWPIPELINE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +class DrawPipeline { +public: + using Commit = std::function; + using Range = std::pair; + enum class DrainReason : std::uint8_t { Packet, Flush, Labels, Capture, Indirect, Submission, Count }; + + static DrawPipeline& Queue0(); + static std::size_t Depth(); + static bool& Active(); + static std::atomic& EpochToken(); + static void FollowEpoch(std::uint64_t token); + + void Enqueue(Commit commit, std::vector writes, std::uint64_t labelAddress = 0, std::vector labelBytes = {}); + void Drain(DrainReason reason, std::uint32_t opcode = 0x100); + bool Busy() const { return outstanding.load(std::memory_order_acquire) != 0; } + bool Overlaps(std::uint64_t address, std::size_t bytes); + std::optional PendingLabel(std::uint64_t address, std::size_t bytes); + +private: + struct Item { + Commit commit; + std::vector writes; + std::uint64_t labelAddress = 0; + std::vector labelBytes; + }; + DrawPipeline() = default; + void run(); + void rethrowFailure(); + void report(std::chrono::steady_clock::time_point now); + + std::mutex mutex; + std::condition_variable wake; + std::condition_variable idle; + std::deque items; + std::exception_ptr failure; + std::atomic outstanding{0}; + std::uint32_t idleWaiters = 0; + bool committerWaiting = false; + std::thread thread; + std::uint64_t commits = 0; + std::uint64_t commitNs = 0; + std::uint64_t commitErrors = 0; + std::uint64_t enqueued = 0; + std::uint64_t fullWaitNs = 0; + std::array(DrainReason::Count)> drains{}; + std::array(DrainReason::Count)> drainWaitNs{}; + std::array drainOpcodes{}; + std::uint64_t depthSum = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 7da8e8022b3..f01ee64f431 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -97,6 +97,11 @@ class Driver { void enqueue(Submission submission); void noteHeldAtSubmit(Submission& submission, std::size_t cursor); static void forgetUnfinishedWrites(QueueWorker& worker, const Submission& submission); + static void forgetUnfinishedWrites(QueueWorker& worker, std::span labelWrites); + static bool& completionDeferred(); + void deferCompletion(const Submission& submission); + void submitOpenWork(); + void completeSubmission(std::uint64_t serial, std::uint64_t received, std::span labelWrites); static bool waitFree(const Submission& submission); bool queue0Before(std::uint64_t received) const; bool orderReleased(std::uint32_t queue, std::uint64_t received) const; @@ -207,6 +212,13 @@ class Driver { static void appendWrittenRanges(const ShaderRecompiler::RecompileResult& compiled, std::vector>& ranges); void noteForeignWriter(std::uint64_t begin, std::uint64_t end, std::uint32_t queue); void noteDrawWriters(std::span stages, std::uint32_t queue); + static std::vector> drawWriteRanges(const Graphics::State& graphics, std::span stages); + void commitDraw(std::shared_ptr localDevice, std::uint32_t queue, const Graphics::State& graphics, const Pm4::DrawParameters& drawParameters, std::span stages, std::span snapshots, const std::shared_ptr& recipe, const std::vector>& recipeStages, std::uint64_t drawKey); + bool enqueueLabelPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue); + bool enqueueDmaPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state); + void commitDmaStore(std::uint32_t queue, std::uint64_t address, std::span bytes); + void commitLabel(std::uint32_t queue, std::uint64_t address, std::span bytes, bool endOfPipeInterrupt); + void submitDueAfterCommit(VulkanDevice* localDevice); std::optional newestWriterLocked(std::uint64_t begin, std::uint64_t end) const; std::optional newestWriter(std::uint64_t begin, std::uint64_t end); std::string describeWriters(std::uint64_t begin, std::uint64_t end); @@ -264,6 +276,9 @@ class Driver { static Graphics::Recorder::LateStatistics& lateCountsSeen(); static EpochBumps& epochBumps(); static void bumpEpoch(std::uint64_t EpochBumps::*counter); + static std::chrono::steady_clock::time_point& lastEpochBump(); + static std::deque& releasedTails(); + void noteReleasedTails(const Submission& submission); static bool packetEpoch(); static bool labelTryEachPacket(); static std::chrono::microseconds labelFlushDeadline(); @@ -372,6 +387,7 @@ class Driver { std::uint32_t idleWaiters = 0; std::uint64_t queue0Executing = 0; + std::deque queue0Uncommitted; std::atomic orderHolders{0}; std::atomic runningWorkers{0}; std::atomic idleEpoch{0}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp new file mode 100644 index 00000000000..b5e1b033103 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp @@ -0,0 +1,18 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAWWRITERANGES_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAWWRITERANGES_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +std::vector> DrawWriteRanges(const Graphics::State& graphics, std::span stages, bool exact); +bool ExactDrawWrites(); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index c00249bc0c0..7002f98de94 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferCopy.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" @@ -87,6 +88,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, pendingDispatchPhases().outcome = DispatchOutcome::FillHle; return; } + if (DrawPipeline::Active()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15); if (!partialLastGroup && indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) { pendingDispatchPhases().outcome = DispatchOutcome::CopyHle; return; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index 881c040c617..1adbd297bf4 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -1,5 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" @@ -30,6 +32,8 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe return DrawVerdict::Drawn; }; auto drawParameters = Pm4::ResolveDraw(packet, queue); + const bool pipelined = DrawPipeline::Active(); + if (pipelined && drawParameters.indirect) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Indirect); bool traceIndirect = false; if (const auto verdict = precheckDraw(queue, submission, packet, drawParameters, rejected, traceIndirect)) return *verdict; phaseTiming.Phase(DrawRowPrecheck); @@ -355,6 +359,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe for (const auto& read : reads) memory.push_back({read.address, std::as_bytes(std::span(read.bytes))}); } + if (pipelined && !deferredLabels().labels.empty()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Labels); if (recordQueuedLabelsAfterCapture(submission.queue, memory)) return draw(queue, packet, submission, rejected); bool rectListBuilt = false; @@ -503,6 +508,23 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe return DrawVerdict::Rejected; } } + if (pipelined && !drawParameters.indirect && deferredLabels().labels.empty()) { + phaseTiming.Phase(DrawRowVectors); + auto writes = drawWriteRanges(graphics, stages); + const auto commitQueue = submission.queue; + const auto commitPacket = (packet[0] >> 8u) & 0xffu; + const auto epoch = DrawPipeline::EpochToken().load(std::memory_order_relaxed); + auto commit = [this, localDevice = std::move(localDevice), decode = std::move(decode), drawParameters, stages = std::move(stages), snapshots = std::move(snapshots), recipe = std::move(recipe), recipeStages = std::move(recipeStages), drawKey, commitQueue, commitPacket, epoch, programs = std::move(programs), results = std::move(results), memory = std::move(memory), linked = std::move(linked), stageCaptures = std::move(stageCaptures), matched = std::move(matched), matchedRegions = std::move(matchedRegions), fresh = std::move(fresh), decodeReads = std::move(decodeReads), shaderMemory = std::move(shaderMemory), entry = std::move(entry), dataEntry = std::move(dataEntry)] { + DrawPipeline::FollowEpoch(epoch); + GuestMemory::SetCurrentPacket(commitPacket, commitQueue); + commitDraw(localDevice, commitQueue, decode->state, drawParameters, stages, snapshots, recipe, recipeStages, drawKey); + }; + auto held = std::make_shared(std::move(commit)); + DrawPipeline::Queue0().Enqueue([held] { (*held)(); }, std::move(writes)); + phaseTiming.Phase(DrawRowLockWait); + timing.Mark("draw_enqueued"); + return drawn(); + } lockForDraw(); noteDrawWriters(stages, submission.queue); phaseTiming.Phase(DrawRowVectors); @@ -523,4 +545,26 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe return drawn(); } +void Driver::commitDraw(std::shared_ptr localDevice, std::uint32_t queue, const Graphics::State& graphics, const Pm4::DrawParameters& drawParameters, std::span stages, std::span snapshots, const std::shared_ptr& recipe, const std::vector>& recipeStages, std::uint64_t drawKey) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (auto current = device.Load(); current != nullptr && current != localDevice) localDevice = std::move(current); + struct SubmitDue { + Driver& driver; + const std::shared_ptr& device; + ~SubmitDue() noexcept(false) { + if (std::uncaught_exceptions() == 0) driver.submitDueAfterCommit(device.get()); + } + } submitDue{*this, localDevice}; + recordLabelsForPacket(localDevice.get(), queue); + noteDrawWriters(stages, queue); + if (recipe != nullptr) { + if (localDevice->DrawFromRecipe(graphics, drawParameters, stages, snapshots, recipe) == RecipeOutcome::Recorded) return; + VulkanDevice::NoteRecipe(VulkanDevice::RecipeEvent::Restart, VulkanDevice::RecipeKind::Draw); + } + std::shared_ptr built; + localDevice->Draw(graphics, drawParameters, stages, snapshots, recipeStages.empty() ? nullptr : &built); + if (built != nullptr) attachDrawRecipe(drawKey, recipeStages, std::move(built)); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp new file mode 100644 index 00000000000..049b061f9d9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp @@ -0,0 +1,185 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +namespace { + +constexpr std::array(DrawPipeline::DrainReason::Count)> DrainNames{"packet", "flush", "labels", "capture", "indirect", "submission"}; + +std::uint64_t elapsedNs(std::chrono::steady_clock::time_point since) { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - since).count()); +} + +} + +DrawPipeline& DrawPipeline::Queue0() { + static DrawPipeline* pipeline = new DrawPipeline(); + return *pipeline; +} + +std::size_t DrawPipeline::Depth() { + static const std::size_t depth = [] { + const char* text = std::getenv("APS5_PIPELINED_DRAWS"); + if (text == nullptr || std::getenv("APS5_LOCKED_DRAW_PREPARE") != nullptr || std::getenv("APS5_DRAW_DRAIN") != nullptr || std::getenv("APS5_DRAIN_ALL") != nullptr || std::getenv("APS5_NO_WORDWISE_CAPTURE") != nullptr) return std::size_t{0}; + const auto value = std::strtoull(text, nullptr, 10); + return value == 0 ? std::size_t{0} : value == 1 ? std::size_t{8} : static_cast(std::min(value, 256)); + }(); + return depth; +} + +bool& DrawPipeline::Active() { + static thread_local bool active = false; + return active; +} + +std::atomic& DrawPipeline::EpochToken() { + static std::atomic token{1}; + return token; +} + +void DrawPipeline::FollowEpoch(std::uint64_t token) { + static thread_local std::uint64_t followed = 0; + if (followed == token) return; + followed = token; + GuestMemory::BumpCollectEpoch(); +} + +void DrawPipeline::rethrowFailure() { + if (failure == nullptr) return; + auto error = std::exchange(failure, nullptr); + std::rethrow_exception(error); +} + +void DrawPipeline::Enqueue(Commit commit, std::vector writes, std::uint64_t labelAddress, std::vector labelBytes) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + std::unique_lock lock(mutex); + rethrowFailure(); + if (items.size() >= Depth()) { + const auto start = std::chrono::steady_clock::now(); + ++idleWaiters; + idle.wait(lock, [&] { return items.size() < Depth(); }); + --idleWaiters; + fullWaitNs += elapsedNs(start); + rethrowFailure(); + } + depthSum += items.size(); + ++enqueued; + items.push_back({std::move(commit), std::move(writes), labelAddress, std::move(labelBytes)}); + outstanding.fetch_add(1, std::memory_order_release); + if (!thread.joinable()) thread = std::thread([this] { run(); }); + if (committerWaiting) wake.notify_one(); + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport >= std::chrono::seconds(10)) report(now); +} + +void DrawPipeline::report(std::chrono::steady_clock::time_point now) { + lastReport = now; + std::string drainText; + for (std::size_t i = 0; i < drains.size(); ++i) { + char item[64]; + std::snprintf(item, sizeof(item), " %s %llu (%.1f ms)", DrainNames[i], static_cast(drains[i]), static_cast(drainWaitNs[i]) / 1e6); + drainText += item; + } + drainText += "; packet drains by opcode:"; + for (int shown = 0; shown < 5; ++shown) { + const auto top = std::max_element(drainOpcodes.begin(), drainOpcodes.begin() + 0x100); + if (*top == 0) break; + char item[32]; + std::snprintf(item, sizeof(item), " 0x%02x x%llu", static_cast(top - drainOpcodes.begin()), static_cast(*top)); + drainText += item; + *top = 0; + } + drainOpcodes = {}; + const auto perCommit = commits == 0 ? 0.0 : static_cast(commitNs) / 1e3 / static_cast(commits); + const auto perDraw = enqueued == 0 ? 0.0 : static_cast(fullWaitNs) / 1e3 / static_cast(enqueued); + const auto queued = enqueued == 0 ? 0.0 : static_cast(depthSum) / static_cast(enqueued); + std::fprintf(stderr, "[draw] pipelined (10 s, depth %zu): %llu commits at %.1f us each on the committer, %llu failed; %llu enqueued behind %.2f on average, waiting %.1f us per draw for room; drains:%s\n", Depth(), static_cast(commits), perCommit, static_cast(commitErrors), static_cast(enqueued), queued, perDraw, drainText.c_str()); + commits = commitNs = commitErrors = enqueued = fullWaitNs = depthSum = 0; + drains = {}; + drainWaitNs = {}; +} + +void DrawPipeline::run() { + for (;;) { + Commit* commit = nullptr; + { + std::unique_lock lock(mutex); + committerWaiting = true; + wake.wait(lock, [&] { return !items.empty(); }); + committerWaiting = false; + commit = &items.front().commit; + } + const auto start = std::chrono::steady_clock::now(); + std::exception_ptr fatal; + bool failed = false; + try { + (*commit)(); + } catch (const std::exception& error) { + failed = true; + static std::atomic reported{0}; + if (reported.fetch_add(1, std::memory_order_relaxed) < 20) std::fprintf(stderr, "[draw] pipelined commit failed: %.200s\n", error.what()); + } catch (...) { + fatal = std::current_exception(); + } + const auto spent = elapsedNs(start); + std::lock_guard lock(mutex); + items.pop_front(); + ++commits; + commitNs += spent; + if (failed) ++commitErrors; + if (fatal != nullptr && failure == nullptr) failure = fatal; + outstanding.fetch_sub(1, std::memory_order_release); + if (idleWaiters != 0) idle.notify_all(); + } +} + +void DrawPipeline::Drain(DrainReason reason, std::uint32_t opcode) { + if (!Busy()) return; + if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline drain under the GPU lock (reason " + std::to_string(static_cast(reason)) + ", opcode " + std::to_string(opcode) + ")"); + const auto start = std::chrono::steady_clock::now(); + std::unique_lock lock(mutex); + ++idleWaiters; + idle.wait(lock, [&] { return items.empty(); }); + --idleWaiters; + const auto index = static_cast(reason); + ++drains[index]; + ++drainOpcodes[std::min(opcode, 0x100)]; + drainWaitNs[index] += elapsedNs(start); + rethrowFailure(); +} + +std::optional DrawPipeline::PendingLabel(std::uint64_t address, std::size_t bytes) { + if (bytes == 0 || bytes > 8) return std::nullopt; + const auto end = address + bytes; + std::lock_guard lock(mutex); + for (auto item = items.rbegin(); item != items.rend(); ++item) { + const bool overlaps = std::any_of(item->writes.begin(), item->writes.end(), [&](const Range& range) { return address < range.second && range.first < end; }); + if (!overlaps) continue; + if (item->labelBytes.empty() || address < item->labelAddress || end > item->labelAddress + item->labelBytes.size()) return std::nullopt; + std::uint64_t value = 0; + std::memcpy(&value, item->labelBytes.data() + (address - item->labelAddress), bytes); + return value; + } + return std::nullopt; +} + +bool DrawPipeline::Overlaps(std::uint64_t address, std::size_t bytes) { + const auto end = address + bytes; + std::lock_guard lock(mutex); + for (const auto& item : items) { + for (const auto& [begin, limit] : item.writes) { + if (address < limit && begin < end) return true; + } + } + return false; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp index 283bdbb2e71..a8a9f85ef43 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp @@ -3,6 +3,7 @@ #include "ThreadOwned.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" @@ -273,16 +274,17 @@ void Driver::storeBufferFill(std::uint32_t queueId, std::uint64_t base, std::siz ++fills; filledBytes += bytes; if (bytes == 0) return; - GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); - std::lock_guard gpuLock(GuestMemory::GpuMutex()); - const auto holdStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; - auto phaseStart = holdStart; - const auto phase = [&](FillPhase which) { - if (!profile) return; - const auto now = std::chrono::steady_clock::now(); - fillPhaseMs[which] += std::chrono::duration(now - phaseStart).count(); - phaseStart = now; - }; + const auto store = [this, queueId, base, bytes, pattern, localDevice, memoryValueFill, noteMetadata] { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto holdStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + auto phaseStart = holdStart; + const auto phase = [&](FillPhase which) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + fillPhaseMs[which] += std::chrono::duration(now - phaseStart).count(); + phaseStart = now; + }; recordLabelsForPacket(localDevice.get(), queueId); phase(FillLabels); @@ -380,5 +382,15 @@ void Driver::storeBufferFill(std::uint32_t queueId, std::uint64_t base, std::siz std::fprintf(stderr, "[fill] %llu buffer fills (%.0f MiB), %llu stored by the CPU; hold phases ms (cumulative):%s; pre-store flushes %llu (%llu stored an image); stored on the GPU: uniform %llu (%.0f MiB), pattern %llu (%.0f MiB), chain copies %llu\n", static_cast(fills.load()), filledBytes.load() / 1048576.0, static_cast(cpuFills.load()), line.c_str(), static_cast(flushes), static_cast(flushed), static_cast(uniformFills), uniformBytes / 1048576.0, static_cast(patternFills), patternBytes / 1048576.0, static_cast(chainCopies)); } } + }; + static const bool orderedFills = std::getenv("APS5_PIPELINE_DRAIN_FILLS") == nullptr; + if (orderedFills && queueId == 0 && DrawPipeline::Active()) { + DrawPipeline::Queue0().Enqueue([store, queueId] { + GuestMemory::SetCurrentPacket(0x15, queueId); + store(); + }, {{base, base + bytes}}); + return; + } + store(); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp index bf0f1183818..fa2ca447d58 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/WriteEvidence.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include @@ -204,6 +205,10 @@ ShaderMemory::PendingWrite Driver::classifyPendingWrite(std::uint64_t address, s } ShaderMemory::PendingWrite Driver::queryPendingWrite(std::uint64_t address, std::size_t bytes, std::span known) { + if (DrawPipeline::Active()) { + auto& pipeline = DrawPipeline::Queue0(); + if (pipeline.Busy() && pipeline.Overlaps(address, bytes)) pipeline.Drain(DrawPipeline::DrainReason::Capture); + } std::uint64_t ValidateCounters::*reason = nullptr; PendingView pending; pending.Load(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp index 682b28dca22..5da9b3dac88 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp @@ -1,10 +1,19 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "Optimization/ShaderStageInputInfo.hpp" +#include +#include namespace AgcDriver::DriverDetail { +namespace { + template -void Driver::forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compiled, TVisit&& visit) { +void visitWrittenBuffers(const ShaderRecompiler::RecompileResult& compiled, TVisit&& visit) { for (const auto& binding : compiled.bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; for (std::uint32_t element = 0; element < binding.count; ++element) { @@ -20,6 +29,13 @@ void Driver::forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compi } } +} + +template +void Driver::forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compiled, TVisit&& visit) { + visitWrittenBuffers(compiled, std::forward(visit)); +} + void Driver::appendWrittenRanges(const ShaderRecompiler::RecompileResult& compiled, std::vector>& ranges) { forEachWrittenBuffer(compiled, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { ranges.emplace_back(begin, end); }); } @@ -46,6 +62,72 @@ void Driver::noteDrawWriters(std::span stages, s } } +bool ExactDrawWrites() { + static const bool exact = std::getenv("APS5_EXACT_DRAW_WRITES") != nullptr; + return exact; +} + +std::vector> DrawWriteRanges(const Graphics::State& graphics, std::span stages, bool exact) { + constexpr std::uint64_t Slack = 64 * 1024; + constexpr std::uint64_t UnknownBytes = 64ull * 1024 * 1024; + std::vector> ranges; + const auto add = [&](std::uint64_t begin, std::uint64_t bytes) { + if (begin != 0) ranges.emplace_back(begin, begin + bytes); + }; + for (const auto& stage : stages) { + if (stage.program == nullptr) continue; + visitWrittenBuffers(*stage.program, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { ranges.emplace_back(begin, end); }); + for (const auto& binding : stage.program->bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages || binding.kind != ShaderRecompiler::DescriptorKind::StorageImage) continue; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const bool written = element >= binding.imageWritten.size() || binding.imageWritten[element]; + if (!written || binding.guestDescriptor.size() < (static_cast(element) + 1) * 8) continue; + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8, 8); + if (words[0] == 0u && (words[1] & 0xffu) == 0u) continue; + const auto resource = Graphics::DecodeTextureResource(words); + if (resource.baseAddress == 0) continue; + const auto surfaceBytes = Graphics::DescribeSurface(resource).guestBytes; + add(resource.baseAddress, surfaceBytes != 0 ? surfaceBytes : UnknownBytes); + add(resource.dccAddress, surfaceBytes != 0 ? surfaceBytes / 256 + 1 : UnknownBytes); + } + } + } + const auto addColor = [&](const Graphics::ColorTarget& color) { + if (exact && color.bytes != 0) { + const auto slices = static_cast(std::max(color.depth, 1)); + const auto bytes = static_cast(color.bytes) * slices; + add(color.address, bytes); + add(color.dccAddress, (bytes + 255) / 256); + return; + } + const std::uint64_t bytes = color.bytes != 0 ? static_cast(color.bytes) : UnknownBytes; + const auto base = color.surfaceAddress != 0 ? std::min(color.surfaceAddress, color.address) : color.address; + if (base != 0) ranges.emplace_back(base, color.address + 2 * bytes + Slack); + add(color.dccAddress, bytes / 16 + Slack); + }; + if (graphics.hasColorTarget) addColor(graphics.color); + for (const auto& color : graphics.colors) addColor(color); + if (graphics.depth) { + const auto& depth = *graphics.depth; + if (exact && depth.extent.width != 0 && depth.extent.height != 0) { + const bool d16 = depth.format == VK_FORMAT_D16_UNORM || depth.format == VK_FORMAT_D16_UNORM_S8_UINT; + add(depth.address, Graphics::DepthSliceBytes(depth.extent, d16 ? 2u : 4u)); + add(depth.stencilAddress, Graphics::DepthSliceBytes(depth.extent, 1u)); + } else { + const auto align = [](std::uint64_t value) { return (value + 255) & ~255ull; }; + const auto pixels = align(depth.extent.width) * align(depth.extent.height); + const auto bytes = pixels != 0 ? pixels * 8 + Slack : UnknownBytes; + add(depth.address, bytes); + add(depth.stencilAddress, bytes); + } + } + return ranges; +} + +std::vector> Driver::drawWriteRanges(const Graphics::State& graphics, std::span stages) { + return DrawWriteRanges(graphics, stages, ExactDrawWrites()); +} + std::optional Driver::newestWriterLocked(std::uint64_t begin, std::uint64_t end) const { for (auto it = writtenBuffers.rbegin(); it != writtenBuffers.rend(); ++it) { if (begin < it->end && it->begin < end) return *it; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index 8daba37b06a..e304f94d4cc 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "ThreadOwned.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" @@ -15,6 +16,32 @@ namespace AgcDriver::DriverDetail { +namespace { + +bool orderedFills() { + static const bool ordered = std::getenv("APS5_PIPELINE_DRAIN_FILLS") == nullptr; + return ordered; +} + +bool keepsDrawPipeline(std::span packet, std::uint32_t header, std::uint32_t opcode) { + if (header == FlipPacketHeader || header == RenderingWaitPacketHeader) return false; + if (Pm4::DrawOpcode(opcode)) return true; + switch (opcode) { + case 0x11: case 0x12: case 0x13: case 0x26: case 0x2a: case 0x2f: case 0x42: case 0x58: case 0x59: case 0x69: case 0x76: case 0x79: case 0x7a: case 0x81: return true; + case 0x46: return packet.size() >= 2 && (packet[1] & 0x3fu) != 0x39u; + case 0x15: return orderedFills(); + case 0x63: case 0x64: case 0x9f: + return packet.size() >= 5 && !DrawPipeline::Queue0().Overlaps(packet[1] | (static_cast(packet[2]) << 32u), static_cast(packet[4]) * 8); + case 0x10: { + const auto operation = (header >> 2u) & 0x3fu; + return operation == 0 || operation == 0x09 || operation == 0x0b || operation == 0x0c || operation == 0x1a; + } + default: return false; + } +} + +} + template void Driver::timed(double WorkerProfile::*bucket, TWork&& work) { static thread_local WorkerProfile profile; @@ -41,6 +68,21 @@ void Driver::execute(const Submission& submission) { } ~FlipHold() { Release(); } } flipHold{*this, submission.holdsFlip}; + const bool pipelined = submission.queue == 0 && DrawPipeline::Depth() != 0; + static const bool orderedWaits = std::getenv("APS5_PIPELINE_DRAIN_LABELS") == nullptr; + DrawPipeline::Active() = pipelined; + struct PipelineDrain { + bool pipelined; + ~PipelineDrain() { + if (!pipelined || std::uncaught_exceptions() == 0) return; + try { + DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Submission); + } catch (...) { + } + } + } pipelineDrain{pipelined}; + static const bool orderedCompletion = std::getenv("APS5_PIPELINE_DRAIN_SUBMISSIONS") == nullptr && orderedWaits; + if (pipelined && (submission.suspend || !orderedCompletion)) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Submission); if (submission.suspend) { PerformanceContext timingContext(submissionTiming); PerformanceTimer timing("Driver.Suspend"); @@ -108,7 +150,11 @@ void Driver::execute(const Submission& submission) { auto& packetProfile = ShaderRecompiler::ThreadOwned(packetProfileSlot); ++packetProfile.submissions; - bumpEpoch(&EpochBumps::submissions); + static const bool epochEach = std::getenv("APS5_SUBMISSION_EPOCH_EACH") != nullptr; + if (epochEach || submission.enqueuedAt == std::chrono::steady_clock::time_point{} || submission.enqueuedAt > lastEpochBump()) { + if (!epochEach) noteReleasedTails(submission); + bumpEpoch(&EpochBumps::submissions); + } clearResolvedAhead(); gateOpened() = true; struct ClearAhead { @@ -126,13 +172,31 @@ void Driver::execute(const Submission& submission) { if (APS5_ENABLE_TIMING_LOG && submission.queue == 0 && pendingFrameTiming == nullptr) includeTimingSubmission(submission, false); PerformanceContext timingContext(submission.queue == 0 ? pendingFrameTiming.get() : FrameTiming::Async()); PerformanceTimer timing("Driver.Packet"); + const bool predicated = Pm4::Predicated(header) && queue.predication.operation != 0; + if (pipelined && !predicated && header != FlipPacketHeader && header != RenderingWaitPacketHeader && (enqueueLabelPacket(packet, opcode, submission.queue) || enqueueDmaPacket(packet, opcode, submission.queue, queue))) { + traceLabel(packet, submission.queue); + recent.Record(cursor); + cursor = nextCursor; + continue; + } + if (pipelined && !predicated && (opcode == 0x3c || opcode == 0x93) && header != RenderingWaitPacketHeader && orderedWaits && DrawPipeline::Queue0().Busy()) { + const std::uint64_t awaited = packet[2] | (static_cast(packet[3]) << 32u); + if (const auto value = DrawPipeline::Queue0().PendingLabel(awaited, Pm4::WaitAwaitedBytes(packet)); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { + ++waitOutcomes().fromRecorderSameQueue; + gateOpened() = true; + recent.Record(cursor); + cursor = nextCursor; + continue; + } + } + if (pipelined && DrawPipeline::Queue0().Busy() && !keepsDrawPipeline(packet, header, opcode)) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, header == FlipPacketHeader ? 0xffu : opcode); std::shared_lock deviceUse(deviceReplacement, std::defer_lock); if (opcode != 0x3c && opcode != 0x93 && header != RenderingWaitPacketHeader && header != FlipPacketHeader) deviceUse.lock(); timing.Mark("device_lock"); GuestMemory::SetCurrentPacket(header == FlipPacketHeader ? 0xffffu : opcode, submission.queue); CaptureTrace::Log("packet submission=%llu queue=%x offset=%zu header=%08x words=%zu", static_cast(submission.serial), submission.queue, cursor, header, packet.size()); - if (Pm4::Predicated(header) && queue.predication.operation != 0) { + if (predicated) { landPendingWaits(submission.queue); recordQueuedLabelsBeforeRead(submission.queue); if (!Pm4::PredicationPasses(queue)) { @@ -249,6 +313,7 @@ void Driver::execute(const Submission& submission) { currentPacketOffset() = cursor; if (gateOpened()) { gateOpened() = false; + if (pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, opcode); timed(&WorkerProfile::dispatchMs, [&] { resolveGroupAhead(submission, queue, cursor); }); } if (opcode == 0x15) timed(&WorkerProfile::dispatchMs, [&] { dispatch(queue, packet, submission); }); @@ -337,6 +402,18 @@ void Driver::execute(const Submission& submission) { PerformanceContext endContext(submission.queue == 0 ? frameTiming() : FrameTiming::Async()); PerformanceTimer endTiming("Driver.EndSubmission"); static const bool submitAtEnd = std::getenv("APS5_SUBMIT_AT_END") != nullptr; + if (pipelined && orderedCompletion && deferredLabels().labels.empty()) { + if (submission.rewindTail == nullptr) { + deferCompletion(submission); + return; + } + if ((std::atomic_ref(*const_cast(submission.rewindTail - 1)).load(std::memory_order_acquire) & 0x80000000u) != 0) { + DrawPipeline::Queue0().Enqueue([this] { submitOpenWork(); }, {}); + executeRewindTail(submission); + return; + } + } + if (pipelined) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Submission); if (!deferredLabels().labels.empty() || Graphics::Recorder::PendingLabelSince().has_value() || Graphics::Recorder::RecordedWorkSinceSubmit() != 0) { auto& costs = submissionCosts(submission.queue); if (!submitAtEnd && submission.rewindTail == nullptr && submission.queue == 0 && workerQueued() != nullptr && workerQueued()->load(std::memory_order_acquire) != 0) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketFlush.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketFlush.cpp index 33863d1b2ca..410a98b3c8c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketFlush.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketFlush.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -50,6 +51,11 @@ void Driver::flushBetweenPackets(std::uint32_t queue, std::uint32_t header, bool return; } if (reap) lastReapTry = now; + if (DrawPipeline::Active() && (submit || record || capped || recordAtLock)) { + static const bool committerSubmits = std::getenv("APS5_PIPELINE_DRAIN_LABELS") == nullptr; + if (committerSubmits && !record && !recordAtLock && DrawPipeline::Queue0().Busy()) return; + DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Flush); + } std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); if (record) GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 3609990f909..f8704cd7554 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -9,6 +10,7 @@ #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include #include #include @@ -265,4 +267,141 @@ void Driver::dumpSampleCounters(std::uint64_t address) { } } +bool Driver::enqueueDmaPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state) { + static const bool enabled = std::getenv("APS5_PIPELINE_DMA") != nullptr; + if (!enabled || opcode != 0x50 || queue != 0 || packet.size() < 7 || !DrawPipeline::Active() || !deferredLabels().labels.empty()) return false; + const auto source = ((packet[1] >> 29u) & 3u) | ((packet[6] >> 24u) & 4u) | ((packet[6] >> 25u) & 8u); + constexpr std::size_t immediateLimit = std::size_t{16} << 20u; + if (source == 2) { + const auto store = Pm4::ResolveStore(packet, state, immediateLimit); + if (!store.has_value() || store->Bytes().empty() || !GuestMemory::Accessible(reinterpret_cast(store->address), store->Bytes().size(), true)) return false; + const auto address = store->address; + std::vector bytes(store->Bytes().begin(), store->Bytes().end()); + const auto size = bytes.size(); + DrawPipeline::Queue0().Enqueue([this, queue, address, bytes = std::move(bytes)] { + GuestMemory::SetCurrentPacket(0x50, queue); + commitDmaStore(queue, address, bytes); + }, {{address, address + size}}); + return true; + } + const auto copy = Pm4::DecodeMemoryCopy(packet); + if (!copy.has_value() || copy->destination % 4 != 0 || copy->bytes % 4 != 0 || !GuestMemory::Accessible(reinterpret_cast(copy->source), copy->bytes) || !GuestMemory::Accessible(reinterpret_cast(copy->destination), copy->bytes, true)) return false; + DrawPipeline::Queue0().Enqueue([this, queue, copy = *copy] { + GuestMemory::SetCurrentPacket(0x50, queue); + constexpr std::size_t gpuStoreLimit = 65536; + if (copy.bytes > gpuStoreLimit) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Copy); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (const auto localDevice = device.Load()) { + recordDeferredLabels(localDevice.get(), queue); + const auto outcome = localDevice->CopyBuffer(copy.destination, copy.source, copy.bytes, 0, std::numeric_limits::max(), 0, 0, queue, [](std::span, std::uint64_t) {}); + if (outcome.path == 1 || outcome.path == 3) return; + } + } + std::vector bytes(copy.bytes); + GuestMemory::Read(copy.source, bytes, 1); + commitDmaStore(queue, copy.destination, bytes); + }, {{copy->destination, copy->destination + copy->bytes}}); + return true; +} + +void Driver::commitDmaStore(std::uint32_t queue, std::uint64_t address, std::span bytes) { + constexpr std::size_t gpuStoreLimit = 65536; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.Load(); + Graphics::StorageTexture::FlushPending(address, bytes.size(), nullptr, "packet store", Graphics::PublishScope::PartialUnits); + recordDeferredLabels(localDevice.get(), queue); + int reason = 4; + std::uint64_t stamp = 0; + if (bytes.size() <= gpuStoreLimit && localDevice != nullptr) { + stamp = ++eventSerial; + reason = localDevice->WriteLabelOnGpu(address, bytes, stamp, queue); + } + if (reason == 0 || reason == 5 || reason == 6) { + noteLabelStore(address, bytes, stamp, queue); + if (reason == 0) ++storesOnGpu; + else ++storesBehindCompletions; + } else { + if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle(); + GuestMemory::Write(address, bytes, 1); + noteLabelStore(address, bytes, stamp != 0 ? stamp : ++eventSerial, queue); + ++storesOnCpu; + } + Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration()); +} + +bool Driver::enqueueLabelPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue) { + static const bool ordered = std::getenv("APS5_PIPELINE_DRAIN_LABELS") == nullptr; + if (!ordered || !DeferLabels() || (opcode != 0x49 && opcode != 0x37)) return false; + const auto interruptSelect = opcode == 0x49 ? (packet[2] >> 24u) & 7u : 0u; + const bool endOfPipeInterrupt = interruptSelect != 0 && interruptSelect != 3; + const auto label = Pm4::DecodeLabelWrite(packet); + if (!label.has_value()) { + if (opcode != 0x49) return false; + const bool storesNothing = (packet[2] >> 29u) == 0 || (packet[3] | (static_cast(packet[4]) << 32u)) == 0; + if (!storesNothing) return false; + if (!endOfPipeInterrupt) { + ++noOpLabels; + return true; + } + } + std::uint64_t address = 0; + std::vector bytes; + std::vector writes; + if (label.has_value()) { + const auto stored = label->Bytes(); + address = label->address; + bytes.assign(stored.begin(), stored.end()); + writes.emplace_back(address, address + bytes.size()); + } + if (endOfPipeInterrupt) bumpEpoch(&EpochBumps::drains); + auto stored = bytes; + DrawPipeline::Queue0().Enqueue([this, queue, opcode, address, bytes = std::move(bytes), endOfPipeInterrupt] { + GuestMemory::SetCurrentPacket(opcode, queue); + commitLabel(queue, address, bytes, endOfPipeInterrupt); + }, std::move(writes), address, std::move(stored)); + return true; +} + +void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span bytes, bool endOfPipeInterrupt) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.Load(); + recordDeferredLabels(localDevice.get(), queue); + const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0; + int reason = localDevice != nullptr ? 0 : 1; + if (!bytes.empty()) { + const auto stamp = ++eventSerial; + reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(address, bytes, stamp, queue) : 4; + if (reason != 0 && reason != 5 && reason != 6) { + if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle(); + GuestMemory::Write(address, bytes, 4); + } + noteLabelStore(address, bytes, stamp, queue); + Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration()); + countLabelOutcome(reason); + ++immediateLabels; + } else { + ++noOpLabels; + } + if (endOfPipeInterrupt) { + bool deferred = false; + if (localDevice != nullptr && (reason == 0 || reason == 5 || reason == 6)) { + deferred = localDevice->AfterRecordedWork([queue] { AgcDriverDeliverEopInterrupt(queue); }, queue == 0); + if (deferred && workOpen) localDevice->SubmitRecorded(queue == 0); + } + if (!deferred) AgcDriverDeliverEopInterrupt(queue); + } + submitDueAfterCommit(localDevice.get()); +} + +void Driver::submitDueAfterCommit(VulkanDevice* localDevice) { + if (localDevice == nullptr) return; + const auto pending = Graphics::Recorder::PendingLabelSince(); + const bool due = pending.has_value() && std::chrono::steady_clock::now() - *pending >= labelFlushDeadline(); + const bool capped = batchCap() != 0 && Graphics::Recorder::RecordedWorkSinceSubmit() >= batchCap(); + if (due || capped) localDevice->SubmitRecorded(true); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp index 56db0c1f685..309b16849eb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp @@ -1,5 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Queues/WorkerAffinity.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp" @@ -54,6 +56,55 @@ void Driver::markCompleted(std::uint64_t serial) { } } +bool& Driver::completionDeferred() { + static thread_local bool deferred = false; + return deferred; +} + +void Driver::deferCompletion(const Submission& submission) { + std::uint64_t received = submission.received; + { + std::lock_guard lock(mutex); + if (queue0Executing != 0) received = std::min(received, queue0Executing); + queue0Uncommitted.push_back(received); + } + completionDeferred() = true; + DrawPipeline::Queue0().Enqueue([this, serial = submission.serial, received, labelWrites = submission.labelWrites] { + completeSubmission(serial, received, labelWrites); + }, {}); +} + +void Driver::submitOpenWork() { + if (deferredLabels().labels.empty() && !Graphics::Recorder::PendingLabelSince().has_value() && Graphics::Recorder::RecordedWorkSinceSubmit() == 0) return; + ++submissionCosts(0).endSubmits; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::End); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.Load(); + recordDeferredLabels(localDevice.get(), 0); + if (localDevice != nullptr) localDevice->SubmitRecorded(true); +} + +void Driver::completeSubmission(std::uint64_t serial, std::uint64_t received, std::span labelWrites) { + struct Complete { + Driver& driver; + std::uint64_t serial, received; + std::span labelWrites; + ~Complete() { + bool notify = true; + { + std::lock_guard lock(driver.mutex); + driver.markCompleted(serial); + driver.forgetUnfinishedWrites(driver.workers.at(0), labelWrites); + const auto found = std::find(driver.queue0Uncommitted.begin(), driver.queue0Uncommitted.end(), received); + if (found != driver.queue0Uncommitted.end()) driver.queue0Uncommitted.erase(found); + notify = driver.idleWaiters != 0 || driver.orderHolders.load(std::memory_order_acquire) != 0; + } + if (notify) driver.changed.notify_all(); + } + } complete{*this, serial, received, labelWrites}; + submitOpenWork(); +} + const std::atomic*& Driver::workerQueued() { static thread_local const std::atomic* queued = nullptr; return queued; @@ -203,8 +254,10 @@ void Driver::run(std::uint32_t id) noexcept { { std::lock_guard lock(mutex); rethrowFailure(); - markCompleted(submission.serial); - forgetUnfinishedWrites(workers.at(id), submission); + if (!std::exchange(completionDeferred(), false)) { + markCompleted(submission.serial); + forgetUnfinishedWrites(workers.at(id), submission); + } if (id == 0) queue0Executing = 0; notify = idleWaiters != 0 || orderHolders.load(std::memory_order_acquire) != 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp index 3d428973f27..b7cfef3d625 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp @@ -7,6 +7,8 @@ #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" #include "prx/libSceAgcDriver/Execution/include/SubmissionCapture.hpp" #include "prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp" +#include "ThreadOwned.hpp" +#include "prx/libc/include/HostThreadLocal.hpp" #include #include #include @@ -36,6 +38,33 @@ bool EopRelease(std::span words) { } +std::deque& Driver::releasedTails() { + struct ReleasedTailsStorage {}; + return HostThreadLocal, ReleasedTailsStorage>(); +} + +void Driver::noteReleasedTails(const Submission& submission) { + auto& released = releasedTails(); + released.clear(); + thread_local Submission* scratchSlot = nullptr; + auto& scratch = ShaderRecompiler::ThreadOwned(scratchSlot); + const std::uint32_t* tail = submission.rewindTail; + std::size_t words = submission.rewindWords; + for (std::size_t chunk = 0; chunk < 256 && tail != nullptr; ++chunk) { + if ((std::atomic_ref(*const_cast(tail - 1)).load(std::memory_order_acquire) & 0x80000000u) == 0) break; + released.push_back(tail - 1); + scratch.rewindTail = nullptr; + scratch.rewindWords = 0; + try { + copyCommands(scratch, tail, words); + } catch (const std::exception&) { + break; + } + tail = scratch.rewindTail; + words = scratch.rewindWords; + } +} + void Driver::copyCommands(Submission& submission, const std::uint32_t* guest, std::size_t words) { submission.commands.clear(); submission.conditionalEnds.clear(); @@ -248,6 +277,14 @@ void Driver::executeRewindTail(const Submission& stalled) { Submission tail{}; tail.queue = stalled.queue; if (APS5_ENABLE_TIMING_LOG) tail.receivedAt = std::chrono::steady_clock::now(); + auto& released = releasedTails(); + if (!released.empty() && released.front() == stalled.rewindTail - 1) { + released.pop_front(); + tail.enqueuedAt = lastEpochBump(); + } else { + released.clear(); + tail.enqueuedAt = std::chrono::steady_clock::now(); + } copyCommands(tail, stalled.rewindTail, stalled.rewindWords); if (APS5_ENABLE_TIMING_LOG) tail.copiedAt = std::chrono::steady_clock::now(); validate(tail, stalled.rewindTail); @@ -315,10 +352,10 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { submission.serial = accepted + 1; submission.received = ++eventSerial; + const auto now = std::chrono::steady_clock::now(); + submission.enqueuedAt = now; if (capture != nullptr) capture->Take(captured, submission.serial, captureTime(), submission.heldAtSubmit); if (profile) { - const auto now = std::chrono::steady_clock::now(); - submission.enqueuedAt = now; ++costs.submissions; costs.validateNs += static_cast(std::chrono::duration_cast(validated - copied).count()); costs.copyNs += static_cast(std::chrono::duration_cast((copied - start) + (now - validated)).count()); @@ -373,6 +410,7 @@ bool Driver::waitFree(const Submission& submission) { bool Driver::queue0Before(std::uint64_t received) const { if (queue0Executing != 0 && queue0Executing < received) return true; + if (!queue0Uncommitted.empty() && queue0Uncommitted.front() < received) return true; const auto worker = workers.find(0); if (worker == workers.end()) return false; for (const auto& pending : worker->second.pending) { @@ -455,7 +493,11 @@ void Driver::noteHeldAtSubmit(Submission& submission, std::size_t cursor) { } void Driver::forgetUnfinishedWrites(QueueWorker& worker, const Submission& submission) { - for (const auto dword : submission.labelWrites) { + forgetUnfinishedWrites(worker, submission.labelWrites); +} + +void Driver::forgetUnfinishedWrites(QueueWorker& worker, std::span labelWrites) { + for (const auto dword : labelWrites) { const auto found = worker.unfinishedWrites.find(dword); if (found == worker.unfinishedWrites.end()) continue; if (--found->second == 0) worker.unfinishedWrites.erase(found); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitStatistics.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitStatistics.cpp index 99466faac39..28dfef04d26 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitStatistics.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitStatistics.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include @@ -24,8 +25,15 @@ EpochBumps& Driver::epochBumps() { return bumps; } +std::chrono::steady_clock::time_point& Driver::lastEpochBump() { + static thread_local std::chrono::steady_clock::time_point at{}; + return at; +} + void Driver::bumpEpoch(std::uint64_t EpochBumps::*counter) { + lastEpochBump() = std::chrono::steady_clock::now(); GuestMemory::BumpCollectEpoch(); + if (DrawPipeline::Active()) DrawPipeline::EpochToken().fetch_add(1, std::memory_order_relaxed); ++(epochBumps().*counter); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index cd907d4a3df..8850453b340 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -1070,6 +1070,26 @@ struct ThreadCollectMemo { }; thread_local ThreadCollectMemo threadCollectMemo; +struct EpochCollectMap { + std::uint64_t epoch = 0; + std::unordered_map ranges; +}; + +EpochCollectMap& epochCollectMap(std::uint64_t epoch) { + thread_local EpochCollectMap* slot = nullptr; + auto& map = ShaderRecompiler::ThreadOwned(slot); + if (map.epoch != epoch) { + map.ranges.clear(); + map.epoch = epoch; + } + return map; +} + +bool epochCollectMapEnabled() { + static const bool enabled = std::getenv("APS5_NO_EPOCH_COLLECT_MAP") == nullptr; + return enabled; +} + bool sharedCollectMemo() { static const bool shared = std::getenv("APS5_SHARED_COLLECT_MEMO") != nullptr; return shared; @@ -1155,6 +1175,13 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi if (useMemo && !sharedCollectMemo()) { // An entry exists only for a completed walk of an in-arena range, so the tracker is // initialized and watched; nothing below the lock needs asking. + if (epochCollectMapEnabled()) { + const auto& map = epochCollectMap(epoch); + if (const auto found = map.ranges.find(first); found != map.ranges.end() && found->second.unwatched == unwatched && stop <= found->second.end) { + collectMemoHits.fetch_add(1, std::memory_order_relaxed); + return tracker.generation.load(std::memory_order_relaxed); + } + } for (const auto& entry : threadCollectMemo.entries) { if (entry.epoch == epoch && entry.unwatched == unwatched && entry.begin <= first && stop <= entry.end) { // The current generation, not the memoized one: blocks stamped since (MarkWritten, an @@ -1193,7 +1220,13 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi if (collectMemoEnabled()) { const auto serial = unwatchSerial.load(std::memory_order_relaxed); if (sharedCollectMemo()) tracker.memo[tracker.nextMemo++ % tracker.memo.size()] = {first, stop, epoch, serial}; - else threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch, serial}; + else { + threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch, serial}; + if (epochCollectMapEnabled()) { + auto& entry = epochCollectMap(epoch).ranges[first]; + if (entry.unwatched != serial || entry.end < stop) entry = {first, stop, epoch, serial}; + } + } } return tracker.generation; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index 77058182fef..efefda3a7f1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -81,6 +81,8 @@ class DescriptorCache { }; SetAllocation Allocate(VkDescriptorSetLayout layout, std::span sizes, bool updateAfterBind = false, std::optional variableCount = std::nullopt); void Free(const SetAllocation& allocation) noexcept; + SetAllocation AllocateRewritten(VkDescriptorSetLayout layout, std::span sizes, bool updateAfterBind = false, std::optional variableCount = std::nullopt); + void Recycle(VkDescriptorSetLayout layout, const SetAllocation& allocation) noexcept; // APS5_PROFILE_DRAW counters: layouts served from the map / created, sets allocated, pools opened. struct Stats { std::uint64_t layoutHits = 0; @@ -106,6 +108,7 @@ class DescriptorCache { std::vector pools; std::vector updateAfterBindPools; std::vector dedicated; + std::unordered_map> recycled; Stats stats; }; @@ -152,6 +155,8 @@ class ShaderResources { std::shared_ptr buffer; }; DescriptorCache* cache = nullptr; + VkDescriptorSetLayout layout = VK_NULL_HANDLE; + VkDescriptorSetLayout fragmentLayout = VK_NULL_HANDLE; DescriptorCache::SetAllocation fragmentAllocation; DescriptorCache::SetAllocation allocation; std::vector snapshots; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp new file mode 100644 index 00000000000..32de92284c2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp @@ -0,0 +1,42 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_SLIMMUTEX_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_SLIMMUTEX_HPP + +#include +#include +#if defined(__x86_64__) || defined(_M_X64) +#include +#endif + +namespace AgcDriver::Graphics { + +class SlimMutex { +public: + SlimMutex() = default; + SlimMutex(const SlimMutex&) = delete; + SlimMutex& operator=(const SlimMutex&) = delete; + + void lock() { + for (unsigned spins = 0;;) { + if (!locked.exchange(true, std::memory_order_acquire)) return; + while (locked.load(std::memory_order_relaxed)) { + if (spins < 128) { + ++spins; +#if defined(__x86_64__) || defined(_M_X64) + _mm_pause(); +#endif + } else { + std::this_thread::yield(); + } + } + } + } + bool try_lock() { return !locked.load(std::memory_order_relaxed) && !locked.exchange(true, std::memory_order_acquire); } + void unlock() { locked.store(false, std::memory_order_release); } + +private: + std::atomic locked{false}; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 4ae8a5074ff..6daab299165 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -1,5 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include #include #include #include @@ -236,7 +238,6 @@ std::optional BufferPool::Take(std::size_t bytes, VkBufferUsag } auto result = found->second.back().allocation; found->second.pop_back(); - if (found->second.empty()) tier.free.erase(found); --tier.slots; tier.retainedBytes -= result.allocationBytes; ++tier.hits; @@ -249,7 +250,6 @@ void BufferPool::evictOldest(Tier& tier, std::vector& evicted) evicted.push_back(oldest->second.front().allocation); tier.retainedBytes -= oldest->second.front().allocation.allocationBytes; oldest->second.pop_front(); - if (oldest->second.empty()) tier.free.erase(oldest); --tier.slots; ++tier.evictions; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 8e5b85e6345..090c7164867 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -3470,6 +3470,34 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l return {set, pool, chain.size() - 1, updateAfterBind}; } +DescriptorCache::SetAllocation DescriptorCache::AllocateRewritten(VkDescriptorSetLayout layout, std::span sizes, bool updateAfterBind, std::optional variableCount) { + { + std::lock_guard lock(mutex); + if (const auto found = recycled.find(layout); found != recycled.end() && !found->second.empty()) { + const auto allocation = found->second.back(); + found->second.pop_back(); + return allocation; + } + } + return Allocate(layout, sizes, updateAfterBind, variableCount); +} + +void DescriptorCache::Recycle(VkDescriptorSetLayout layout, const SetAllocation& allocation) noexcept { + if (allocation.set == VK_NULL_HANDLE || allocation.pool == VK_NULL_HANDLE) return; + { + std::lock_guard lock(mutex); + try { + auto& list = recycled[layout]; + if (list.size() < 4096) { + list.push_back(allocation); + return; + } + } catch (...) { + } + } + Free(allocation); +} + void DescriptorCache::Free(const SetAllocation& allocation) noexcept { if (allocation.set == VK_NULL_HANDLE || allocation.pool == VK_NULL_HANDLE) return; std::lock_guard lock(mutex); @@ -4387,8 +4415,14 @@ bool ShaderResources::LayoutUpdateAfterBind() const { } ShaderResources::DrawBindings::~DrawBindings() { - if (cache != nullptr && allocation.set != VK_NULL_HANDLE) cache->Free(allocation); - if (cache != nullptr && fragmentAllocation.set != VK_NULL_HANDLE) cache->Free(fragmentAllocation); + if (cache == nullptr) return; + const auto release = [&](VkDescriptorSetLayout setLayout, const DescriptorCache::SetAllocation& set) { + if (set.set == VK_NULL_HANDLE) return; + if (setLayout != VK_NULL_HANDLE) cache->Recycle(setLayout, set); + else cache->Free(set); + }; + release(layout, allocation); + release(fragmentLayout, fragmentAllocation); } std::optional> ShaderResources::MovedReadOnlyBuffers(std::span shaders, Recorder& recorder) const { @@ -4451,7 +4485,9 @@ std::shared_ptr ShaderResources::PrepareDrawBindi const auto maxSnapshotBytes = maxDrawSnapshotBytes; std::size_t totalSnapshotBytes = 0; auto result = std::make_shared(); - std::vector selected; + thread_local std::vector* selectedSlot = nullptr; + auto& selected = ShaderRecompiler::ThreadOwned(selectedSlot); + selected.clear(); std::array stamp{}; bool stamped = false; for (std::size_t index = 0; index < allocations.size(); ++index) { @@ -4539,11 +4575,13 @@ std::shared_ptr ShaderResources::PrepareDrawBindi const auto variableOf = [&](std::size_t set) { return tableBindings[set] != std::numeric_limits::max() ? std::optional(std::max(1u, static_cast(bindings[tableBindings[set]].imageAllocations.size()))) : std::nullopt; }; result->cache = context.descriptorCache; if (!descriptions[0].empty() || splitSets) { - result->allocation = result->cache->Allocate(_layout, sizesOf(0), updateAfterBind, variableOf(0)); + result->allocation = result->cache->AllocateRewritten(_layout, sizesOf(0), updateAfterBind, variableOf(0)); + result->layout = _layout; Require(result->allocation.set != VK_NULL_HANDLE, "draw snapshot descriptor allocation failed"); } if (!descriptions[1].empty() || splitSets) { - result->fragmentAllocation = result->cache->Allocate(fragment.layout, sizesOf(1), updateAfterBind, variableOf(1)); + result->fragmentAllocation = result->cache->AllocateRewritten(fragment.layout, sizesOf(1), updateAfterBind, variableOf(1)); + result->fragmentLayout = fragment.layout; Require(result->fragmentAllocation.set != VK_NULL_HANDLE, "draw snapshot descriptor allocation failed"); } auto buffers = descriptorBuffers; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index ebd2feb5ad2..e999c167a48 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp" #include "prx/libc/include/General.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" @@ -1189,7 +1190,7 @@ class PendingList { // Storage images whose results have not reached guest memory yet. struct PendingWrites { - std::mutex mutex; + SlimMutex mutex; PendingList textures; // Images taken out of `textures` by a FlushPending still storing them (see adjacentPendingUnchanged). std::vector flushing; @@ -3758,9 +3759,11 @@ bool StorageTexture::ScanPending(std::span queries) { query.found = nullptr; if (query.end <= query.begin) continue; const auto bytes = static_cast(query.end - query.begin); - for (const auto* texture : pending.textures) { - if (query.found == nullptr && texture->descriptor.baseAddress == query.begin && texture->guestBytes >= bytes) query.found = texture; - if (texture != query.except && texture->overlaps(query.begin, bytes)) query.overlaps = true; + if (pending.textures.MayOverlap(query.begin, bytes)) { + for (const auto* texture : pending.textures) { + if (query.found == nullptr && texture->descriptor.baseAddress == query.begin && texture->guestBytes >= bytes) query.found = texture; + if (texture != query.except && texture->overlaps(query.begin, bytes)) query.overlaps = true; + } } for (const auto* texture : pending.flushing) { if (texture != query.except && texture->overlaps(query.begin, bytes)) query.overlaps = true; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp index a6a8c6af1de..45d57d64da3 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" +#include "prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" @@ -97,7 +98,7 @@ struct UnitShadow { }; struct Shadows { - std::mutex mutex; + SlimMutex mutex; std::map> byBase; std::uint64_t liveBytes = 0; std::uint64_t peakBytes = 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp new file mode 100644 index 00000000000..56a46eeb120 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp @@ -0,0 +1,126 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using Range = std::pair; +using Ranges = std::vector; + +void Require(bool condition, const std::string& what) { + if (!condition) throw std::runtime_error("draw write ranges: " + what); +} + +bool Contains(const Ranges& ranges, std::uint64_t begin, std::uint64_t end) { + return std::any_of(ranges.begin(), ranges.end(), [&](const Range& range) { return range.first <= begin && end <= range.second; }); +} + +bool Touches(const Ranges& ranges, std::uint64_t begin, std::uint64_t end) { + return std::any_of(ranges.begin(), ranges.end(), [&](const Range& range) { return range.first < end && begin < range.second; }); +} + +std::array ImageWords(std::uint64_t base, std::uint32_t width, std::uint32_t height) { + std::array words{}; + const auto base40 = base >> 8u; + words[0] = static_cast(base40); + words[1] = static_cast((base40 >> 32u) & 0xffu) | (56u << 20u) | (((width - 1u) & 0x3u) << 30u); + words[2] = ((width - 1u) >> 2u) | ((height - 1u) << 14u); + words[3] = 4u | (5u << 3u) | (6u << 6u) | (7u << 9u) | (9u << 28u); + return words; +} + +} + +int main() { + try { + using AgcDriver::DriverDetail::DrawWriteRanges; + constexpr std::uint64_t written = 0x100000000ull; + constexpr std::uint64_t readOnly = 0x200000000ull; + ShaderRecompiler::RecompileResult program; + ShaderRecompiler::DescriptorBinding images; + images.kind = ShaderRecompiler::DescriptorKind::StorageImage; + images.role = ShaderRecompiler::DescriptorRole::GuestImages; + images.count = 2; + for (const auto base : {written, readOnly}) { + const auto words = ImageWords(base, 256, 128); + images.guestDescriptor.insert(images.guestDescriptor.end(), words.begin(), words.end()); + } + images.imageWritten = {true, false}; + program.bindings.push_back(images); + const std::array stages{{{ShaderRecompiler::ShaderStage::Fragment, &program, 0}}}; + AgcDriver::Graphics::State state{}; + const auto resource = AgcDriver::Graphics::DecodeTextureResource(std::span(images.guestDescriptor).subspan(0, 8)); + const auto surfaceBytes = AgcDriver::Graphics::DescribeSurface(resource).guestBytes; + Require(resource.baseAddress == written && surfaceBytes >= 256u * 128u * 4u, "the test T# does not decode as meant"); + for (const bool exact : {false, true}) { + const auto ranges = DrawWriteRanges(state, stages, exact); + Require(Contains(ranges, written, written + surfaceBytes), "a storage image the draw stores to is not a write range"); + Require(!Touches(ranges, readOnly, readOnly + surfaceBytes), "a storage image the draw only reads is a write range"); + } + program.bindings[0].imageWritten.clear(); + Require(Contains(DrawWriteRanges(state, stages, true), readOnly, readOnly + surfaceBytes), "an image without written flags is not a write range"); + auto nullImage = ImageWords(0, 256, 128); + nullImage[0] = 0; + nullImage[1] &= ~0xffu; + std::copy(nullImage.begin(), nullImage.end(), program.bindings[0].guestDescriptor.begin() + 8); + program.bindings[0].imageWritten = {true, true}; + for (const bool exact : {false, true}) { + const auto ranges = DrawWriteRanges(state, stages, exact); + Require(Contains(ranges, written, written + surfaceBytes), "a written image next to a null one is not a write range"); + Require(!Touches(ranges, 0, surfaceBytes), "a null storage image is a write range"); + } + AgcDriver::Graphics::State targets{}; + targets.hasColorTarget = true; + auto& color = targets.color; + color.address = 0x300000000ull; + color.surfaceAddress = color.address; + color.extent = {1920, 1080}; + color.tileMode = AgcDriver::Graphics::ColorTileMode::RenderTarget; + color.elementBytes = 4; + const AgcDriver::Graphics::ColorTargetLayout layout(1920, 1080, color.tileMode, 4); + color.bytes = layout.Bytes(); + color.dccAddress = 0x380000000ull; + AgcDriver::Graphics::DepthTarget depth{}; + depth.address = 0x400000000ull; + depth.stencilAddress = 0x480000000ull; + depth.extent = {1920, 1080}; + depth.format = VK_FORMAT_D32_SFLOAT_S8_UINT; + targets.depth = depth; + const std::span none; + const auto exact = DrawWriteRanges(targets, none, true); + const auto estimated = DrawWriteRanges(targets, none, false); + std::size_t last = 0; + for (std::uint32_t y = 0; y < 1080; y += 7) { + for (std::uint32_t x = 0; x < 1920; x += 5) last = std::max(last, layout.Offset(x, y)); + } + last = std::max(last, layout.Offset(1919, 1079)); + Require(last + 4 <= color.bytes, "the color layout addresses past its bytes"); + Require(Contains(exact, color.address, color.address + color.bytes), "the exact color range misses the layout"); + Require(!Touches(exact, color.address + color.bytes, color.address + color.bytes + 1), "the exact color range runs past the layout"); + Require(Contains(exact, color.dccAddress, color.dccAddress + color.bytes / 256) && !Touches(exact, color.dccAddress + (color.bytes + 255) / 256, color.dccAddress + (color.bytes + 255) / 256 + 1), "the exact DCC range is not one key byte per 256 surface bytes"); + const auto depthBytes = AgcDriver::Graphics::DepthSliceBytes(depth.extent, 4); + const auto stencilBytes = AgcDriver::Graphics::DepthSliceBytes(depth.extent, 1); + Require(Contains(exact, depth.address, depth.address + depthBytes) && !Touches(exact, depth.address + depthBytes, depth.address + depthBytes + 1), "the exact depth range is not DepthSliceBytes"); + Require(Contains(exact, depth.stencilAddress, depth.stencilAddress + stencilBytes) && !Touches(exact, depth.stencilAddress + stencilBytes, depth.stencilAddress + stencilBytes + 1), "the exact stencil range is not DepthSliceBytes"); + for (const auto& [begin, end] : exact) Require(Contains(estimated, begin, end), "an exact range is not inside the estimates"); + std::printf("draw write ranges: color %zu bytes (estimate %llu), depth %llu, stencil %llu\n", color.bytes, static_cast(2 * color.bytes + 65536), static_cast(depthBytes), static_cast(stencilBytes)); + std::puts("draw write ranges tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDma.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDma.cpp new file mode 100644 index 00000000000..c470b4fdc0a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDma.cpp @@ -0,0 +1,68 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "prx/libc/include/Shutdown.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void check(bool condition, const std::string& what) { + if (!condition) throw std::runtime_error("pipelined DMA: " + what); +} + +std::array DmaData(bool immediate, std::uint64_t source, std::uint32_t data, std::uint64_t destination, std::uint32_t bytes) { + return {0xc0055000u, immediate ? (2u << 29u) : 0u, immediate ? data : static_cast(source), immediate ? 0u : static_cast(source >> 32u), static_cast(destination), static_cast(destination >> 32u), bytes}; +} + +void Submit(std::vector& words) { + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + check(sceAgcDriverSubmitDcb(&packet) == 0, "submission failed"); +} + +} + +int main() { +#ifdef _WIN32 + _putenv_s("APS5_PIPELINED_DRAWS", "1"); + _putenv_s("APS5_PIPELINE_DMA", "1"); +#else + setenv("APS5_PIPELINED_DRAWS", "1", 1); + setenv("APS5_PIPELINE_DMA", "1", 1); +#endif + try { + alignas(256) static std::array first{}; + alignas(256) static std::array second{}; + first.fill(0); + second.fill(0); + using namespace AgcDriver::DriverDetail; + const auto drainedBefore = storesDrained.load(); + const auto cpuBefore = storesOnCpu.load() + storesOnGpu.load() + storesBehindCompletions.load(); + std::vector words; + const auto fill = DmaData(true, 0, 0x5eed1234u, reinterpret_cast(first.data()), static_cast(sizeof(first))); + const auto copy = DmaData(false, reinterpret_cast(first.data()), 0, reinterpret_cast(second.data()), 512); + words.insert(words.end(), fill.begin(), fill.end()); + words.insert(words.end(), copy.begin(), copy.end()); + Submit(words); + AgcDriverWaitIdle_nid_postfix(); + for (const auto word : first) check(word == 0x5eed1234u, "the immediate fill did not land"); + for (std::size_t i = 0; i < second.size(); ++i) check(second[i] == (i < 128 ? 0x5eed1234u : 0u), "the copy did not see the fill before it, or wrote past its bytes"); + const auto stores = storesOnCpu.load() + storesOnGpu.load() + storesBehindCompletions.load() - cpuBefore; + check(stores == 2, "the DMA_DATA packets were not committed (" + std::to_string(stores) + " committed stores)"); + check(storesDrained.load() == drainedBefore, "a DMA_DATA packet still drained"); + LibcRunShutdown_nid_postfix(); + std::puts("pipelined DMA tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { LibcRunShutdown_nid_postfix(); } catch (...) {} + return 1; + } +}