perf(agc): keep unimported label stores on the CPU

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -2023,6 +2023,15 @@ if(NOT ANYPS5_AGC_EXECUTION_TESTS OR "PooledBufferRange" IN_LIST ANYPS5_AGC_EXEC
 endif()
 
 target_link_libraries(agc_driver_gpu_map_on_fiber_tests PRIVATE libSceFiber)
+target_link_libraries(agc_driver_same_queue_label_wait_tests PRIVATE libkernel)
+
+if(BUILD_TESTING)
+    add_test(NAME agc_driver_cpu_label_wake COMMAND agc_driver_same_queue_label_wait_tests --cpu-label-wake-only)
+    set_tests_properties(agc_driver_cpu_label_wake PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_driver_cpu_label_wake PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
 
 target_link_libraries(agc_driver_rect_list_corner_tests PRIVATE shader_recompiler_agc_driver)
 target_link_libraries(agc_driver_queue_zero_late_interrupt_tests PRIVATE libkernel)

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp

@@ -12,7 +12,7 @@ struct SubmissionCosts {
     std::atomic<std::uint64_t> submissions{0}, validateNs{0}, copyNs{0}, dequeueNs{0}, completeNs{0}, suspendNs{0}, suspends{0}, suspendsSkipped{0}, endSubmits{0}, endSkipped{0}, notifiesSkipped{0};
 };
 
-extern std::atomic<std::uint64_t> gpuLabels, completionLabels, noOpLabels, unlockedDrains;
+extern std::atomic<std::uint64_t> gpuLabels, completionLabels, noOpLabels, unlockedDrains, syncedUnimportedLabels;
 
 extern std::atomic<std::uint64_t> labelFallbacks[5];
 
@@ -30,7 +30,7 @@ enum TrySite { TryLabel = 0, TryFlush, TryReap, TryPoll, TryIdle, TryCapture, Tr
 
 extern std::atomic<std::uint64_t> triesFailed[TrySites];
 
-extern std::atomic<std::uint64_t> storesOnGpu, storesBehindCompletions, storesOnCpu, storesDrained, dmaFills, copiesToHostMemory, copiesBehindPendingSources;
+extern std::atomic<std::uint64_t> storesOnGpu, storesBehindCompletions, storesOnCpu, storesSyncedUnimported, storesDrained, dmaFills, copiesToHostMemory, copiesBehindPendingSources;
 
 extern std::map<std::uint32_t, std::uint64_t> drainCounts;
 

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp

@@ -4,6 +4,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libkernel/Time/include/SleepWake.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Eq/include/Event.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
@@ -44,8 +45,8 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
                 const auto bytes = label->Bytes();
                 const auto stamp = ++eventSerial;
                 reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4;
-                if (reason == 1) GuestMemory::Write(label->address, bytes, 4);
-                if (reason == 0 || reason == 1 || reason == 5) {
+                if (reason == 1 || reason == 7) GuestMemory::Write(label->address, bytes, 4);
+                if (reason == 0 || reason == 1 || reason == 5 || reason == 7) {
                     noteLabelStore(label->address, bytes, stamp, submission.queue);
                     Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration());
                 }
@@ -59,7 +60,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
                 interruptDeferred = localDevice->AfterRecordedWork([this, queueId] { deliverGuestEopInterrupt(queueId); }, submission.queue == 0);
                 if (interruptDeferred && workOpen) localDevice->SubmitRecorded(submission.queue == 0);
                 wroteOnGpu = true;
-            } else if (reason == 1) {
+            } else if (reason == 1 || reason == 7) {
                 wroteOnGpu = true;
             }
         }
@@ -94,7 +95,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
                 const auto stamp = ++eventSerial;
                 const auto reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4;
                 wroteOnGpu = reason == 0 || reason == 5;
-                if (reason == 1) {
+                if (reason == 1 || reason == 7) {
 
                     GuestMemory::Write(label->address, bytes, 4);
                     wroteOnGpu = true;
@@ -155,15 +156,16 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
                 recordDeferredLabels(localDevice.get(), submission.queue);
                 const auto stamp = ++eventSerial;
                 const auto reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(store->address, bytes, stamp, submission.queue) : 4;
-                if (reason == 0 || reason == 1 || reason == 5) noteLabelStore(store->address, bytes, stamp, submission.queue);
+                if (reason == 0 || reason == 1 || reason == 5 || reason == 7) noteLabelStore(store->address, bytes, stamp, submission.queue);
                 if (reason == 0 || reason == 5) {
                     if (reason == 0) ++storesOnGpu;
                     else ++storesBehindCompletions;
                     wroteOnGpu = true;
                     drained = false;
-                } else if (reason == 1) {
+                } else if (reason == 1 || reason == 7) {
                     GuestMemory::Write(store->address, bytes, 1);
-                    ++storesOnCpu;
+                    if (reason == 7) ++storesSyncedUnimported;
+                    else ++storesOnCpu;
                     wroteOnGpu = true;
                     drained = false;
                 }
@@ -372,10 +374,11 @@ void Driver::commitDmaStore(std::uint32_t queue, std::uint64_t address, std::spa
         if (reason == 0) ++storesOnGpu;
         else ++storesBehindCompletions;
     } else {
-        if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle();
+        if (reason != 1 && reason != 7 && localDevice != nullptr) localDevice->WaitIdle();
         GuestMemory::Write(address, bytes, 1);
         noteLabelStore(address, bytes, stamp != 0 ? stamp : ++eventSerial, queue);
-        ++storesOnCpu;
+        if (reason == 7) ++storesSyncedUnimported;
+        else ++storesOnCpu;
     }
     Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration());
 }
@@ -424,8 +427,9 @@ void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span<c
         const auto stamp = ++eventSerial;
         reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(address, bytes, stamp, queue) : 4;
         if (reason != 0 && reason != 5 && reason != 6) {
-            if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle();
+            if (reason != 1 && reason != 7 && localDevice != nullptr) localDevice->WaitIdle();
             GuestMemory::Write(address, bytes, 4);
+            KernelWakeSleepers_nid_no_patch();
         }
         noteLabelStore(address, bytes, stamp, queue);
         Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration());

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp

@@ -2,6 +2,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libkernel/Time/include/SleepWake.hpp"
 #include "ThreadOwned.hpp"
 #include <cstdlib>
 
@@ -118,8 +119,9 @@ void Driver::recordDeferredLabels(VulkanDevice* localDevice, std::uint32_t queue
         first = false;
         countLabelOutcome(reason);
         if (reason != 0 && reason != 5) {
-            if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle();
+            if (reason != 1 && reason != 7 && localDevice != nullptr) localDevice->WaitIdle();
             GuestMemory::Write(label.address, bytes, 4);
+            KernelWakeSleepers_nid_no_patch();
         }
         noteLabelStore(label.address, bytes, stamp, queue);
     }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/SynchronizationStatistics.cpp

@@ -5,7 +5,7 @@
 
 namespace AgcDriver::DriverDetail {
 
-std::atomic<std::uint64_t> gpuLabels{0}, completionLabels{0}, noOpLabels{0}, unlockedDrains{0};
+std::atomic<std::uint64_t> gpuLabels{0}, completionLabels{0}, noOpLabels{0}, unlockedDrains{0}, syncedUnimportedLabels{0};
 
 std::atomic<std::uint64_t> labelFallbacks[5] = {};
 
@@ -21,7 +21,7 @@ std::atomic<std::uint64_t> packetLockRecordUs{0}, packetLockSubmitUs{0};
 
 std::atomic<std::uint64_t> triesFailed[TrySites] = {};
 
-std::atomic<std::uint64_t> storesOnGpu{0}, storesBehindCompletions{0}, storesOnCpu{0}, storesDrained{0}, dmaFills{0}, copiesToHostMemory{0}, copiesBehindPendingSources{0};
+std::atomic<std::uint64_t> storesOnGpu{0}, storesBehindCompletions{0}, storesOnCpu{0}, storesSyncedUnimported{0}, storesDrained{0}, dmaFills{0}, copiesToHostMemory{0}, copiesBehindPendingSources{0};
 
 std::map<std::uint32_t, std::uint64_t> drainCounts;
 
@@ -39,12 +39,13 @@ SubmissionCosts& submissionCosts(std::uint32_t queue) {
 void reportSync() {
     std::string report;
     for (const auto& [code, count] : drainCounts) report += " " + (code == 0xffffu ? std::string("flip") : Pm4::Name(code << 8u)) + "=" + std::to_string(count);
-    AgcDriver::ProfilePrint_nid_no_patch("[sync] %llu device drains by packet:%s (%llu waited without the GPU mutex); %llu labels written on the GPU, %llu deferred behind completions, %llu no-op, fallbacks: idle %llu, completions %llu, not imported %llu, undecodable %llu; %llu labels queued per worker, recorded in %llu groups, %llu recorded at once; stores: %llu recorded on the GPU, %llu behind completions, %llu on the CPU (idle), %llu synced (drained), %llu DMA fills stored as buffer fills; %llu copies into host memory synced on their source; %llu copies of up to 64 KiB recorded on the GPU behind their source's pending stores\n", static_cast<unsigned long long>(drainTotal), report.c_str(), static_cast<unsigned long long>(unlockedDrains.load()), static_cast<unsigned long long>(gpuLabels.load()), static_cast<unsigned long long>(completionLabels.load()), static_cast<unsigned long long>(noOpLabels.load()), static_cast<unsigned long long>(labelFallbacks[1].load()), static_cast<unsigned long long>(labelFallbacks[2].load()), static_cast<unsigned long long>(labelFallbacks[3].load()), static_cast<unsigned long long>(labelFallbacks[4].load()), static_cast<unsigned long long>(queuedLabels.load()), static_cast<unsigned long long>(labelGroups.load()), static_cast<unsigned long long>(immediateLabels.load()), static_cast<unsigned long long>(storesOnGpu.load()), static_cast<unsigned long long>(storesBehindCompletions.load()), static_cast<unsigned long long>(storesOnCpu.load()), static_cast<unsigned long long>(storesDrained.load()), static_cast<unsigned long long>(dmaFills.load()), static_cast<unsigned long long>(copiesToHostMemory.load()), static_cast<unsigned long long>(copiesBehindPendingSources.load()));
+    AgcDriver::ProfilePrint_nid_no_patch("[sync] %llu device drains by packet:%s (%llu waited without the GPU mutex); %llu labels written on the GPU, %llu deferred behind completions, %llu no-op, fallbacks: idle %llu, no covering import (recorder synced) %llu, completions %llu, not imported %llu, undecodable %llu; %llu labels queued per worker, recorded in %llu groups, %llu recorded at once; stores: %llu recorded on the GPU, %llu behind completions, %llu on the CPU (idle), %llu on the CPU with no covering import (recorder synced), %llu synced (drained), %llu DMA fills stored as buffer fills; %llu copies into host memory synced on their source; %llu copies of up to 64 KiB recorded on the GPU behind their source's pending stores\n", static_cast<unsigned long long>(drainTotal), report.c_str(), static_cast<unsigned long long>(unlockedDrains.load()), static_cast<unsigned long long>(gpuLabels.load()), static_cast<unsigned long long>(completionLabels.load()), static_cast<unsigned long long>(noOpLabels.load()), static_cast<unsigned long long>(labelFallbacks[1].load()), static_cast<unsigned long long>(syncedUnimportedLabels.load()), static_cast<unsigned long long>(labelFallbacks[2].load()), static_cast<unsigned long long>(labelFallbacks[3].load()), static_cast<unsigned long long>(labelFallbacks[4].load()), static_cast<unsigned long long>(queuedLabels.load()), static_cast<unsigned long long>(labelGroups.load()), static_cast<unsigned long long>(immediateLabels.load()), static_cast<unsigned long long>(storesOnGpu.load()), static_cast<unsigned long long>(storesBehindCompletions.load()), static_cast<unsigned long long>(storesOnCpu.load()), static_cast<unsigned long long>(storesSyncedUnimported.load()), static_cast<unsigned long long>(storesDrained.load()), static_cast<unsigned long long>(dmaFills.load()), static_cast<unsigned long long>(copiesToHostMemory.load()), static_cast<unsigned long long>(copiesBehindPendingSources.load()));
 }
 
 void countLabelOutcome(int reason) {
     if (reason == 0) ++gpuLabels;
     else if (reason == 5) ++completionLabels;
+    else if (reason == 7) ++syncedUnimportedLabels;
     else ++labelFallbacks[std::min(reason, 4)];
 }
 

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -2306,8 +2306,14 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::by
     if (recorder.Idle()) return 1;
     static const bool drain = std::getenv("APS5_DRAIN_COMPLETION_LABELS") != nullptr;
     const auto context = graphicsContext();
+    const bool syncUnimported = queue == 0 && !recorder.SubmitHeld();
+    const auto unimported = [&] {
+        recorder.Sync();
+        return 7;
+    };
+    if (syncUnimported && !Graphics::HostImportCovers(context, address, bytes.size())) return unimported();
     const auto* import = Graphics::HostImportFor(context, address, bytes.size());
-    if (import == nullptr) return 3;
+    if (import == nullptr) return syncUnimported ? unimported() : 3;
     // A unit shadow's results in the label's unit reach the import before the store lands over
     // part of it (the store's stamp then makes the unit stale); the deferred label paths call no
     // FlushPending of their own.

core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp

@@ -198,6 +198,7 @@ class Recorder {
     // Ends and submits the open batch without waiting.
     void Submit();
     void HoldNextSubmit(VkSemaphore semaphore, std::uint64_t value);
+    bool SubmitHeld() const;
     // Submits the open batch and returns the serial of the newest submitted batch (0 when nothing was
     // ever submitted); WaitSerial(serial) then covers all recorded work.
     std::uint64_t SubmitAndEpoch();
@@ -809,6 +810,7 @@ class Recorder {
     VkSemaphore timeline = VK_NULL_HANDLE;
     VkSemaphore submitHold = VK_NULL_HANDLE;
     std::uint64_t submitHoldValue = 0;
+    std::uint64_t heldSerial = 0;
     struct ProgressWatcher;
     std::unique_ptr<ProgressWatcher> progressWatcher;
     // Identity for a thread that released the mutex around a wait (syncThroughUnlocked): the

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -3441,10 +3441,11 @@ bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped&
     } else if (state.refreshedGeneration > generation) {
         return true;
     }
-    if (!adopting) current = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
     for (const auto& mapped : ranges) {
         const auto range = mapped.lock();
-        if (range == nullptr || leasedRangeOwner(current, range->address) != range) continue;
+        if (range == nullptr || range->bytes <= 0x10000) continue;
+        if (current.empty()) current = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
+        if (leasedRangeOwner(current, range->address) != range) continue;
         try {
             if (!GuestMemory::GpuMutex().HeldByThisThread()) {
                 const auto run = importRun(current, range->address, std::numeric_limits<std::uint64_t>::max(), effectiveBudget(context, state));

core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp

@@ -3517,6 +3517,10 @@ void Recorder::HoldNextSubmit(VkSemaphore semaphore, std::uint64_t value) {
     submitHoldValue = value;
 }
 
+bool Recorder::SubmitHeld() const {
+    return submitHold != VK_NULL_HANDLE || (heldSerial != 0 && unsignaled(heldSerial));
+}
+
 void Recorder::Submit() {
     PerformanceTimer frameTiming("Recorder.Submit");
     // The work count is cleared even when nothing is open: the driver counts a dispatch after its
@@ -3592,6 +3596,7 @@ void Recorder::Submit() {
     Check(result, "vkQueueSubmit recorder");
     batch->submitted = true;
     batch->serial = ++submissions;
+    if (hold != VK_NULL_HANDLE) heldSerial = batch->serial;
     batch->submittedAt = std::chrono::steady_clock::now();
     if (DrawProfiled()) {
         const auto us = std::chrono::duration<double, std::micro>(batch->submittedAt - submitStart).count();