From 36df6631b6bccbe789dd5146989c00816912222e Mon Sep 17 00:00:00 2001 From: romainveurrier Date: Sat, 17 Oct 2026 00:04:37 +0200 Subject: [PATCH] fix(agc): keep guest stores over freed storage-image memory --- .../Execution/include/GuestMemory.hpp | 2 + .../src/Driver/Packets/PacketMemory.cpp | 1 + .../Execution/src/GuestMemory.cpp | 46 +++ .../Execution/src/VulkanDevice.cpp | 1 + .../Graphics/include/Texture.hpp | 3 + .../libSceAgcDriver/Graphics/src/Texture.cpp | 56 ++- .../execution/StorageSameBytesReuseR64Kbx.cpp | 279 ++++++++++++++ .../StorageWriteBackWindowR64Kbx.cpp | 348 ++++++++++++++++++ 8 files changed, 724 insertions(+), 12 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/StorageSameBytesReuseR64Kbx.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/StorageWriteBackWindowR64Kbx.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index 23886849973..614316f6651 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -50,6 +50,8 @@ void Write(std::uint64_t address, std::span source, std::size_t // Stores the parts of `current` that differ from `original` (the guest bytes the GPU started from), so // guest writes made meanwhile to untouched bytes survive. Compares in 256-byte blocks. void WriteChanged(std::uint64_t address, std::span current, std::span original); +bool WriteChangedOverOriginal(std::uint64_t address, std::span current, std::span original); +bool WriteChangedOverOriginalCommitted(std::uint64_t address, std::span current, std::span original); // Write tracking. The guest arena is reserved with page write watching, so CollectWrites learns which // pages of a range the CPU wrote since they were last collected, stamps their 64 KiB blocks with a new diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 452c674ab32..d0fa3286a7c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -50,6 +50,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue countLabelOutcome(reason); ++immediateLabels; } else { + if (localDevice != nullptr) Graphics::StorageTexture::StoreBeforeGuestCompletion("interrupt"); ++noOpLabels; } if (reason == 0 || reason == 5) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index d0859ab0610..7487195987b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -2225,6 +2225,52 @@ void WriteChanged(std::uint64_t address, std::span current, std }); } +bool WriteChangedOverOriginal(std::uint64_t address, std::span current, std::span original) { + require(current.size() == original.size(), "write-back snapshot sizes differ"); + if (current.empty()) return true; + const ReadSiteScope site(ReadSite::Store); + FlushGpuWrites(address, current.size()); + const TimedAccess timed(CounterChangedWrite, current.size()); + auto* destination = reinterpret_cast(address); + CheckRange(destination, current.size(), 1, true); + const auto size = current.size(); + bool complete = true; + storeOwn(address, size, [&] { + std::size_t firstChanged = size; + std::size_t lastChanged = 0; + const auto exchange = [&](std::size_t at) { + TPiece expected, desired; + std::memcpy(&expected, original.data() + at, sizeof(TPiece)); + std::memcpy(&desired, current.data() + at, sizeof(TPiece)); + if (expected == desired) return; + if (!std::atomic_ref(*reinterpret_cast(destination + at)).compare_exchange_strong(expected, desired, std::memory_order_relaxed)) { + complete = false; + return; + } + firstChanged = std::min(firstChanged, at); + lastChanged = std::max(lastChanged, at + sizeof(TPiece)); + }; + std::size_t at = 0; + for (; at < size && (address + at) % sizeof(std::uint64_t) != 0; ++at) exchange.template operator()(at); + for (; at + sizeof(std::uint64_t) <= size; at += sizeof(std::uint64_t)) exchange.template operator()(at); + for (; at < size; ++at) exchange.template operator()(at); + return firstChanged < lastChanged ? std::pair{address + firstChanged, address + lastChanged} : std::pair{0, 0}; + }); + return complete; +} + +bool WriteChangedOverOriginalCommitted(std::uint64_t address, std::span current, std::span original) { + require(current.size() == original.size(), "write-back snapshot sizes differ"); + if (Accessible(reinterpret_cast(address), current.size(), true)) return WriteChangedOverOriginal(address, current, original); + bool complete = true; + for (const auto& [begin, end] : CommittedRanges(address, current.size(), true)) { + const auto offset = static_cast(begin - address); + const auto length = static_cast(end - begin); + complete = WriteChangedOverOriginal(begin, current.subspan(offset, length), original.subspan(offset, length)) && complete; + } + return complete; +} + void Write(std::uint64_t address, std::span source, std::size_t alignment) { if (source.empty()) return; // A store, as in WriteChanged. diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 44c1dce86b7..db48030da28 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -2160,6 +2160,7 @@ bool VulkanDevice::StoresPendingOver(std::uint64_t address, std::size_t bytes) c int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) { static const bool storeAtLabel = Graphics::StorageTexture::StoreAtLabelRequested(std::getenv("APS5_STORE_AT_LABEL")); if (storeAtLabel) Graphics::StorageTexture::FlushAllPending("label"); + Graphics::StorageTexture::StoreBeforeGuestCompletion("label"); if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4; auto& recorder = *state->recorder; // Batches that already finished are retired before the checks: their completions ran, so they diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index dfe074f738d..22fcb4e274a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -195,6 +195,7 @@ class StorageTexture : public std::enable_shared_from_this { // overlapping pending image, as before). static bool FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr, const char* reason = "memory access", PublishScope scope = PublishScope::Whole, bool* published = nullptr); static void FlushAllPending(const char* reason); + static void StoreBeforeGuestCompletion(const char* reason); static void SaveBeforeUnwatch(std::uint64_t address, std::size_t bytes); static constexpr std::uint64_t SmallStoreBytes = 64 * 1024; static std::size_t StoreSmallPending(const char* reason); @@ -349,6 +350,7 @@ class StorageTexture : public std::enable_shared_from_this { static std::uint64_t SinglePassMoves(); static std::uint64_t RefreshesProved(); static std::uint64_t UntrackedCompares(); + static std::uint64_t CpuWonStores(); std::uint64_t GuestBytes() const; std::size_t OriginalSnapshotBytes() const { return original.capacity(); } VkDeviceSize AllocationBytes() const { return memoryBytes; } @@ -590,6 +592,7 @@ class StorageTexture : public std::enable_shared_from_this { VkFormat storageFormat = VK_FORMAT_UNDEFINED; // Results are on the GPU only (guarded by the pending-write registry lock). bool dirty = false; + std::uint64_t pendingSinceCompletion = 0; std::uint64_t version = 0; // `original` holds the guest bytes; false after a GPU-side clear, which never read them. std::vector layerOriginal; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index ed284ea6581..c5aaa70f409 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -87,6 +87,8 @@ std::atomic partialUploads{0}, partialUploadBytes{0}, partialWrit // writeBackLayers), and pending units a DCC clear -> uncompressed key flip kept as the texels // (Refresh). std::atomic pretestSkipped{0}, unregisteredDropped{0}, keyFlipKept{0}; +std::atomic cpuWonStores{0}; +std::atomic guestCompletions{0}; // Images stored by a FlushPending after a hook skip of theirs (AccessKeptByCpu), of which by the // hook for the read site of the last skip; images evictStale dropped (the [hooksync] and [storage] lines). std::atomic flushedAfterSkip{0}, flushedAfterSkipSameSite{0}, staleEvicted{0}; @@ -2054,6 +2056,10 @@ std::uint64_t StorageTexture::UntrackedCompares() { return untrackedCompares.load(std::memory_order_relaxed); } +std::uint64_t StorageTexture::CpuWonStores() { + return cpuWonStores.load(std::memory_order_relaxed); +} + bool StorageTexture::ServesKeysAt(std::uint64_t dccAddress) const { const bool locked = GuestMemory::GpuMutex().HeldByThisThread(); const auto readFollowed = [&] { @@ -3123,6 +3129,7 @@ void StorageTexture::markLayersPending(std::uint32_t first, std::uint32_t count) anyUnsaved = anyUnsaved || unsaved[layer]; } if (anyUnsaved && !GuestMemory::Watched(descriptor.baseAddress, static_cast(guestBytes))) captureGuestBytes(&unsaved); + if (!anyLayerPending()) pendingSinceCompletion = guestCompletions.load(std::memory_order_acquire); for (std::uint32_t layer = first; layer < first + count; ++layer) layerPending[layer] = true; // Results of this image now cover the alias's results borrowed into these units: this image // stores them, the alias no longer has to (its results, unchanged since the borrow, are in @@ -3485,6 +3492,22 @@ void StorageTexture::FlushAllPending(const char* reason) { static_cast(FlushPending(0, std::numeric_limits::max(), nullptr, reason)); } +void StorageTexture::StoreBeforeGuestCompletion(const char* reason) { + static const bool deferred = std::getenv("APS5_DEFER_COMPARED_RESULTS") != nullptr; + if (!deferred) { + std::vector> ranges; + { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + for (const auto* texture : pending.textures) ranges.emplace_back(texture->descriptor.baseAddress, static_cast(texture->guestBytes)); + } + for (const auto& [address, bytes] : ranges) { + if (!GuestMemory::Watched(address, bytes)) static_cast(FlushPending(address, bytes, nullptr, reason, PublishScope::None)); + } + } + guestCompletions.fetch_add(1, std::memory_order_acq_rel); +} + std::size_t StorageTexture::StoreSmallPending(const char* reason) { std::vector> small; { @@ -4564,6 +4587,18 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { constexpr std::uint64_t block = 65536; const auto spanBegin = firstStored & ~(block - 1); const auto spanBlocks = static_cast((lastStored - spanBegin + block - 1) / block); + static const bool recordedWriteBack = std::getenv("APS5_RECORDED_WRITEBACK") != nullptr; + std::vector segments; + if (recordedWriteBack || pendingSinceCompletion == guestCompletions.load(std::memory_order_acquire)) { + if (const auto* import = HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes))) segments.push_back({import, descriptor.baseAddress, descriptor.baseAddress + guestBytes}); + else segments = importSegments(); + } + if (segments.empty()) { + original.resize(static_cast(guestBytes)); + std::vector unsaved = layerOriginal; + unsaved.flip(); + for (const auto& [begin, end] : unitRuns(unsaved)) GuestMemory::ReadCommitted(descriptor.baseAddress + begin, std::span(original).subspan(static_cast(begin), static_cast(end - begin))); + } // The keep-blocks decision must see every CPU write to the stored span up to now, so it // bypasses the per-packet collect memo (a Refresh collect earlier in the same packet would // satisfy it); the unselected layers keep their own generations and are collected by their @@ -4643,7 +4678,6 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { // own stamps, so every later import writer stamps newer. std::vector shadowed; std::vector> imported; - std::vector segments; const auto markShadowed = [&](std::uint64_t generation) { for (const auto& segment : segments) { std::vector ranges; @@ -4679,8 +4713,6 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { settle(true); return; } - if (const auto* import = HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes))) segments.push_back({import, descriptor.baseAddress, descriptor.baseAddress + guestBytes}); - else segments = importSegments(); if (!segments.empty()) { Require(!unsavedOutsideWatch, "storing storage image results pending over memory outside the write watch without the guest bytes saved before them is not implemented"); // Outside the write watch the stored bytes are what the next store compares with. @@ -4854,15 +4886,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { } return; } - original.resize(static_cast(guestBytes)); - // The store below compares against the guest bytes, read now when `original` does not hold - // them (a GPU clear, a dropped or kept unit). Reading them says nothing about the image: a - // stale unit (dropped, or kept for the CPU) still differs from them, so `original` vouches for - // the content again only where it did before, or once every unit is stored. Require(!unsavedOutsideWatch, "storing storage image results pending over memory outside the write watch without the guest bytes saved before them is not implemented"); - std::vector unsaved = layerOriginal; - unsaved.flip(); - for (const auto& [begin, end] : unitRuns(unsaved)) GuestMemory::ReadCommitted(descriptor.baseAddress + begin, std::span(original).subspan(static_cast(begin), static_cast(end - begin))); DeviceBuffer linear(context, static_cast(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); DeviceBuffer tiled(context, original.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); Buffer host(context, original.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); @@ -4936,7 +4960,15 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { for (const auto& [from, to] : keep) { const auto offset = static_cast(from - descriptor.baseAddress); const auto length = static_cast(to - from); - GuestMemory::WriteChangedCommitted(from, current.subspan(offset, length), std::span(original).subspan(offset, length)); + if (!GuestMemory::WriteChangedOverOriginalCommitted(from, current.subspan(offset, length), std::span(original).subspan(offset, length))) { + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!layers[layer] || layerBegin(layer) >= to || layerBegin(layer) + layerBytes(layer) <= from) continue; + skippedLayer[layer] = true; + } + cpuWonStores.fetch_add(1, std::memory_order_relaxed); + static std::atomic reports{0}; + if (reports.fetch_add(1) < 8) std::fprintf(stderr, "[gpu] storage image 0x%llx: the CPU stored into 0x%llx+0x%zx during its write-back; keeping the CPU's bytes\n", static_cast(descriptor.baseAddress), static_cast(from), length); + } std::memcpy(original.data() + offset, current.data() + offset, length); } // The texels now hold the whole image, so later reads must see them rather than a fast clear diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StorageSameBytesReuseR64Kbx.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StorageSameBytesReuseR64Kbx.cpp new file mode 100644 index 00000000000..7d0f1855991 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StorageSameBytesReuseR64Kbx.cpp @@ -0,0 +1,279 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::StorageTexture; +using Clock = std::chrono::steady_clock; + +constexpr std::size_t Block = 0x10000; +constexpr std::uint32_t Side = 32; +constexpr std::uint32_t Format8888Unorm = 56; +constexpr std::uint32_t TileR64KBX = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint8_t Before = 0xa5; +constexpr std::uint32_t Attempts = 8; +constexpr std::uint32_t Texel = 0x00ffffffu; +constexpr std::array LinkOffsets{0x0, 0x8}; +constexpr std::size_t TriggerOffset = 0x8000; + +alignas(256) constexpr std::array StoreCode{ + 0x7e02020c, 0x7e0402f2, 0x7e0602f2, 0x7e0802f2, 0x7e0a0280, 0xf0201f08, 0x00010200, 0xbf810000, +}; + +std::uint64_t AddressOf(const void* data) { + return static_cast(reinterpret_cast(data)); +} + +std::uint32_t Low(const volatile void* pointer) { + return static_cast(reinterpret_cast(pointer)); +} + +std::uint32_t High(const volatile void* pointer) { + return static_cast(static_cast(reinterpret_cast(pointer)) >> 32u); +} + +std::array TextureDescriptor(std::uint64_t address) { + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888Unorm << 20u) | (((Side - 1u) & 3u) << 30u), + ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), + 0xfacu | (TileR64KBX << 20u) | (Type2D << 28u), + 0u, 0u, 0u, 0u, + }; +} + +class GuestBlock { +public: + explicit GuestBlock(bool watched = false) : watched(watched) { + if (watched) { +#ifdef _WIN32 + bytes = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(Block, Block)); + if (bytes != nullptr) GuestArena::GuestArenaCommit_nid_postfix(bytes, Block, PAGE_READWRITE, Block); +#else + void* mapped = mmap(nullptr, 2 * Block, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + Block - 1) & ~std::uintptr_t{Block - 1}; + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned != begin + Block) munmap(reinterpret_cast(aligned + Block), begin + Block - aligned); + bytes = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(bytes, Block); + } +#endif + } else { +#ifdef _WIN32 + bytes = static_cast(VirtualAlloc(nullptr, Block, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + bytes = static_cast(std::aligned_alloc(Block, Block)); +#endif + } + Require(bytes != nullptr, "storage same-bytes reuse: cannot allocate a guest block"); + std::memset(bytes, Before, Block); + GuestAllocations::Mutation().Add(bytes, Block, true, true, true); + } + ~GuestBlock() { + GuestAllocations::Mutation().Remove(bytes); + if (watched) { +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(bytes, Block); + GuestArena::GuestArenaRelease_nid_postfix(bytes, Block); +#else + munmap(bytes, Block); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(bytes, Block); +#endif + return; + } +#ifdef _WIN32 + VirtualFree(bytes, 0, MEM_RELEASE); +#else + std::free(bytes); +#endif + } + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + std::uint8_t* Data() const { return bytes; } + std::uint64_t Address() const { return AddressOf(bytes); } + +private: + bool watched; + std::uint8_t* bytes = nullptr; +}; + +void StoreThenLabel(const GuestBlock& image, volatile std::uint32_t* label, std::uint32_t value) { + std::vector words; + const auto registers = [&](std::uint32_t first, std::span values) { + words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); + words.push_back(first); + words.insert(words.end(), values.begin(), values.end()); + }; + const std::array shape{Side, 1, 1}; + registers(0x207, shape); + const auto code = AddressOf(StoreCode.data()); + const std::array program{static_cast(code >> 8u), static_cast(code >> 40u)}; + registers(0x20c, program); + std::array userData{}; + const auto descriptor = TextureDescriptor(image.Address()); + std::copy(descriptor.begin(), descriptor.end(), userData.begin() + 4); + const std::array layout{(static_cast(userData.size()) << 1u) | 0x80u}; + registers(0x213, layout); + registers(0x240, userData); + words.insert(words.end(), {0xc0031500u, Side, 1, 1, 0x41u}); + words.insert(words.end(), {0xc0064900u, 0x514u, (1u << 29u) | (2u << 24u), Low(label), High(label), value, 0u, 0u}); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); +} + +bool AwaitLabel(const volatile std::uint32_t* label, std::uint32_t value) { + const auto deadline = Clock::now() + std::chrono::seconds(10); + while (*label != value) { + if (Clock::now() >= deadline) return false; + std::this_thread::yield(); + } + return true; +} + +std::uint64_t Word(const GuestBlock& block, std::size_t offset) { + std::uint64_t value = 0; + std::memcpy(&value, block.Data() + offset, sizeof(value)); + return value; +} + +std::uint64_t LinkValue(std::uint32_t attempt, std::size_t index) { + return 0x0000000280000000ull + (static_cast(attempt) << 16u) + index * 0x40u + 0x10u; +} + +constexpr std::uint64_t TexelPair = (static_cast(Texel) << 32u) | Texel; + +bool Pending(const GuestBlock& image) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + return StorageTexture::FindPending(image.Address(), Block) != nullptr; +} + +void Publish(const GuestBlock& image) { + std::array read{}; + AgcDriver::GuestMemory::Read(image.Address() + TriggerOffset, read); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + StorageTexture::FlushAllPending("storage same-bytes reuse fixture"); +} + +struct Counts { + std::uint32_t kept = 0; + std::uint32_t lostToTexels = 0; + std::uint32_t lost = 0; + std::uint32_t notPending = 0; +}; + +void Attempt(GuestBlock& image, GuestBlock& labels, std::uint32_t attempt, bool sameLinks, Counts& counts) { + auto* label = reinterpret_cast(labels.Data()); + const auto value = 100 + attempt; + *label = 0; + std::memset(image.Data(), Before, Block); + for (std::size_t index = 0; index < LinkOffsets.size(); ++index) { + const auto stale = LinkValue(attempt, index); + std::memcpy(image.Data() + LinkOffsets[index], &stale, sizeof(stale)); + } + StoreThenLabel(image, label, value); + Require(AwaitLabel(label, value), "storage same-bytes reuse: a label never arrived"); + if (!Pending(image)) ++counts.notPending; + for (std::size_t index = 0; index < LinkOffsets.size(); ++index) { + const auto freed = sameLinks ? LinkValue(attempt, index) : LinkValue(attempt + 1000, index); + std::memcpy(image.Data() + LinkOffsets[index], &freed, sizeof(freed)); + } + Publish(image); + bool kept = true; + bool texels = true; + for (std::size_t index = 0; index < LinkOffsets.size(); ++index) { + const auto observed = Word(image, LinkOffsets[index]); + kept = kept && observed == (sameLinks ? LinkValue(attempt, index) : LinkValue(attempt + 1000, index)); + texels = texels && observed == TexelPair; + } + if (kept) ++counts.kept; + else if (texels) ++counts.lostToTexels; + else ++counts.lost; + std::printf("[%s %u] link0=0x%016llx link1=0x%016llx result=%s\n", sameLinks ? "same links" : "new links", attempt + 1, static_cast(Word(image, LinkOffsets[0])), + static_cast(Word(image, LinkOffsets[1])), kept ? "KEPT" : texels ? "LOST_TO_TEXELS" : "LOST"); +} + +enum class Verdict { Pass, Skip, Fail }; + +Verdict Run(bool watchedMemory) { + GuestBlock image(watchedMemory); + GuestBlock labels; + const char* kind = watchedMemory ? "watched" : "compared"; + const bool watched = AgcDriver::GuestMemory::Watched(image.Address(), Block); + std::printf("[same-bytes reuse #5934] memory=%s surface=32x32 RGBA8 R64KBX watched=%d hostImport=%s attempts=%u+%u\n", kind, watched, std::getenv("APS5_NO_HOST_IMPORT") != nullptr ? "disabled" : "allowed", + Attempts, Attempts); + if (watched != watchedMemory) { + std::printf("[same-bytes reuse #5934] memory=%s SKIP: the write watch does not cover the surface as asked\n", kind); + return Verdict::Skip; + } + Counts changed; + for (std::uint32_t attempt = 0; attempt < Attempts; ++attempt) Attempt(image, labels, attempt, false, changed); + Counts same; + for (std::uint32_t attempt = 0; attempt < Attempts; ++attempt) Attempt(image, labels, Attempts + attempt, true, same); + std::printf("[same-bytes reuse #5934] memory=%s summary: new links kept/lostToTexels/lost=%u/%u/%u, same links kept/lostToTexels/lost=%u/%u/%u, notPendingAtLabel=%u\n", kind, changed.kept, + changed.lostToTexels, changed.lost, same.kept, same.lostToTexels, same.lost, changed.notPending + same.notPending); + const auto lost = changed.lostToTexels + changed.lost + same.lostToTexels + same.lost; + if (lost != 0) { + std::printf("[same-bytes reuse #5934] memory=%s FAIL: %u attempts lost the links the CPU stored at the free\n", kind, lost); + return Verdict::Fail; + } + std::printf("[same-bytes reuse #5934] memory=%s PASS: the links stored at the free survived, new or identical (%u of %u results already in memory at their label)\n", kind, + changed.notPending + same.notPending, 2 * Attempts); + return Verdict::Pass; +} + +} + +int main() { + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + struct ShutdownGuard { + ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} } + } shutdown; + const bool watchedRun = std::getenv("APS5_TEST_WATCHED_SAME_BYTES") != nullptr; + std::array verdicts{Run(false), watchedRun ? Run(true) : Verdict::Skip}; + if (!watchedRun) std::puts("[same-bytes reuse #5934] memory=watched NOT RUN (APS5_TEST_WATCHED_SAME_BYTES=1 runs it)"); + const auto count = [&](Verdict verdict) { return std::count(verdicts.begin(), verdicts.end(), verdict); }; + std::printf("[same-bytes reuse #5934] PASS=%td SKIP=%td FAIL=%td\n", count(Verdict::Pass), count(Verdict::Skip), count(Verdict::Fail)); + if (count(Verdict::Fail) != 0) return 1; + return count(Verdict::Pass) != 0 ? 0 : VulkanTestSkipped; + } catch (const std::exception& error) { + std::cerr << "[same-bytes reuse #5934] FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StorageWriteBackWindowR64Kbx.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StorageWriteBackWindowR64Kbx.cpp new file mode 100644 index 00000000000..5a4ca03843e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StorageWriteBackWindowR64Kbx.cpp @@ -0,0 +1,348 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::StorageTexture; +using Clock = std::chrono::steady_clock; + +constexpr std::size_t Block = 0x10000; +constexpr std::uint32_t Side = 32; +constexpr std::uint32_t Format8888Unorm = 56; +constexpr std::uint32_t TileR64KBX = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint8_t Before = 0xa5; +constexpr std::uint32_t Attempts = 96; +constexpr std::uint32_t Texel = 0x00ffffffu; +constexpr std::array LinkOffsets{0x0, 0x8}; +constexpr std::size_t TriggerOffset = 0x8000; +constexpr std::size_t ResultOffset = 0x10; + +alignas(256) constexpr std::array StoreCode{ + 0x7e02020c, 0x7e0402f2, 0x7e0602f2, 0x7e0802f2, 0x7e0a0280, 0xf0201f08, 0x00010200, 0xbf810000, +}; + +std::uint64_t AddressOf(const void* data) { + return static_cast(reinterpret_cast(data)); +} + +std::uint32_t Low(const volatile void* pointer) { + return static_cast(reinterpret_cast(pointer)); +} + +std::uint32_t High(const volatile void* pointer) { + return static_cast(static_cast(reinterpret_cast(pointer)) >> 32u); +} + +std::array TextureDescriptor(std::uint64_t address) { + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888Unorm << 20u) | (((Side - 1u) & 3u) << 30u), + ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), + 0xfacu | (TileR64KBX << 20u) | (Type2D << 28u), + 0u, 0u, 0u, 0u, + }; +} + +class GuestBlock { +public: + explicit GuestBlock(bool watched = false) : watched(watched) { + if (watched) { +#ifdef _WIN32 + bytes = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(Block, Block)); + if (bytes != nullptr) GuestArena::GuestArenaCommit_nid_postfix(bytes, Block, PAGE_READWRITE, Block); +#else + void* mapped = mmap(nullptr, 2 * Block, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + Block - 1) & ~std::uintptr_t{Block - 1}; + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned != begin + Block) munmap(reinterpret_cast(aligned + Block), begin + Block - aligned); + bytes = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(bytes, Block); + } +#endif + } else { +#ifdef _WIN32 + bytes = static_cast(VirtualAlloc(nullptr, Block, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + bytes = static_cast(std::aligned_alloc(Block, Block)); +#endif + } + Require(bytes != nullptr, "storage write-back window: cannot allocate a guest block"); + std::memset(bytes, Before, Block); + GuestAllocations::Mutation().Add(bytes, Block, true, true, true); + } + ~GuestBlock() { + GuestAllocations::Mutation().Remove(bytes); + if (watched) { +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(bytes, Block); + GuestArena::GuestArenaRelease_nid_postfix(bytes, Block); +#else + munmap(bytes, Block); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(bytes, Block); +#endif + return; + } +#ifdef _WIN32 + VirtualFree(bytes, 0, MEM_RELEASE); +#else + std::free(bytes); +#endif + } + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + std::uint8_t* Data() const { return bytes; } + std::uint64_t Address() const { return AddressOf(bytes); } + +private: + bool watched; + std::uint8_t* bytes = nullptr; +}; + +void StoreThenLabel(const GuestBlock& image, volatile std::uint32_t* label, std::uint32_t value) { + std::vector words; + const auto registers = [&](std::uint32_t first, std::span values) { + words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); + words.push_back(first); + words.insert(words.end(), values.begin(), values.end()); + }; + const std::array shape{Side, 1, 1}; + registers(0x207, shape); + const auto code = AddressOf(StoreCode.data()); + const std::array program{static_cast(code >> 8u), static_cast(code >> 40u)}; + registers(0x20c, program); + std::array userData{}; + const auto descriptor = TextureDescriptor(image.Address()); + std::copy(descriptor.begin(), descriptor.end(), userData.begin() + 4); + const std::array layout{(static_cast(userData.size()) << 1u) | 0x80u}; + registers(0x213, layout); + registers(0x240, userData); + words.insert(words.end(), {0xc0031500u, Side, 1, 1, 0x41u}); + words.insert(words.end(), {0xc0064900u, 0x514u, (1u << 29u) | (2u << 24u), Low(label), High(label), value, 0u, 0u}); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); +} + +bool AwaitLabel(const volatile std::uint32_t* label, std::uint32_t value) { + const auto deadline = Clock::now() + std::chrono::seconds(10); + while (*label != value) { + if (Clock::now() >= deadline) return false; + std::this_thread::yield(); + } + return true; +} + +std::uint64_t Word(const GuestBlock& block, std::size_t offset) { + std::uint64_t value = 0; + std::memcpy(&value, block.Data() + offset, sizeof(value)); + return value; +} + +std::uint64_t LinkValue(std::uint32_t attempt, std::size_t index) { + return 0x0000000280000000ull + (static_cast(attempt) << 16u) + index * 0x40u + 0x10u; +} + +constexpr std::uint64_t TexelPair = (static_cast(Texel) << 32u) | Texel; + +bool Pending(const GuestBlock& image) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + return StorageTexture::FindPending(image.Address(), Block) != nullptr; +} + +void Publish(const GuestBlock& image) { + std::array read{}; + AgcDriver::GuestMemory::Read(image.Address() + TriggerOffset, read); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + StorageTexture::FlushAllPending("storage write-back window fixture"); +} + +enum class Phase { BeforeTrigger, DuringTrigger, AfterTrigger }; + +const char* PhaseName(Phase phase) { + return phase == Phase::BeforeTrigger ? "before-trigger" : phase == Phase::DuringTrigger ? "during-trigger" : "after-trigger"; +} + +struct Tally { + std::array kept{}; + std::array lost{}; + std::uint32_t notPending = 0; + std::uint32_t publishedOverLinks = 0; + std::uint32_t wonInsideStore = 0; +}; + +bool Control(GuestBlock& image, GuestBlock& labels) { + auto* label = reinterpret_cast(labels.Data()); + *label = 0; + std::memset(image.Data(), Before, Block); + StoreThenLabel(image, label, 1); + Require(AwaitLabel(label, 1), "storage write-back window: the control label never arrived"); + const bool pending = Pending(image); + Publish(image); + bool covered = true; + for (const auto offset : LinkOffsets) covered = covered && Word(image, offset) == TexelPair; + std::printf("[control] pendingAfterLabel=%d link0=0x%016llx link1=0x%016llx coveredByTexels=%d\n", pending, + static_cast(Word(image, LinkOffsets[0])), static_cast(Word(image, LinkOffsets[1])), covered); + return covered; +} + +void Attempt(GuestBlock& image, GuestBlock& labels, std::uint32_t attempt, std::chrono::nanoseconds delay, Tally& tally) { + auto* label = reinterpret_cast(labels.Data()); + const auto value = 100 + attempt; + *label = 0; + std::memset(image.Data(), Before, Block); + StoreThenLabel(image, label, value); + Require(AwaitLabel(label, value), "storage write-back window: a label never arrived"); + if (!Pending(image)) ++tally.notPending; + std::atomic ready{0}; + std::atomic go{false}; + Clock::time_point triggerStart, triggerEnd, linksStored; + const auto triggerDelay = delay < std::chrono::nanoseconds::zero() ? -delay : std::chrono::nanoseconds::zero(); + const auto storeDelay = delay < std::chrono::nanoseconds::zero() ? std::chrono::nanoseconds::zero() : delay; + std::thread driver([&] { + ready.fetch_add(1); + while (!go.load(std::memory_order_acquire)) {} + const auto until = Clock::now() + triggerDelay; + while (Clock::now() < until) {} + triggerStart = Clock::now(); + std::array read{}; + AgcDriver::GuestMemory::Read(image.Address() + TriggerOffset, read); + triggerEnd = Clock::now(); + }); + std::thread allocator([&] { + ready.fetch_add(1); + while (!go.load(std::memory_order_acquire)) {} + const auto until = Clock::now() + storeDelay; + while (Clock::now() < until) {} + for (std::size_t index = 0; index < LinkOffsets.size(); ++index) { + const auto link = LinkValue(attempt, index); + std::memcpy(image.Data() + LinkOffsets[index], &link, sizeof(link)); + } + linksStored = Clock::now(); + }); + const auto wonBefore = StorageTexture::CpuWonStores(); + while (ready.load() != 2) {} + go.store(true, std::memory_order_release); + driver.join(); + allocator.join(); + Publish(image); + const auto phase = linksStored < triggerStart ? Phase::BeforeTrigger : linksStored <= triggerEnd ? Phase::DuringTrigger : Phase::AfterTrigger; + bool kept = true; + bool texels = true; + for (std::size_t index = 0; index < LinkOffsets.size(); ++index) { + const auto observed = Word(image, LinkOffsets[index]); + kept = kept && observed == LinkValue(attempt, index); + texels = texels && observed == TexelPair; + } + auto& counts = kept ? tally.kept : tally.lost; + ++counts[static_cast(phase)]; + const bool resultsAround = Word(image, ResultOffset) == TexelPair; + const bool won = StorageTexture::CpuWonStores() != wonBefore; + if (kept && won) ++tally.wonInsideStore; + if (!kept && texels) ++tally.publishedOverLinks; + const auto micros = [](Clock::duration duration) { return std::chrono::duration(duration).count(); }; + std::printf("[attempt %u] delay=%.1fus trigger=[0, %.1fus] linksStoredAt=%.1fus phase=%s link0=0x%016llx link1=0x%016llx resultsAround=%d cpuWonStore=%d result=%s\n", attempt + 1, micros(delay), + micros(triggerEnd - triggerStart), micros(linksStored - triggerStart), PhaseName(phase), static_cast(Word(image, LinkOffsets[0])), + static_cast(Word(image, LinkOffsets[1])), resultsAround, won, kept ? "KEPT" : texels ? "LOST_TO_TEXELS" : "LOST"); +} + +enum class Verdict { Pass, Skip, Fail }; + +Verdict Run(bool watchedMemory) { + GuestBlock image(watchedMemory); + GuestBlock labels; + const char* kind = watchedMemory ? "watched" : "compared"; + const auto resource = AgcDriver::Graphics::DecodeTextureResource(TextureDescriptor(image.Address())); + Require(resource.width == Side && resource.height == Side && resource.format == Format8888Unorm && resource.tileMode == AgcDriver::Graphics::TextureTileMode::kR64KBX && + AgcDriver::Graphics::DescribeSurface(resource).guestBytes == Block, + "storage write-back window: the descriptor is not a 64 KiB 32x32 RGBA8 R64KBX surface"); + const bool watched = AgcDriver::GuestMemory::Watched(image.Address(), Block); + std::printf("[write-back window #5934] memory=%s surface=32x32 RGBA8 R64KBX footprint=0x%zx watched=%d hostImport=%s attempts=%u\n", kind, Block, watched, + std::getenv("APS5_NO_HOST_IMPORT") != nullptr ? "disabled" : "allowed", Attempts); + if (watched != watchedMemory) { + std::printf("[write-back window #5934] memory=%s SKIP: the write watch does not cover the surface as asked\n", kind); + return Verdict::Skip; + } + if (!Control(image, labels)) { + std::printf("[write-back window #5934] memory=%s SKIP: the published result does not cover the link offsets\n", kind); + return Verdict::Skip; + } + std::mt19937 random(5934); + Tally tally; + for (std::uint32_t attempt = 0; attempt < Attempts; ++attempt) { + const auto delay = std::chrono::nanoseconds(std::uniform_int_distribution(-300'000, 1'300'000)(random)); + Attempt(image, labels, attempt, delay, tally); + } + std::printf("[write-back window #5934] memory=%s summary: kept before/during/after=%u/%u/%u lost before/during/after=%u/%u/%u lostToTexels=%u wonInsideStore=%u notPendingAtLabel=%u\n", kind, + tally.kept[0], tally.kept[1], tally.kept[2], tally.lost[0], tally.lost[1], tally.lost[2], tally.publishedOverLinks, tally.wonInsideStore, tally.notPending); + const auto lost = tally.lost[0] + tally.lost[1] + tally.lost[2]; + if (lost != 0) { + std::printf("[write-back window #5934] memory=%s FAIL: %u of %u attempts lost free-list links the CPU stored after the guest observed the dispatch complete\n", kind, lost, Attempts); + return Verdict::Fail; + } + if (tally.notPending == Attempts) { + std::printf("[write-back window #5934] memory=%s PASS: every result reached memory before its label, so no CPU store could follow the label into a write-back\n", kind); + return Verdict::Pass; + } + if (tally.kept[0] == 0 || tally.wonInsideStore == 0) { + std::printf("[write-back window #5934] memory=%s SKIP: no CPU store landed before the trigger, or none after the store chose its blocks\n", kind); + return Verdict::Skip; + } + std::printf("[write-back window #5934] memory=%s PASS: every CPU store survived, including %u made after the write-back chose its blocks\n", kind, tally.wonInsideStore); + return Verdict::Pass; +} + +} + +int main() { + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + struct ShutdownGuard { + ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} } + } shutdown; + std::array verdicts{Run(false), Run(true)}; + const auto count = [&](Verdict verdict) { return std::count(verdicts.begin(), verdicts.end(), verdict); }; + std::printf("[write-back window #5934] PASS=%td SKIP=%td FAIL=%td\n", count(Verdict::Pass), count(Verdict::Skip), count(Verdict::Fail)); + if (count(Verdict::Fail) != 0) return 1; + return count(Verdict::Pass) != 0 ? 0 : VulkanTestSkipped; + } catch (const std::exception& error) { + std::cerr << "[write-back window #5934] FAIL: " << error.what() << '\n'; + return 1; + } +}