From 7b6e398c6ad2c3913fa5a3460ee5062cd93f3dde Mon Sep 17 00:00:00 2001 From: Elaine Bell Date: Thu, 8 Oct 2026 23:08:49 +0000 Subject: [PATCH 1/4] fix(agc): preserve GPU edge results after unchanged CPU writes Merge partial storage-image edges against bounded immutable baselines while preserving real CPU changes, newer driver stores, and allocation ownership. Cover deferred uploads, clears, copies, aliases, fallback, and writeback with runtime regressions. --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 24 + .../Execution/include/GuestMemory.hpp | 2 + .../Execution/src/GuestMemory.cpp | 70 ++ .../Graphics/include/Texture.hpp | 27 +- .../libSceAgcDriver/Graphics/src/Texture.cpp | 249 ++++++- .../prx/libSceAgcDriver/tests/Recorder.cpp | 152 +---- .../libSceAgcDriver/tests/RecorderDevice.hpp | 172 +++++ .../tests/StorageEdgeWriteBack.cpp | 606 ++++++++++++++++++ .../tests/StorageEdgeWriteWatch.cpp | 91 +++ .../tests/StorageEdgeWriteWatch.hpp | 9 + 10 files changed, 1233 insertions(+), 169 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/RecorderDevice.hpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteWatch.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteWatch.hpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 775e5901edf..5fb6d8f518f 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -463,6 +463,30 @@ target_link_libraries(agc_driver_recorder_tests PRIVATE set_target_properties(agc_driver_recorder_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_driver_recorder_tests) +add_test_executable(agc_storage_edge_writeback_tests EXCLUDE_FROM_ALL tests/StorageEdgeWriteBack.cpp) +target_include_directories(agc_storage_edge_writeback_tests PRIVATE + ${LIBS_INCLUDE_DIR} + ${CMAKE_SOURCE_DIR}/core/shader/recompiler + ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include +) +target_link_libraries(agc_storage_edge_writeback_tests PRIVATE libSceAgcDriver SDL2-static libc) +set_target_properties(agc_storage_edge_writeback_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_storage_edge_writeback_tests) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_sources(agc_storage_edge_writeback_tests PRIVATE tests/StorageEdgeWriteWatch.cpp) + target_compile_definitions(agc_storage_edge_writeback_tests PRIVATE ANYPS5_TEST_WRITE_WATCH=1) + set_target_properties(agc_storage_edge_writeback_tests PROPERTIES ENABLE_EXPORTS ON) +endif() +if(BUILD_TESTING) + set_target_properties(agc_storage_edge_writeback_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_storage_edge_writeback COMMAND agc_storage_edge_writeback_tests) + set_tests_properties(agc_storage_edge_writeback PROPERTIES SKIP_RETURN_CODE 77) + if(WIN32) + set_tests_properties(agc_storage_edge_writeback PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_recorder_kept_bytes_tests EXCLUDE_FROM_ALL tests/RecorderKeptBytes.cpp) target_include_directories(agc_recorder_kept_bytes_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) target_link_libraries(agc_recorder_kept_bytes_tests PRIVATE libSceAgcDriver libc) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index 966fb6c0469..0ada4a69932 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -45,6 +45,7 @@ Compare CompareMapped(std::uint64_t address, std::span bytes); // re-read of a differing run). Compare CopyMapped(std::uint64_t address, std::span out); void WriteChangedCommitted(std::uint64_t address, std::span current, std::span original); +bool MergeChangedCommitted(std::uint64_t address, std::span current, std::span original, std::uint64_t generation); void Write(std::uint64_t address, std::span source, std::size_t alignment = 1); // Stores the parts of `current` that differ from `original` (the guest bytes the GPU started from), so // guest writes made meanwhile to untouched bytes survive. Compares in 256-byte blocks. @@ -76,6 +77,7 @@ bool UnchangedSinceAll(std::span queries); std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes); std::uint64_t StoreOwnBytes(std::uint64_t address, std::size_t bytes, const std::function& store); bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation); +bool DriverStoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation); // Collect epoch: within one epoch a range already collected is not walked again, CollectWrites // returns the current generation instead. A guest write landing between two collects of the same // epoch is seen by the next epoch, which is the ordering real hardware gives a CPU write made while a diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index ee0c8817058..022dc6898a7 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -768,6 +768,7 @@ struct WriteTracker { std::vector cpuBlocks; std::vector writtenBlocks; WriteWatchCoverage coverage; + std::vector driverBlocks; #else static constexpr std::size_t LeafBlocks = std::size_t{1} << 16; static constexpr std::size_t LeafCount = std::size_t{1} << 15; @@ -775,6 +776,7 @@ struct WriteTracker { std::array blocks{}; std::array cpuBlocks{}; std::array writtenBlocks{}; + std::array driverBlocks{}; }; std::vector> leaves; #endif @@ -812,6 +814,7 @@ struct WriteTracker { cpuBlocks.assign(size / WriteBlockBytes + 1, 0); writtenBlocks.assign(size / WriteBlockBytes + 1, 0); coverage.Initialize(base, size); + driverBlocks.assign(size / WriteBlockBytes + 1, 0); pages.resize(1u << 16); GuestArena::GuestArenaSetPrivateMappingObserver_nid_postfix(&watchPrivateMapping); #else @@ -919,6 +922,15 @@ struct WriteTracker { #endif } + std::uint32_t driverStampOf(std::uint64_t block) const { +#ifdef _WIN32 + return driverBlocks[block]; +#else + const auto& leaf = leaves[block / LeafBlocks]; + return leaf != nullptr ? leaf->driverBlocks[block % LeafBlocks] : 0; +#endif + } + void stamp(std::uint64_t block, std::uint32_t stampGeneration, StampKind kind) { const bool cpu = kind != StampKind::Driver; const bool written = kind != StampKind::ImportWindow; @@ -926,14 +938,37 @@ struct WriteTracker { blocks[block] = stampGeneration; if (cpu) cpuBlocks[block] = stampGeneration; if (written) writtenBlocks[block] = stampGeneration; + if (kind == StampKind::Driver) driverBlocks[block] = stampGeneration; #else auto& leaf = leaves[block / LeafBlocks]; if (leaf == nullptr) leaf = std::make_unique(); leaf->blocks[block % LeafBlocks] = stampGeneration; if (cpu) leaf->cpuBlocks[block % LeafBlocks] = stampGeneration; if (written) leaf->writtenBlocks[block % LeafBlocks] = stampGeneration; + if (kind == StampKind::Driver) leaf->driverBlocks[block % LeafBlocks] = stampGeneration; #endif } + bool driverStoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation) const { + if (bytes == 0) return false; + if (!watched || generation == 0 || bytes > std::numeric_limits::max() - address || !covers(address, bytes)) return true; + const auto end = address + bytes; + const auto first = blockOf(address); + const auto last = blockOf(end - 1); + for (auto block = first; block <= last; ++block) { + if (driverStampOf(block) <= generation) continue; + const auto found = driverPieces.find(block); + if (found == driverPieces.end()) return true; + const auto& entry = found->second; + if (entry.whole > generation || entry.dropped > generation) return true; + const auto begin = blockBegin(block); + const auto from = std::max(address, begin) - begin; + const auto to = std::min(end, begin + WriteBlockBytes) - begin; + for (const auto& piece : entry.pieces) { + if (piece.generation > generation && piece.begin < to && from < piece.end) return true; + } + } + return false; + } }; WriteTracker& Tracker() { @@ -1305,6 +1340,13 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati return false; } +bool DriverStoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation) { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + tracker.initialize(); + return tracker.driverStoredOver(address, bytes, generation); +} + std::uint64_t TrackerGeneration() { auto& tracker = Tracker(); // Under the mutex: a collect bumps the generation before its walk, and a walk still running @@ -1797,6 +1839,34 @@ void WriteChangedCommitted(std::uint64_t address, std::span cur } } +bool MergeChangedCommitted(std::uint64_t address, std::span current, std::span original, std::uint64_t generation) { + require(current.size() == original.size(), "write-back snapshot sizes differ"); + if (current.empty()) return true; + const ReadSiteScope site(ReadSite::Store); + FlushGpuWrites(address, current.size()); + auto* destination = reinterpret_cast(address); + if (!Accessible(destination, current.size(), true)) return false; + const TimedAccess timed(CounterChangedWrite, current.size()); + bool merged = false; + storeOwn(address, current.size(), [&] { + if (Tracker().driverStoredOver(address, current.size(), generation)) return std::pair{0, 0}; + CheckRange(destination, current.size(), 1, true); + merged = true; + std::size_t firstChanged = current.size(); + std::size_t lastChanged = 0; + for (std::size_t byte = 0; byte < current.size(); ++byte) { + if (current[byte] == original[byte]) continue; + auto expected = original[byte]; + if (std::atomic_ref(destination[byte]).compare_exchange_strong(expected, current[byte], std::memory_order_relaxed)) { + firstChanged = std::min(firstChanged, byte); + lastChanged = byte + 1; + } + } + return firstChanged < lastChanged ? std::pair{address + firstChanged, address + lastChanged} : std::pair{0, 0}; + }); + return merged; +} + void WriteChanged(std::uint64_t address, std::span current, std::span original) { require(current.size() == original.size(), "write-back snapshot sizes differ"); if (current.empty()) return; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 0ce33ebbe2c..93a59f0de39 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -18,10 +18,15 @@ #include #include +namespace GuestAllocations { +struct Range; +} + namespace AgcDriver::Graphics { class ResidentColor; class CommandBatch; +class Buffer; class StorageTexture; struct HostImport; @@ -304,6 +309,25 @@ class StorageTexture : public std::enable_shared_from_this { // APS5_BLOCK_WRITEBACK_EACH=1 stores the touched units only). void writeBack(std::uint64_t address, std::size_t bytes); void writeBackLayers(const std::vector& layers); + struct EdgeSnapshot { + std::uint64_t begin; + std::uint64_t end; + std::shared_ptr bytes; + std::weak_ptr allocation; + }; + struct EdgeMerge { + EdgeSnapshot baseline; + std::shared_ptr result; + std::uint64_t generation; + }; + const EdgeSnapshot* edgeSnapshot(std::uint64_t begin, std::uint64_t end) const; + void prepareEdgeCapture(std::uint64_t begin, std::uint64_t end); + void captureEdges(VkCommandBuffer commands, const HostImport& import, VkBuffer source, std::span copies, std::span> ranges); + void captureCpuEdges(std::span> ranges); + void captureCpuEdges(std::span> ranges, std::span source, std::span> allocations); + void invalidateEdges(std::uint64_t begin, std::uint64_t end); + void finishEdgeMerges(std::span merges); + static std::vector> withoutEdgeMerges(std::span> ranges, std::span merges); // Tracked units as 64 KiB write-stamp blocks (`blockUnits`: a thin tiled surface at a 64 KiB // aligned base; APS5_NO_BLOCK_TRACKING=1 tracks array layers as above instead): a fill of one // layer, a CPU write or another image's store then costs the blocks it touched, moved through @@ -354,7 +378,7 @@ class StorageTexture : public std::enable_shared_from_this { // The write-back retiles into the import's unit shadow where a slab takes the piece // (`shadowed` receives those guest ranges, `imported` the pieces written to the import). std::uint64_t uploadWindows(const HostImport& import, std::span> runs, bool discard = false); - std::uint64_t writeBackWindows(const HostImport& import, std::span> keep, std::uint64_t firstStored, std::uint64_t lastStored, std::vector& shadowed, std::vector>& imported); + std::uint64_t writeBackWindows(const HostImport& import, std::span> keep, std::uint64_t firstStored, std::uint64_t lastStored, std::vector& shadowed, std::vector>& imported, std::span merges); // Per-layer validity (thin array surfaces whose layers are 64 KiB-aligned guest slices; every // other surface is one tracked layer): a layer's generation is the write generation its guest // bytes were last known to match the image at, and a pending layer holds results (a clear, a @@ -425,6 +449,7 @@ class StorageTexture : public std::enable_shared_from_this { std::uint64_t sliceLinearBytes = 0; SurfaceGeometry geometry; std::vector original; + std::vector edgeSnapshots; // DCC keys the image content was uploaded under: a fast-cleared surface starts as its clear value. DccKeys uploadedKeys = DccKeys::Uncompressed; mutable DccKeys filledKeys = DccKeys::Uncompressed; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 86c3338ebaa..a111bd0ba8d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -703,6 +703,9 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, trackedLayerBytes = blockUnits ? 65536 : guestBytes / trackedLayers; layerGeneration.assign(trackedLayers, 0); layerPending.assign(trackedLayers, false); + const auto surfaceEnd = descriptor.baseAddress + guestBytes; + if (descriptor.baseAddress % 65536 != 0) edgeSnapshots.push_back({descriptor.baseAddress, std::min(surfaceEnd, (descriptor.baseAddress + 65535) & ~std::uint64_t{65535}), {}, {}}); + if (surfaceEnd % 65536 != 0 && (edgeSnapshots.empty() || edgeSnapshots.back().end < surfaceEnd)) edgeSnapshots.push_back({std::max(descriptor.baseAddress, surfaceEnd & ~std::uint64_t{65535}), surfaceEnd, {}, {}}); sliceLinearBytes = geometry.sliceLinearBytes; const auto linearBytes = sliceLinearBytes * arrayLayers; @@ -1210,11 +1213,13 @@ bool StorageTexture::Refresh() { auto alias = pendingAlias(); if (alias != nullptr && !(ProvedClearKeys(descriptor, guestBytes, keyProof) == DccKeys::Uncompressed && HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) != nullptr)) alias = nullptr; if (alias != nullptr) { + GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes)); std::vector aliasStamped(trackedLayers); if (!GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), alias->layerGeneration, aliasStamped)) alias = nullptr; for (std::uint32_t unit = 0; alias != nullptr && unit < trackedLayers; ++unit) { if (!alias->layerPending[unit]) continue; - if (alias->layerGeneration[unit] == 0 || aliasStamped[unit] == GuestMemory::BlockMaybeWritten || (aliasStamped[unit] == GuestMemory::BlockUnchanged && layerPending[unit])) { + const bool mergeable = aliasStamped[unit] == GuestMemory::BlockWritten && alias->edgeSnapshot(alias->layerBegin(unit), alias->layerBegin(unit) + alias->layerBytes(unit)) != nullptr; + if (alias->layerGeneration[unit] == 0 || aliasStamped[unit] == GuestMemory::BlockMaybeWritten || mergeable || (aliasStamped[unit] == GuestMemory::BlockUnchanged && layerPending[unit])) { alias = nullptr; break; } @@ -1364,6 +1369,7 @@ bool StorageTexture::Refresh() { const auto begin = layerBegin(unit); const auto bytes = layerBytes(unit); const bool edge = begin % 65536 != 0 || bytes != 65536; + if (edge && edgeSnapshot(begin, begin + bytes) != nullptr) return false; return !edge || GuestMemory::StoredOver(begin, static_cast(bytes), layerGeneration[unit]); }; // A clear code -> uncompressed flip on an image with results pending: an unstamped pending @@ -1396,6 +1402,7 @@ bool StorageTexture::Refresh() { if (blockUnits && droppable(layer)) { CaptureTrace::Log("drop-unit image=%llx unit=%u generation=%llu stamped=%d keys=%d", static_cast(descriptor.baseAddress), layer, static_cast(layerGeneration[layer]), layer < stampedBlocks.size() && stampedBlocks[layer] != 0, static_cast(keys)); layerPending[layer] = false; + invalidateEdges(layerBegin(layer), layerBegin(layer) + layerBytes(layer)); dropped = true; continue; } @@ -1477,6 +1484,123 @@ void traceKeyStore(const char* path, const GuestTextureResource& descriptor, std } +const StorageTexture::EdgeSnapshot* StorageTexture::edgeSnapshot(std::uint64_t begin, std::uint64_t end) const { + for (const auto& edge : edgeSnapshots) { + if (edge.bytes != nullptr && edge.begin <= begin && end <= edge.end && !edge.allocation.expired()) return &edge; + } + return nullptr; +} + +void StorageTexture::invalidateEdges(std::uint64_t begin, std::uint64_t end) { + for (auto& edge : edgeSnapshots) { + if (edge.begin < end && begin < edge.end) edge.bytes.reset(); + } +} + +void StorageTexture::prepareEdgeCapture(std::uint64_t begin, std::uint64_t end) { + invalidateEdges(begin, end); + if (std::none_of(edgeSnapshots.begin(), edgeSnapshots.end(), [&](const EdgeSnapshot& edge) { return begin <= edge.begin && edge.end <= end && GuestMemory::Watched(edge.begin, static_cast(edge.end - edge.begin)); })) return; + auto* recorder = Recorder::Active(); + if (recorder != nullptr) { + for (const auto& edge : edgeSnapshots) { + if (edge.begin < begin || end < edge.end || !GuestMemory::Watched(edge.begin, static_cast(edge.end - edge.begin))) continue; + const auto bytes = static_cast(edge.end - edge.begin); + recorder->FlushStoresOverlapping(edge.begin, bytes); + if (recorder->PendingWriteOverlaps(edge.begin, bytes)) recorder->SyncThrough(edge.begin, bytes); + } + } + const std::pair captured{begin, end}; + captureCpuEdges(std::span(&captured, 1)); +} + +void StorageTexture::captureEdges(VkCommandBuffer commands, const HostImport& import, VkBuffer source, std::span copies, std::span> ranges) { + bool barrier = false; + auto* recorder = Recorder::Active(); + for (auto& edge : edgeSnapshots) { + if (std::none_of(ranges.begin(), ranges.end(), [&](const auto& range) { return range.first <= edge.begin && edge.end <= range.second; })) continue; + edge.bytes.reset(); + if (!GuestMemory::Watched(edge.begin, static_cast(edge.end - edge.begin))) continue; + auto snapshot = std::make_shared(context, static_cast(edge.end - edge.begin), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + if (!barrier) { + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + barrier = true; + } + if (recorder != nullptr) recorder->Keep(snapshot, static_cast(edge.end - edge.begin)); + for (const auto& copy : copies) { + const auto begin = import.base + copy.dstOffset; + const auto from = std::max(begin, edge.begin); + const auto to = std::min(begin + copy.size, edge.end); + if (from < to) CopyBuffer(context, commands, source, copy.srcOffset + from - begin, snapshot->Handle(), from - edge.begin, to - from); + } + edge.bytes = std::move(snapshot); + edge.allocation = import.range; + } + if (barrier) RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); +} + +void StorageTexture::captureCpuEdges(std::span> ranges) { + if (edgeSnapshots.empty()) return; + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + captureCpuEdges(ranges, {}, lease); +} + +void StorageTexture::captureCpuEdges(std::span> ranges, std::span source, std::span> allocations) { + for (auto& edge : edgeSnapshots) { + if (std::none_of(ranges.begin(), ranges.end(), [&](const auto& range) { return range.first <= edge.begin && edge.end <= range.second; })) continue; + edge.bytes.reset(); + if (!GuestMemory::Watched(edge.begin, static_cast(edge.end - edge.begin))) continue; + const auto owner = std::find_if(allocations.begin(), allocations.end(), [&](const auto& range) { return range->readable && range->writable && range->address <= edge.begin && edge.end <= range->address + range->bytes; }); + if (owner == allocations.end()) continue; + auto snapshot = std::make_shared(context, static_cast(edge.end - edge.begin), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + if (source.empty()) { + if (GuestMemory::CopyMapped(edge.begin, snapshot->Bytes()) != GuestMemory::Compare::Equal) continue; + } else { + const auto bytes = source.subspan(static_cast(edge.begin - descriptor.baseAddress), static_cast(edge.end - edge.begin)); + std::memcpy(snapshot->Bytes().data(), bytes.data(), bytes.size()); + } + edge.bytes = std::move(snapshot); + edge.allocation = *owner; + } +} + +std::vector> StorageTexture::withoutEdgeMerges(std::span> ranges, std::span merges) { + std::vector> result; + for (const auto& [begin, end] : ranges) { + auto cursor = begin; + for (const auto& merge : merges) { + if (merge.baseline.end <= cursor || end <= merge.baseline.begin) continue; + if (cursor < merge.baseline.begin) result.emplace_back(cursor, merge.baseline.begin); + cursor = std::max(cursor, merge.baseline.end); + } + if (cursor < end) result.emplace_back(cursor, end); + } + return result; +} + +void StorageTexture::finishEdgeMerges(std::span merges) { + if (merges.empty()) return; + struct Exempt { + const StorageTexture* previous; + ~Exempt() { refreshing = previous; } + } exempt{std::exchange(refreshing, this)}; + if (auto* recorder = Recorder::Active()) recorder->Sync(); + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + for (const auto& merge : merges) { + const auto& edge = merge.baseline; + const auto owner = edge.allocation.lock(); + if (owner == nullptr || GuestMemory::DriverStoredOver(edge.begin, static_cast(edge.end - edge.begin), merge.generation) || std::none_of(lease.begin(), lease.end(), [&](const auto& range) { return range == owner && range->readable && range->writable && range->address <= edge.begin && edge.end <= range->address + range->bytes; })) { + invalidateEdges(edge.begin, edge.end); + continue; + } + edge.bytes->Invalidate(); + merge.result->Invalidate(); + const auto baseline = edge.bytes->Bytes(); + const auto result = merge.result->Bytes(); + GuestMemory::MergeChangedCommitted(edge.begin, result, baseline, merge.generation); + invalidateEdges(edge.begin, edge.end); + } +} + void StorageTexture::upload(const std::vector* layers) { CaptureTrace::Log("upload image=%llx bytes=%llu generation=%llu reason=%s partial=%d", static_cast(descriptor.baseAddress), static_cast(guestBytes), static_cast(generation), uploadReason, layers != nullptr); const bool profile = LookupOutcomes::Profiled(); @@ -1487,7 +1611,7 @@ void StorageTexture::upload(const std::vector* layers) { // A layer selection applies to the direct path alone (the others upload everything); it names // array layers, which the tracked layers are when there are several. if (layers != nullptr && ((!blockUnits && trackedLayers != arrayLayers) || std::all_of(layers->begin(), layers->end(), [](bool selected) { return selected; }))) layers = nullptr; - const auto now = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); + auto now = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); const auto stampLayers = [&](bool selectedOnly) { for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { if (!selectedOnly || layers == nullptr || (*layers)[layer]) layerGeneration[layer] = now; @@ -1498,6 +1622,8 @@ void StorageTexture::upload(const std::vector* layers) { filledKeys = DccKeys::Uncompressed; VkClearColorValue clearValue{}; if (IsDccClear(uploadedKeys) && ClearColorFor(storageFormat, uploadedKeys, clearValue)) { + prepareEdgeCapture(descriptor.baseAddress, descriptor.baseAddress + guestBytes); + if (std::any_of(edgeSnapshots.begin(), edgeSnapshots.end(), [](const EdgeSnapshot& edge) { return edge.bytes != nullptr; })) now = GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes)); // A fast-cleared surface is cleared on the GPU; its texel memory is neither read nor filled. // The clear is recorded into the open batch like a direct upload (the image kept by it): a // batch of its own submitted the recorder's work first and waited for all of it, 25-40 ms @@ -1553,7 +1679,20 @@ void StorageTexture::upload(const std::vector* layers) { // The surface lives in host-imported memory: the detiler reads it in place, no guest bytes // are copied, and write tracking alone validates the image (a change re-runs this). originalValid = false; - stampLayers(true); + const auto captureSelectedEdges = [&](std::span> ranges) { + bool prepared = false; + for (const auto& [begin, end] : ranges) { + if (std::none_of(edgeSnapshots.begin(), edgeSnapshots.end(), [&](const EdgeSnapshot& edge) { return begin <= edge.begin && edge.end <= end && GuestMemory::Watched(edge.begin, static_cast(edge.end - edge.begin)); })) continue; + prepareEdgeCapture(begin, end); + prepared = true; + } + if (prepared) { + import = HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)); + Require(import != nullptr, "storage upload lost its host import while preparing edge snapshots"); + now = GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes)); + } + stampLayers(true); + }; // A whole-surface upload of a block-unit image goes through the windows too when a unit // shadow holds part of the surface (the detile then reads the slabs; a new image has no // layout yet); a partial one always does. @@ -1568,6 +1707,9 @@ void StorageTexture::upload(const std::vector* layers) { forgetBorrowed(0, trackedLayers); runs.emplace_back(0, guestBytes); } + std::vector> captured; + for (const auto& [begin, end] : runs) captured.emplace_back(descriptor.baseAddress + begin, descriptor.baseAddress + end); + captureSelectedEdges(captured); const auto uploadedBytes = uploadWindows(*import, runs, layers == nullptr && version == 0); countStorageUpload(1, uploadedBytes); if (layers != nullptr) { @@ -1588,6 +1730,11 @@ void StorageTexture::upload(const std::vector* layers) { if ((*layers)[layer]) uploadedBytes += trackedLayerBytes; } } + std::vector> captured; + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (layers == nullptr || (*layers)[layer]) captured.emplace_back(layerBegin(layer), layerBegin(layer) + layerBytes(layer)); + } + captureSelectedEdges(captured); auto linear = std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); @@ -1655,7 +1802,10 @@ void StorageTexture::upload(const std::vector* layers) { originalValid = true; forgetBorrowed(0, trackedLayers); stampLayers(false); + const auto uploadLease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); GuestMemory::ReadCommitted(descriptor.baseAddress, original); + const std::pair captured{descriptor.baseAddress, descriptor.baseAddress + guestBytes}; + captureCpuEdges(std::span(&captured, 1), original, uploadLease); { Buffer staging(context, original.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); if (uploadedKeys == DccKeys::Uncompressed) std::memcpy(staging.Bytes().data(), original.data(), original.size()); @@ -1989,7 +2139,7 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< return uploadedBytes; } -std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::span> keep, std::uint64_t firstStored, std::uint64_t lastStored, std::vector& shadowed, std::vector>& imported) { +std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::span> keep, std::uint64_t firstStored, std::uint64_t lastStored, std::vector& shadowed, std::vector>& imported, std::span merges) { const auto elementBytes = BytesPerElement(descriptor.format); std::vector> runs; std::uint64_t storedBytes = 0; @@ -1998,6 +2148,10 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp storedBytes += to - from; } if (runs.empty()) return 0; + const auto directRanges = withoutEdgeMerges(keep, merges); + for (const auto& edge : edgeSnapshots) { + if (std::any_of(keep.begin(), keep.end(), [&](const auto& range) { return range.first < edge.end && edge.begin < range.second; })) PublishShadow(edge.begin, static_cast(edge.end - edge.begin), PublishScope::PartialUnits, PublishReason::Upload); + } auto windows = sliceWindows(runs); // The linear rows and the tiled bytes of every window, each at a 256-byte position; the tail // levels of a layer retile into one scratch region (their block is shared). @@ -2032,8 +2186,13 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp std::vector seeds; const auto addPiece = [&](std::uint64_t scratchOffset, std::uint64_t guestBegin, std::uint64_t guestEnd) { while (guestBegin < guestEnd) { - const auto pieceEnd = UnitShadowEnabled() ? std::min(guestEnd, SlabBoundary(import, guestBegin)) : guestEnd; - const auto destination = ShadowDestinationFor(context, import, guestBegin, pieceEnd); + auto pieceEnd = UnitShadowEnabled() ? std::min(guestEnd, SlabBoundary(import, guestBegin)) : guestEnd; + for (const auto& snapshot : edgeSnapshots) { + if (guestBegin < snapshot.begin) pieceEnd = std::min(pieceEnd, snapshot.begin); + else if (guestBegin < snapshot.end) pieceEnd = std::min(pieceEnd, snapshot.end); + } + const bool edge = std::any_of(edgeSnapshots.begin(), edgeSnapshots.end(), [&](const EdgeSnapshot& snapshot) { return snapshot.begin < pieceEnd && guestBegin < snapshot.end; }); + const auto destination = edge ? std::optional{} : ShadowDestinationFor(context, import, guestBegin, pieceEnd); if (destination.has_value()) { if (slabPieces.empty() || slabPieces.back().slab != destination->slab) slabPieces.push_back({destination->slab, destination->pin, {}}); slabPieces.back().copies.push_back({scratchOffset, destination->offset, pieceEnd - guestBegin}); @@ -2055,9 +2214,9 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp const auto& window = windows[i]; // The tail levels follow each other, sharing the previous one's region. if (i != 0 && scratchPositions[i] == scratchPositions[i - 1]) continue; - for (const auto& [from, to] : runs) { - const auto begin = std::max(from, window.tiledBegin); - const auto end = std::min(to, window.tiledEnd); + for (const auto& [from, to] : directRanges) { + const auto begin = std::max(from - descriptor.baseAddress, window.tiledBegin); + const auto end = std::min(to - descriptor.baseAddress, window.tiledEnd); if (begin >= end) continue; addPiece(scratchPositions[i] + (begin - window.tiledBegin), descriptor.baseAddress + begin, descriptor.baseAddress + end); } @@ -2096,6 +2255,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); recorder->Keep(linear, linear->Size()); recorder->Keep(tiledScratch, tiledScratch->Size()); + for (const auto& merge : merges) recorder->Keep(merge.result, static_cast(merge.baseline.end - merge.baseline.begin)); for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab); for (const auto& slab : padding.slabs) recorder->Keep(slab); if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); @@ -2164,7 +2324,17 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp const auto copyBuffer = context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer"); for (const auto& pieces : slabPieces) copyBuffer(commands, tiledScratch->Handle(), pieces.slab->buffer, static_cast(pieces.copies.size()), pieces.copies.data()); if (!importCopies.empty()) copyBuffer(commands, tiledScratch->Handle(), import.buffer, static_cast(importCopies.size()), importCopies.data()); + for (std::size_t i = 0; i < windows.size(); ++i) { + if (i != 0 && scratchPositions[i] == scratchPositions[i - 1]) continue; + const auto& window = windows[i]; + for (const auto& merge : merges) { + const auto begin = std::max(merge.baseline.begin - descriptor.baseAddress, window.tiledBegin); + const auto end = std::min(merge.baseline.end - descriptor.baseAddress, window.tiledEnd); + if (begin < end) CopyBuffer(context, commands, tiledScratch->Handle(), scratchPositions[i] + begin - window.tiledBegin, merge.result->Handle(), descriptor.baseAddress + begin - merge.baseline.begin, end - begin); + } + } } + captureEdges(commands, import, tiledScratch->Handle(), importCopies, directRanges); VkImageMemoryBarrier backToGeneral = toSource; backToGeneral.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; backToGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; @@ -2184,6 +2354,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp // shadowed ones reach it through a publish, which notes its own. if (!imported.empty()) recorder->NotePendingWrites(imported); } + finishEdgeMerges(merges); partialWriteBacks.fetch_add(1, std::memory_order_relaxed); partialWriteBackBytes.fetch_add(storedBytes, std::memory_order_relaxed); return storedBytes; @@ -2243,6 +2414,7 @@ void StorageTexture::markLayersPending(std::uint32_t first, std::uint32_t count) borrowedUnits[unit] = false; if (source->version != borrowedVersion || !source->layerPending[unit]) continue; source->layerPending[unit] = false; + source->invalidateEdges(source->layerBegin(unit), source->layerBegin(unit) + source->layerBytes(unit)); unitsSuperseded.fetch_add(1, std::memory_order_relaxed); superseded = true; } @@ -2483,6 +2655,9 @@ bool StorageTexture::blocksKept(std::span> const auto end = std::min(accessEnd, texture->layerBegin(layer) + texture->layerBytes(layer)); if (end <= begin) continue; if (texture->layerGeneration[layer] == 0) return false; + for (const auto& edge : texture->edgeSnapshots) { + if (edge.begin < end && begin < edge.end && texture->edgeSnapshot(edge.begin, edge.end) != nullptr) return false; + } for (auto at = begin & ~(block - 1); at < end; at += block) { const auto from = std::max(at, begin); const auto to = std::min(at + block, end); @@ -2506,6 +2681,7 @@ void StorageTexture::ClassifyAccess(std::uint64_t address, std::size_t bytes, Ac if (bytes > ClassifyLimit) return; out.checked = true; GuestMemory::CollectWritesUncached(address, bytes); + std::lock_guard gpu(GuestMemory::GpuMutex()); out.allCpuWritten = blocksKept(overlapping, address, bytes); } @@ -2580,6 +2756,9 @@ bool StorageTexture::evictStale(std::uint64_t address, std::size_t bytes, std::s if (!selected(layer)) continue; const auto begin = layerBegin(layer); const auto end = begin + layerBytes(layer); + for (const auto& edge : edgeSnapshots) { + if (edge.begin < end && begin < edge.end && edgeSnapshot(edge.begin, edge.end) != nullptr) return false; + } for (auto at = begin & ~(block - 1); at < end; at += block) { const auto from = std::max(at, begin); if (!GuestMemory::WrittenSince(from, static_cast(std::min(at + block, end) - from), layerGeneration[layer])) return false; @@ -2588,7 +2767,10 @@ bool StorageTexture::evictStale(std::uint64_t address, std::size_t bytes, std::s // As the empty write-back leaves the layers: no results pending, the generation unchanged (the // stamps make their next Refresh re-upload), `original` no longer vouching for the bytes. for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { - if (selected(layer)) layerPending[layer] = false; + if (selected(layer)) { + layerPending[layer] = false; + invalidateEdges(layerBegin(layer), layerBegin(layer) + layerBytes(layer)); + } } originalValid = false; if (!anyLayerPending()) { @@ -2651,6 +2833,7 @@ std::size_t StorageTexture::DiscardPendingInside(std::uint64_t address, std::siz texture->dirty = false; texture->layerPending.assign(texture->trackedLayers, false); texture->originalValid = false; + texture->invalidateEdges(begin, begin + texture->guestBytes); it = pending.textures.erase(it); ++discarded; } else if (texture != exempt && texture->blockUnits && texture->overlaps(address, bytes)) { @@ -2662,6 +2845,7 @@ std::size_t StorageTexture::DiscardPendingInside(std::uint64_t address, std::siz const auto unitBegin = texture->layerBegin(unit); if (!texture->layerPending[unit] || unitBegin < address || unitBegin + texture->layerBytes(unit) > end) continue; texture->layerPending[unit] = false; + texture->invalidateEdges(unitBegin, unitBegin + texture->layerBytes(unit)); unitsDropped.fetch_add(1, std::memory_order_relaxed); dropped = true; } @@ -2843,6 +3027,7 @@ bool StorageTexture::clearByKeysFill(DccKeys keys, std::uint8_t key) { BumpPendingSerial(); } } + prepareEdgeCapture(descriptor.baseAddress, descriptor.baseAddress + guestBytes); const auto commands = recorder->Commands(); const auto timing = recorder->BeginGpuTiming(Recorder::CommandClass::DccClear); if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); @@ -2966,6 +3151,7 @@ bool StorageTexture::FillClear(std::span pattern, std::u for (auto* texture : live.textures) { if (texture == this || texture->released || !texture->overlaps(begin, bytes)) continue; texture->originalValid = false; + texture->invalidateEdges(begin, begin + bytes); if (texture->descriptor.dccAddress != 0) aliasedKeys = true; } } @@ -2979,6 +3165,7 @@ bool StorageTexture::FillClear(std::span pattern, std::u // below makes those units stale (nothing for a 64 KiB-multiple surface). PublishShadow(begin, bytes, PublishScope::PartialUnits, PublishReason::FillClear); GuestMemory::MarkWritten(begin, bytes); + prepareEdgeCapture(begin, begin + bytes); const auto commands = recorder->Commands(); const auto timing = recorder->BeginGpuTiming(Recorder::CommandClass::FillClear); Recorder::CountBarriers(Recorder::CommandClass::FillClear, 2); @@ -3092,10 +3279,14 @@ bool StorageTexture::CopyFrom(StorageTexture& source, const char*& refusal) { auto& live = Live(); std::lock_guard lock(live.mutex); for (auto* texture : live.textures) { - if (texture != this && !texture->released && texture->overlaps(descriptor.baseAddress, static_cast(guestBytes))) texture->originalValid = false; + if (texture != this && !texture->released && texture->overlaps(descriptor.baseAddress, static_cast(guestBytes))) { + texture->originalValid = false; + texture->invalidateEdges(descriptor.baseAddress, descriptor.baseAddress + guestBytes); + } } } GuestMemory::MarkWritten(descriptor.baseAddress, static_cast(guestBytes)); + prepareEdgeCapture(descriptor.baseAddress, descriptor.baseAddress + guestBytes); const auto commands = recorder->Commands(); const auto timing = recorder->BeginGpuTiming(Recorder::CommandClass::Copy); Recorder::CountBarriers(Recorder::CommandClass::Copy, 2); @@ -3312,7 +3503,14 @@ std::uint64_t StorageTexture::borrowUnits(StorageTexture& source, const std::vec borrowedVersion = source.version; source.lent = true; for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { - if (units[unit]) borrowedUnits[unit] = true; + if (units[unit]) { + borrowedUnits[unit] = true; + invalidateEdges(layerBegin(unit), layerBegin(unit) + layerBytes(unit)); + for (auto& edge : edgeSnapshots) { + if (edge.begin < layerBegin(unit) || layerBegin(unit) + layerBytes(unit) < edge.end) continue; + if (const auto* inherited = source.edgeSnapshot(edge.begin, edge.end)) edge = *inherited; + } + } } ++version; return bytes; @@ -3380,6 +3578,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { // may have reused the memory for something else entirely (see layerGeneration). A generation of // zero means no tracking, and the layer is stored whole. std::vector> keep; + std::vector merges; std::vector skippedLayer(trackedLayers, false); bool skippedAny = false; std::size_t skipped = 0; @@ -3424,7 +3623,12 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { skippedAny = true; skippedLayer[layer] = true; ++skipped; - continue; + const auto* baseline = edge ? edgeSnapshot(from, to) : nullptr; + if (baseline == nullptr || GuestMemory::DriverStoredOver(from, static_cast(to - from), layerGeneration[layer])) { + invalidateEdges(from, to); + continue; + } + merges.push_back({*baseline, std::make_shared(context, static_cast(to - from), VK_BUFFER_USAGE_TRANSFER_DST_BIT), layerGeneration[layer]}); } if (!keep.empty() && keep.back().second == from) keep.back().second = to; else keep.emplace_back(from, to); @@ -3480,7 +3684,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { if (blockUnits) { // The kept blocks alone pass through the retiler (windows of their slices). shadowImport = import; - const auto storedBytes = writeBackWindows(*import, keep, firstStored, lastStored, shadowed, imported); + const auto storedBytes = writeBackWindows(*import, keep, firstStored, lastStored, shadowed, imported, merges); countStorageWriteBack(storedBytes, true); if (profile) Profile().storageGpu += timer.lap(); originalValid = false; @@ -3488,7 +3692,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { if (!IsDccClear(filledKeys)) MarkDccUncompressed(context, descriptor.dccAddress, guestBytes, DccKeyCount(descriptor, guestBytes)); uploadedKeys = DccKeys::Uncompressed; for (const auto& [from, to] : keep) GuestMemory::MarkWritten(from, static_cast(to - from)); - settle(true); + settle(merges.empty()); ++Profile().storageDirectWriteBacks; return; } @@ -3501,6 +3705,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { // produced the image; nothing crosses to the CPU. auto linear = std::make_shared(context, static_cast(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); auto tiledScratch = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto directRanges = withoutEdgeMerges(keep, merges); std::vector copied; for (const auto& [from, to] : keep) copied.push_back({from - descriptor.baseAddress, to - descriptor.baseAddress, from - descriptor.baseAddress}); const auto padding = paddingSeeds(*import, std::move(copied)); @@ -3518,6 +3723,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); recorder->Keep(linear, linear->Size()); recorder->Keep(tiledScratch, tiledScratch->Size()); + for (const auto& merge : merges) recorder->Keep(merge.result, static_cast(merge.baseline.end - merge.baseline.begin)); for (const auto& slab : padding.slabs) recorder->Keep(slab); // The image itself must outlive the recorded retile: the cache may evict it right after. if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); @@ -3568,8 +3774,10 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { const auto scratchDone = WholeBufferBarrier(tiledScratch->Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &scratchDone, 0, nullptr); std::vector copies; - for (const auto& [from, to] : keep) copies.push_back({from - descriptor.baseAddress, importOffset + (from - descriptor.baseAddress), to - from}); + for (const auto& [from, to] : directRanges) copies.push_back({from - descriptor.baseAddress, importOffset + (from - descriptor.baseAddress), to - from}); if (!copies.empty()) context.Function("vkCmdCopyBuffer")(commands, tiledScratch->Handle(), import->buffer, static_cast(copies.size()), copies.data()); + for (const auto& merge : merges) CopyBuffer(context, commands, tiledScratch->Handle(), merge.baseline.begin - descriptor.baseAddress, merge.result->Handle(), 0, merge.baseline.end - merge.baseline.begin); + captureEdges(commands, *import, tiledScratch->Handle(), copies, directRanges); } VkImageMemoryBarrier backToGeneral = toSource; backToGeneral.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; @@ -3585,6 +3793,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { recorder->MarkShaderReadsCovered(); recorder->NotePendingWrite(firstStored, static_cast(lastStored - firstStored)); } + finishEdgeMerges(merges); countStorageWriteBack(storedBytes, true); if (profile) Profile().storageGpu += timer.lap(); // The guest bytes now differ from `original`; other caches of the range see the write. @@ -3599,7 +3808,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { // The walk covers this surface's pages only: an adjacent image's later CPU write is stamped // newer than this value when its own range is collected. No CPU wrote the pages here // (MarkWritten made the only stamps), so the memoized collect is exact. - settle(true); + settle(merges.empty()); ++Profile().storageDirectWriteBacks; return; } @@ -3694,12 +3903,16 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { // Store only the kept ranges' blocks that changed, so concurrent CPU writes to untouched texels // survive; `original` follows for the stored layers (the others' guest bytes are unchanged). const auto current = host.Bytes(); - for (const auto& [from, to] : keep) { + const auto directRanges = withoutEdgeMerges(keep, merges); + for (const auto& [from, to] : directRanges) { const auto offset = static_cast(from - descriptor.baseAddress); const auto length = static_cast(to - from); GuestMemory::WriteChangedCommitted(from, current.subspan(offset, length), std::span(original).subspan(offset, length)); std::memcpy(original.data() + offset, current.data() + offset, length); } + for (const auto& merge : merges) std::memcpy(merge.result->Bytes().data(), current.data() + (merge.baseline.begin - descriptor.baseAddress), static_cast(merge.baseline.end - merge.baseline.begin)); + finishEdgeMerges(merges); + captureCpuEdges(directRanges); // The texels now hold the whole image, so later reads must see them rather than a fast clear // (the keys may be host-imported although the texels were not: then a recorded fill, else a CPU store). traceKeyStore("cpu write-back", descriptor, guestBytes); diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index a8c50a234aa..0223d33c768 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -15,6 +15,7 @@ #include "prx/libc/include/GuestArena.hpp" #include "prx/libc/include/GuestWriteWatch.hpp" #include "ResidentPresent.hpp" +#include "RecorderDevice.hpp" #include "SampleLod_spv.h" #include "SampleArray_spv.h" #include @@ -76,156 +77,7 @@ void ReleaseWatched(void* block, std::size_t bytes) { // A compute-capable device with host imports (VK_EXT_external_memory_host) when the host offers // them, as the driver creates its own; the recorder's batches need a real queue and real fences. -class Device { -public: - Device() { -#ifdef _WIN32 - library = SDL_LoadObject("vulkan-1.dll"); -#else - library = SDL_LoadObject("libvulkan.so.1"); -#endif - Require(library != nullptr, "cannot load Vulkan"); - try { - instanceProc = reinterpret_cast(SDL_LoadFunction(library, "vkGetInstanceProcAddr")); - Require(instanceProc != nullptr, "missing Vulkan instance resolver"); - VkApplicationInfo application{VK_STRUCTURE_TYPE_APPLICATION_INFO}; - application.apiVersion = VK_API_VERSION_1_1; - VkInstanceCreateInfo info{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; - info.pApplicationInfo = &application; - Check(function("vkCreateInstance")(&info, nullptr, &instance), "vkCreateInstance"); - std::uint32_t count = 0; - const auto enumerate = function("vkEnumeratePhysicalDevices"); - Check(enumerate(instance, &count, nullptr), "vkEnumeratePhysicalDevices"); - Require(count != 0, "no Vulkan device"); - std::vector devices(count); - Check(enumerate(instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); - const auto rankDeviceType = [](VkPhysicalDeviceType type) { - switch (type) { - case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return 3; - case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return 2; - case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return 1; - default: return 0; - } - }; - const auto physicalProperties = function("vkGetPhysicalDeviceProperties"); - int selectedRank = -1; - for (const auto physical : devices) { - VkPhysicalDeviceProperties candidate{}; - physicalProperties(physical, &candidate); - if (candidate.apiVersion < VK_API_VERSION_1_1) continue; - const int rank = rankDeviceType(candidate.deviceType); - if (rank <= selectedRank) continue; - context.physical = physical; - selectedRank = rank; - } - Require(context.physical != VK_NULL_HANDLE, "no Vulkan 1.1 device"); - const auto extensions = function("vkEnumerateDeviceExtensionProperties"); - Check(extensions(context.physical, nullptr, &count, nullptr), "vkEnumerateDeviceExtensionProperties"); - std::vector available(count); - Check(extensions(context.physical, nullptr, &count, available.data()), "vkEnumerateDeviceExtensionProperties"); - const auto hasExtension = [&](const char* name) { - for (const auto& extension : available) { - if (std::strcmp(extension.extensionName, name) == 0) return true; - } - return false; - }; - auto bytes = AgcDriver::QueryBdaByteFeatures(context.physical, function("vkGetPhysicalDeviceFeatures2"), available); - auto address = AgcDriver::QueryBdaFeatures(context.physical, function("vkGetPhysicalDeviceFeatures2"), available); - const auto queues = function("vkGetPhysicalDeviceQueueFamilyProperties"); - queues(context.physical, &count, nullptr); - std::vector families(count); - queues(context.physical, &count, families.data()); - std::uint32_t family = 0; - while (family < count && (families[family].queueFlags & VK_QUEUE_COMPUTE_BIT) == 0) ++family; - Require(family < count, "no Vulkan compute queue"); - const float priority = 1; - VkDeviceQueueCreateInfo queue{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; - queue.queueFamilyIndex = family; - queue.queueCount = 1; - queue.pQueuePriorities = &priority; - VkPhysicalDeviceFeatures enabled{}; - enabled.shaderInt64 = VK_TRUE; - address.pNext = &bytes; - std::vector extensionsEnabled{VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, VK_KHR_8BIT_STORAGE_EXTENSION_NAME}; - VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; - if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME)) { - extensionsEnabled.push_back(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME); - VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT, &driverProperties}; - VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties}; - function("vkGetPhysicalDeviceProperties2")(context.physical, &properties); - context.hostImportAlignment = hostProperties.minImportedHostPointerAlignment; - } - if (driverProperties.driverID != VK_DRIVER_ID_NVIDIA_PROPRIETARY && hasExtension(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME) && hasExtension(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME)) { - extensionsEnabled.push_back(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME); - extensionsEnabled.push_back(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME); - context.dmaBufImport = true; - } - VkPhysicalDeviceImageViewMinLodFeaturesEXT minLod{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_VIEW_MIN_LOD_FEATURES_EXT}; - if (hasExtension(VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME)) { - VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &minLod}; - function("vkGetPhysicalDeviceFeatures2")(context.physical, &features); - context.imageViewMinLod = minLod.minLod == VK_TRUE; - } - minLod = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_VIEW_MIN_LOD_FEATURES_EXT}; - minLod.minLod = VK_TRUE; - if (context.imageViewMinLod) { - extensionsEnabled.push_back(VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME); - minLod.pNext = address.pNext; - address.pNext = &minLod; - } - VkDeviceCreateInfo device{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, &address}; - device.queueCreateInfoCount = 1; - device.pQueueCreateInfos = &queue; - device.enabledExtensionCount = static_cast(extensionsEnabled.size()); - device.ppEnabledExtensionNames = extensionsEnabled.data(); - device.pEnabledFeatures = &enabled; - Check(function("vkCreateDevice")(context.physical, &device, nullptr, &context.device), "vkCreateDevice"); - context.deviceProc = function("vkGetDeviceProcAddr"); - function("vkGetPhysicalDeviceMemoryProperties")(context.physical, &context.memory); - VkPhysicalDeviceProperties properties{}; - function("vkGetPhysicalDeviceProperties")(context.physical, &properties); - context.limits = properties.limits; - context.bufferDeviceAddress = true; - context.formatProperties = function("vkGetPhysicalDeviceFormatProperties"); - context.Function("vkGetDeviceQueue")(context.device, family, 0, &context.queue); - VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; - pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; - pool.queueFamilyIndex = family; - Check(context.Function("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); - } catch (...) { - release(); - throw; - } - } - - ~Device() { release(); } - const Context& GetContext() const { return context; } - void WaitQueue() const { Check(context.Function("vkQueueWaitIdle")(context.queue), "vkQueueWaitIdle"); } - -private: - template - TFunction function(const char* name) const { - const auto result = reinterpret_cast(instanceProc(instance, name)); - Require(result != nullptr, name); - return result; - } - - void release() noexcept { - if (context.pool != VK_NULL_HANDLE) context.Function("vkDestroyCommandPool")(context.device, context.pool, nullptr); - context.bufferPool.reset(); - if (context.device != VK_NULL_HANDLE) { - DestroyShadows(context.device); - function("vkDestroyDevice")(context.device, nullptr); - } - if (instance != VK_NULL_HANDLE) function("vkDestroyInstance")(instance, nullptr); - if (library != nullptr) SDL_UnloadObject(library); - } - - void* library = nullptr; - PFN_vkGetInstanceProcAddr instanceProc = nullptr; - VkInstance instance = VK_NULL_HANDLE; - Context context{}; -}; +using Device = AgcDriver::Tests::RecorderDevice; using Kind = Recorder::ReadKind; diff --git a/core/libs/prx/libSceAgcDriver/tests/RecorderDevice.hpp b/core/libs/prx/libSceAgcDriver/tests/RecorderDevice.hpp new file mode 100644 index 00000000000..02296f632ed --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/RecorderDevice.hpp @@ -0,0 +1,172 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_RECORDERDEVICE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_RECORDERDEVICE_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" +#include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" +#include +#include +#include +#include + +namespace AgcDriver::Tests { + +using Graphics::Check; +using Graphics::Context; +using Graphics::DestroyShadows; +using Graphics::Require; + +class RecorderDevice { +public: + RecorderDevice() { +#ifdef _WIN32 + library = SDL_LoadObject("vulkan-1.dll"); +#else + library = SDL_LoadObject("libvulkan.so.1"); +#endif + Require(library != nullptr, "cannot load Vulkan"); + try { + instanceProc = reinterpret_cast(SDL_LoadFunction(library, "vkGetInstanceProcAddr")); + Require(instanceProc != nullptr, "missing Vulkan instance resolver"); + VkApplicationInfo application{VK_STRUCTURE_TYPE_APPLICATION_INFO}; + application.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo info{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + info.pApplicationInfo = &application; + Check(function("vkCreateInstance")(&info, nullptr, &instance), "vkCreateInstance"); + std::uint32_t count = 0; + const auto enumerate = function("vkEnumeratePhysicalDevices"); + Check(enumerate(instance, &count, nullptr), "vkEnumeratePhysicalDevices"); + Require(count != 0, "no Vulkan device"); + std::vector devices(count); + Check(enumerate(instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); + const auto rankDeviceType = [](VkPhysicalDeviceType type) { + switch (type) { + case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return 3; + case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return 2; + case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return 1; + default: return 0; + } + }; + const auto physicalProperties = function("vkGetPhysicalDeviceProperties"); + int selectedRank = -1; + for (const auto physical : devices) { + VkPhysicalDeviceProperties candidate{}; + physicalProperties(physical, &candidate); + if (candidate.apiVersion < VK_API_VERSION_1_1) continue; + const int rank = rankDeviceType(candidate.deviceType); + if (rank <= selectedRank) continue; + context.physical = physical; + selectedRank = rank; + } + Require(context.physical != VK_NULL_HANDLE, "no Vulkan 1.1 device"); + const auto extensions = function("vkEnumerateDeviceExtensionProperties"); + Check(extensions(context.physical, nullptr, &count, nullptr), "vkEnumerateDeviceExtensionProperties"); + std::vector available(count); + Check(extensions(context.physical, nullptr, &count, available.data()), "vkEnumerateDeviceExtensionProperties"); + const auto hasExtension = [&](const char* name) { + for (const auto& extension : available) { + if (std::strcmp(extension.extensionName, name) == 0) return true; + } + return false; + }; + auto bytes = AgcDriver::QueryBdaByteFeatures(context.physical, function("vkGetPhysicalDeviceFeatures2"), available); + auto address = AgcDriver::QueryBdaFeatures(context.physical, function("vkGetPhysicalDeviceFeatures2"), available); + const auto queues = function("vkGetPhysicalDeviceQueueFamilyProperties"); + queues(context.physical, &count, nullptr); + std::vector families(count); + queues(context.physical, &count, families.data()); + std::uint32_t family = 0; + while (family < count && (families[family].queueFlags & VK_QUEUE_COMPUTE_BIT) == 0) ++family; + Require(family < count, "no Vulkan compute queue"); + const float priority = 1; + VkDeviceQueueCreateInfo queue{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + queue.queueFamilyIndex = family; + queue.queueCount = 1; + queue.pQueuePriorities = &priority; + VkPhysicalDeviceFeatures enabled{}; + enabled.shaderInt64 = VK_TRUE; + address.pNext = &bytes; + std::vector extensionsEnabled{VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, VK_KHR_8BIT_STORAGE_EXTENSION_NAME}; + VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; + if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME)) { + extensionsEnabled.push_back(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME); + VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT, &driverProperties}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties}; + function("vkGetPhysicalDeviceProperties2")(context.physical, &properties); + context.hostImportAlignment = hostProperties.minImportedHostPointerAlignment; + } + if (driverProperties.driverID != VK_DRIVER_ID_NVIDIA_PROPRIETARY && hasExtension(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME) && hasExtension(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME)) { + extensionsEnabled.push_back(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME); + extensionsEnabled.push_back(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME); + context.dmaBufImport = true; + } + VkPhysicalDeviceImageViewMinLodFeaturesEXT minLod{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_VIEW_MIN_LOD_FEATURES_EXT}; + if (hasExtension(VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &minLod}; + function("vkGetPhysicalDeviceFeatures2")(context.physical, &features); + context.imageViewMinLod = minLod.minLod == VK_TRUE; + } + minLod = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_VIEW_MIN_LOD_FEATURES_EXT}; + minLod.minLod = VK_TRUE; + if (context.imageViewMinLod) { + extensionsEnabled.push_back(VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME); + minLod.pNext = address.pNext; + address.pNext = &minLod; + } + VkDeviceCreateInfo device{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, &address}; + device.queueCreateInfoCount = 1; + device.pQueueCreateInfos = &queue; + device.enabledExtensionCount = static_cast(extensionsEnabled.size()); + device.ppEnabledExtensionNames = extensionsEnabled.data(); + device.pEnabledFeatures = &enabled; + Check(function("vkCreateDevice")(context.physical, &device, nullptr, &context.device), "vkCreateDevice"); + context.deviceProc = function("vkGetDeviceProcAddr"); + function("vkGetPhysicalDeviceMemoryProperties")(context.physical, &context.memory); + VkPhysicalDeviceProperties properties{}; + function("vkGetPhysicalDeviceProperties")(context.physical, &properties); + context.limits = properties.limits; + context.bufferDeviceAddress = true; + context.formatProperties = function("vkGetPhysicalDeviceFormatProperties"); + context.Function("vkGetDeviceQueue")(context.device, family, 0, &context.queue); + VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + pool.queueFamilyIndex = family; + Check(context.Function("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); + } catch (...) { + release(); + throw; + } + } + + ~RecorderDevice() { release(); } + const Context& GetContext() const { return context; } + void WaitQueue() const { Check(context.Function("vkQueueWaitIdle")(context.queue), "vkQueueWaitIdle"); } + +private: + template + TFunction function(const char* name) const { + const auto result = reinterpret_cast(instanceProc(instance, name)); + Require(result != nullptr, name); + return result; + } + + void release() noexcept { + if (context.pool != VK_NULL_HANDLE) context.Function("vkDestroyCommandPool")(context.device, context.pool, nullptr); + context.bufferPool.reset(); + if (context.device != VK_NULL_HANDLE) { + DestroyShadows(context.device); + function("vkDestroyDevice")(context.device, nullptr); + } + if (instance != VK_NULL_HANDLE) function("vkDestroyInstance")(instance, nullptr); + if (library != nullptr) SDL_UnloadObject(library); + } + + void* library = nullptr; + PFN_vkGetInstanceProcAddr instanceProc = nullptr; + VkInstance instance = VK_NULL_HANDLE; + Context context{}; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp b/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp new file mode 100644 index 00000000000..714c0c2f9de --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp @@ -0,0 +1,606 @@ +#include "RecorderDevice.hpp" +#include "StorageEdgeWriteWatch.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif + +namespace { + +using namespace AgcDriver::Graphics; +namespace Memory = AgcDriver::GuestMemory; +constexpr std::size_t UnitBytes = 65536; +constexpr std::byte Initial{0xcd}; +constexpr std::byte Neighbour{0x19}; + +void noteWrite(std::uintptr_t address, std::size_t bytes) { +#ifdef ANYPS5_TEST_WRITE_WATCH + StorageEdgeNoteCpuWrite(address, bytes); +#else + static_cast(address); + static_cast(bytes); +#endif +} + +class Surface { +public: + Surface(const Context& baseContext, Recorder& recorder, std::size_t offset, std::uint32_t width = 512, std::uint32_t height = 512, bool queuedUpload = false, bool forceCpuFallback = false, bool defer = false) : context(baseContext), recorder(recorder), offset(offset) { + if (forceCpuFallback) context.hostImportAlignment = 0; + descriptor.width = width; + descriptor.height = height; + descriptor.mipCount = 1; + descriptor.tileMode = TextureTileMode::kStandard4KB; + descriptor.dimension = TextureDimension::k2D; + descriptor.format = 1; + descriptor.dstSelX = 4; + descriptor.dstSelY = 0; + descriptor.dstSelZ = 0; + descriptor.dstSelW = 1; + surfaceBytes = static_cast(DescribeSurface(descriptor).guestBytes); + allocationBytes = (offset + surfaceBytes + UnitBytes - 1) / UnitBytes * UnitBytes + UnitBytes; +#ifdef _WIN32 + allocation = GuestArena::GuestArenaAllocate_nid_postfix(allocationBytes, UnitBytes); + GuestArena::GuestArenaCommit_nid_postfix(allocation, allocationBytes, PAGE_READWRITE, allocationBytes); +#else + void* raw = mmap(nullptr, allocationBytes + UnitBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot allocate edge storage memory"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + UnitBytes - 1) & ~(static_cast(UnitBytes) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + allocationBytes != begin + allocationBytes + UnitBytes) munmap(reinterpret_cast(aligned + allocationBytes), begin + UnitBytes - aligned); + allocation = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(allocation, allocationBytes); +#endif + Require(allocation != nullptr, "cannot allocate edge storage memory"); + const auto base = reinterpret_cast(allocation); + descriptor.baseAddress = base + offset; + std::memset(allocation, std::to_integer(Neighbour), allocationBytes); + std::memset(reinterpret_cast(descriptor.baseAddress), std::to_integer(Initial), surfaceBytes); + noteWrite(base, allocationBytes); + { + GuestAllocations::Mutation mutation; + mutation.Add(allocation, allocationBytes, true, true); + } + const auto* hostImport = HostImportFor(context, base, allocationBytes); + Require(forceCpuFallback ? hostImport == nullptr : hostImport != nullptr, "edge test did not select the expected upload path"); + Require(Memory::Watched(base, allocationBytes), "edge test memory is not write watched"); + if (queuedUpload) { + const auto* import = HostImportFor(context, descriptor.baseAddress, surfaceBytes); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdFillBuffer")(commands, import->buffer, descriptor.baseAddress - import->base, surfaceBytes, 0x77777777u); + recorder.NotePendingWrite(descriptor.baseAddress, surfaceBytes); + Memory::MarkWritten(descriptor.baseAddress, surfaceBytes); + } + image = std::make_shared(context, *context.detiler, descriptor, 0); + if (!defer) Sync(); + Require(image->Generation() != 0, "edge image has no tracked upload generation"); + Require(Memory::UnchangedSince(descriptor.baseAddress, surfaceBytes, image->Generation()), "edge upload did not establish a current generation"); + } + + ~Surface() { + recorder.Sync(); + alias.reset(); + image.reset(); + { + GuestAllocations::Mutation mutation; + mutation.Remove(allocation); + } + HostImportFor(context, reinterpret_cast(allocation), allocationBytes); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(allocation, allocationBytes); + GuestArena::GuestArenaRelease_nid_postfix(allocation, allocationBytes); +#else + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(allocation, allocationBytes); + munmap(allocation, allocationBytes); +#endif + } + + void Draw(std::uint8_t value, bool defer = false, bool aliased = false) { + const auto& target = aliased ? alias : image; + Require(target != nullptr, "the image draw fixture is not created"); + const auto commands = recorder.Commands(); + recorder.Keep(target); + const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + VkClearColorValue clear{{static_cast(value) / 255.0f, 0.0f, 0.0f, 1.0f}}; + if (aliased) clear.uint32[0] = value; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdClearColorImage")(commands, target->Image(), VK_IMAGE_LAYOUT_GENERAL, &clear, 1, &range); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); + target->MarkDirty(); + if (!defer) Sync(); + } + + void CpuStore(std::size_t at, std::byte value, bool aliased = false) { + const auto& target = aliased ? alias : image; + Require(target != nullptr, "the CPU store fixture is not created"); + Require(at < surfaceBytes, "CPU edge store is outside the surface"); + *reinterpret_cast(descriptor.baseAddress + at) = value; + noteWrite(descriptor.baseAddress + at, 1); + Memory::CollectWritesUncached(descriptor.baseAddress + at, 1); + Require(Memory::WrittenSince(descriptor.baseAddress + at, 1, target->Generation()), "CPU edge store did not acquire a newer write stamp"); + } + + void CpuStoreUncollected(std::size_t at, std::byte value) { + Require(at < surfaceBytes, "uncollected CPU edge store is outside the surface"); + Require(Memory::UnchangedSince(descriptor.baseAddress, surfaceBytes, image->Generation()), "uncollected CPU fixture already has a newer write stamp"); + const auto before = Memory::TrackerGeneration(); + *reinterpret_cast(descriptor.baseAddress + at) = value; + noteWrite(descriptor.baseAddress + at, 1); + Require(Memory::TrackerGeneration() == before, "an uncollected CPU store advanced the write tracker"); + Require(Memory::UnchangedSince(descriptor.baseAddress, surfaceBytes, image->Generation()), "an uncollected CPU store reached the tracker before refresh"); + } + + void StoreNeighbour() { + Require(offset >= 4096, "test surface has no head neighbour"); + auto* bytes = static_cast(allocation); + bytes[16] = std::byte{0x41}; + noteWrite(reinterpret_cast(allocation) + 16, 1); + Memory::CollectWritesUncached(reinterpret_cast(allocation), 4096); + neighbours[16] = std::byte{0x41}; + } + + void DriverStore(std::size_t at, std::byte value) { + const auto address = descriptor.baseAddress + at; + const auto* import = HostImportFor(context, address, 4); + Require(import != nullptr, "driver edge store lost its host import"); + const std::array bytes{value, value, value, value}; + recorder.RecordStore(import->buffer, address - import->base, bytes, address); + recorder.NotePendingWrite(address, bytes.size()); + Require(Memory::MarkWritten(address, bytes.size()) > image->Generation(), "driver edge store did not acquire a newer stamp"); + } + + void Reregister() { + GuestAllocations::Mutation mutation; + mutation.Remove(allocation); + mutation.Add(allocation, allocationBytes, true, true); + } + + void WriteBack(bool aliased = false) { + const auto& target = aliased ? alias : image; + Require(target != nullptr, "the write-back fixture is not created"); + target->WriteBack(); + Sync(); + } + + void Refresh() { + image->Refresh(); + Sync(); + } + + void CreateAlias() { + image->SetCached(true); + auto resource = descriptor; + resource.format = 5; + alias = std::make_shared(context, *context.detiler, resource, 0); + Sync(); + } + + void RefreshAlias() { + Require(alias != nullptr, "the alias fixture is not created"); + alias->Refresh(); + Sync(); + } + + void Fill(std::uint8_t value, bool defer = false) { + const std::uint32_t repeated = static_cast(value) * 0x01010101u; + const std::array pattern{repeated, repeated, repeated, repeated}; + const char* refusal = nullptr; + const bool cleared = image->FillClear(pattern, StorageTexture::WholeImage, refusal); + Require(cleared, std::string("edge fill fixture was refused: ") + (refusal != nullptr ? refusal : "unknown")); + if (!defer) Sync(); + } + + void CopyFrom(Surface& source, bool defer = false) { + image->SetCached(true); + source.image->SetCached(true); + const char* refusal = nullptr; + const bool copied = image->CopyFrom(*source.image, refusal); + Require(copied, std::string("edge copy fixture was refused: ") + (refusal != nullptr ? refusal : "unknown")); + if (!defer) Sync(); + } + + void CheckMemory(std::byte expected, const std::map& exceptions, const std::string& what) { + std::vector bytes(allocationBytes); + Memory::Read(reinterpret_cast(allocation), bytes); + for (std::size_t at = 0; at < allocationBytes; ++at) { + auto wanted = Neighbour; + if (at >= offset && at < offset + surfaceBytes) { + const auto found = exceptions.find(at - offset); + wanted = found == exceptions.end() ? expected : found->second; + } else if (const auto found = neighbours.find(at); found != neighbours.end()) { + wanted = found->second; + } + if (bytes[at] != wanted) throw std::runtime_error(what + ": byte " + std::to_string(at) + " expected " + std::to_string(std::to_integer(wanted)) + " got " + std::to_string(std::to_integer(bytes[at]))); + } + } + + void CheckImage(std::byte expected, const std::map& exceptions, const std::string& what, bool aliased = false) { + const auto& source = aliased ? alias : image; + Require(source != nullptr, "the image readback fixture is not created"); + const auto texelBytes = static_cast(descriptor.width) * descriptor.height; + Buffer readback(context, texelBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {descriptor.width, descriptor.height, 1}; + context.Function("vkCmdCopyImageToBuffer")(commands, source->Image(), VK_IMAGE_LAYOUT_GENERAL, readback.Handle(), 1, ©); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + Sync(); + std::map counts; + for (const auto byte : readback.Bytes().first(texelBytes)) { + if (byte != expected) ++counts[byte]; + } + Require(counts == exceptions, what + ": image does not contain the merged texels"); + } + + std::size_t Size() const { return surfaceBytes; } + std::uint64_t Address() const { return descriptor.baseAddress; } + std::uint64_t Generation() const { return image->Generation(); } + +private: + void Sync() { + recorder.Submit(); + Check(context.Function("vkQueueWaitIdle")(context.queue), "vkQueueWaitIdle edge test"); + recorder.Sync(); + } + + Context context; + Recorder& recorder; + std::size_t offset; + std::size_t surfaceBytes = 0; + std::size_t allocationBytes = 0; + void* allocation = nullptr; + GuestTextureResource descriptor{}; + std::shared_ptr image; + std::shared_ptr alias; + std::map neighbours; +}; + +void noOpEdgeTests(const Context& context, Recorder& recorder) { + for (const bool refresh : {false, true}) { + Surface surface(context, recorder, 8192); + surface.Draw(128); + surface.CpuStore(32, Initial); + surface.CpuStore(surface.Size() - 32, Initial); + surface.StoreNeighbour(); + if (refresh) { + surface.Refresh(); + surface.CheckImage(std::byte{128}, {}, "no-op CPU edge store during refresh"); + } else { + surface.WriteBack(); + } + surface.CheckMemory(std::byte{128}, {}, "no-op CPU edge store discarded GPU results"); + } +} + +void mixedEdgeTests(const Context& context, Recorder& recorder) { + Surface surface(context, recorder, 8192); + surface.Draw(128); + surface.CpuStore(17, std::byte{0x31}); + surface.CpuStore(surface.Size() - 23, std::byte{0x32}); + surface.CpuStore(4096 + 17, Initial); + surface.StoreNeighbour(); + surface.WriteBack(); + surface.CheckMemory(std::byte{128}, {{17, std::byte{0x31}}, {surface.Size() - 23, std::byte{0x32}}}, "mixed CPU and GPU edge writes"); + surface.Refresh(); + surface.CheckImage(std::byte{128}, {{std::byte{0x31}, 1}, {std::byte{0x32}, 1}}, "mixed edge refresh"); + surface.Draw(64); + surface.CpuStore(17, std::byte{0x31}); + surface.CpuStore(surface.Size() - 23, std::byte{0x52}); + surface.WriteBack(); + surface.CheckMemory(std::byte{64}, {{surface.Size() - 23, std::byte{0x52}}}, "second edge write-back reused an obsolete baseline"); +} + +void shortSurfaceTests(const Context& context, Recorder& recorder) { + for (const std::size_t offset : {0, 8192}) { + for (const bool refresh : {false, true}) { + for (const bool collected : {true, false}) { + for (const bool changed : {false, true}) { + Surface surface(context, recorder, offset, 64, 64); + Require(surface.Size() == 4096, "short edge fixture is not one page"); + surface.Draw(128); + if (collected) { + surface.CpuStore(32, Initial); + if (changed) surface.CpuStore(17, std::byte{0x31}); + } else { + surface.CpuStoreUncollected(32, Initial); + if (changed) surface.CpuStoreUncollected(17, std::byte{0x31}); + } + const std::map texels = changed ? std::map{{std::byte{0x31}, 1}} : std::map{}; + const std::map memory = changed ? std::map{{17, std::byte{0x31}}} : std::map{}; + if (refresh) { + surface.Refresh(); + surface.CheckImage(std::byte{128}, texels, "refresh discarded a short edge surface"); + } + surface.CheckMemory(std::byte{128}, memory, "a read hook discarded a whole short edge surface"); + } + } + } + } +} + +void fullBlockTests(const Context& context, Recorder& recorder) { + Surface surface(context, recorder, 0); + surface.Draw(128); + surface.CpuStore(32, Initial); + surface.WriteBack(); + std::map kept; + for (std::size_t at = 0; at < UnitBytes; ++at) kept.emplace(at, Initial); + surface.CheckMemory(std::byte{128}, kept, "an aligned CPU-stamped block changed ownership"); +} + +void uploadOrderTests(const Context& context, Recorder& recorder) { + Surface surface(context, recorder, 8192, 512, 512, true); + surface.Draw(128); + surface.CpuStore(32, std::byte{0x77}); + surface.CpuStore(surface.Size() - 32, std::byte{0x77}); + surface.WriteBack(); + surface.CheckMemory(std::byte{128}, {}, "edge baseline preceded an ordered GPU upload source write"); +} + +void writeBackBaselineTests(const Context& context, Recorder& recorder) { + for (const bool cpu : {false, true}) { + for (const bool shortSurface : {false, true}) { + Surface surface(context, recorder, shortSurface ? 0 : 8192, shortSurface ? 64 : 512, shortSurface ? 64 : 512, false, cpu); + surface.Draw(128); + surface.WriteBack(); + surface.Draw(64); + surface.CpuStore(32, std::byte{128}); + surface.WriteBack(); + surface.CheckMemory(std::byte{64}, {}, "a normal write-back did not renew the edge baseline"); + } + } +} + +void driverStoreTests(const Context& context, Recorder& recorder) { + for (const bool cpu : {false, true}) { + Surface surface(context, recorder, 8192); + surface.Draw(128); + if (cpu) surface.CpuStore(17, std::byte{0x31}); + surface.DriverStore(64, Initial); + surface.WriteBack(); + std::map kept; + for (std::size_t at = 0; at < UnitBytes - 8192; ++at) kept.emplace(at, Initial); + if (cpu) kept[17] = std::byte{0x31}; + surface.CheckMemory(std::byte{128}, kept, "a same-value ordered driver store lost edge ownership"); + } +} + +void driverProvenanceTests(const Context& context, Recorder& recorder) { + Surface surface(context, recorder, 8192); + const auto address = surface.Address(); + const auto generation = surface.Generation(); + Require(!Memory::DriverStoredOver(address, 64, generation), "an untouched edge has a driver store"); + surface.CpuStore(32, Initial); + Require(!Memory::DriverStoredOver(address, 64, generation), "a CPU-only write was attributed to the driver"); + Require(Memory::MarkWritten(address + 512, 4) > generation, "the driver piece was not stamped"); + Require(Memory::DriverStoredOver(address + 512, 4, generation), "an exact driver store was forgotten"); + Require(!Memory::DriverStoredOver(address, 64, generation), "a disjoint driver piece owns the CPU-written edge"); + const auto whole = address + UnitBytes - 8192; + Require(Memory::MarkWritten(whole, UnitBytes) > generation, "the whole driver block was not stamped"); + surface.CpuStore(UnitBytes - 8192 + 8, Initial); + Require(Memory::DriverStoredOver(whole, 16, generation), "a later CPU stamp erased a whole driver store"); + const auto history = Memory::TrackerGeneration(); + const auto pieces = whole + UnitBytes; + for (std::uint64_t piece = 0; piece < 6; ++piece) Memory::MarkWritten(pieces + piece * 64, 4); + Require(Memory::DriverStoredOver(pieces, 4, history), "evicted newer driver history was treated as no store"); + Require(Memory::DriverStoredOver(address, 64, 0), "an unknown baseline generation claims no driver store"); + std::array unwatched{}; + const auto outside = reinterpret_cast(unwatched.data()); + Require(!Memory::Watched(outside, unwatched.size()), "the untracked driver-store fixture is watched"); + Require(Memory::DriverStoredOver(outside, unwatched.size(), Memory::TrackerGeneration()), "unwatched memory claims no driver store"); + const auto huge = std::numeric_limits::max() - static_cast(address); + Require(Memory::DriverStoredOver(address, huge, generation), "a huge uncovered range claims no driver store"); + Require(Memory::DriverStoredOver(address, std::numeric_limits::max(), generation), "an overflowing range claims no driver store"); + Require(!Memory::DriverStoredOver(address, 0, generation), "an empty watched range contains a driver store"); + Require(!Memory::DriverStoredOver(outside, 0, 0), "an empty unknown range contains a driver store"); +} + +void reregisteredEdgeTests(const Context& context, Recorder& recorder) { + Surface surface(context, recorder, 8192, 64, 64); + surface.Draw(128); + surface.Reregister(); + surface.CpuStore(32, Initial); + surface.WriteBack(); + surface.CheckMemory(Initial, {}, "a retired edge baseline overwrote a replacement allocation"); +} + +void aliasEdgeTests(const Context& context, Recorder& recorder) { + for (const bool collected : {true, false}) { + for (const bool changed : {false, true}) { + Surface surface(context, recorder, 0, 64, 64); + surface.CreateAlias(); + surface.Draw(128); + if (collected) { + surface.CpuStore(32, Initial); + if (changed) surface.CpuStore(17, std::byte{0x31}); + } else { + surface.CpuStoreUncollected(32, Initial); + if (changed) surface.CpuStoreUncollected(17, std::byte{0x31}); + } + surface.RefreshAlias(); + const std::map texels = changed ? std::map{{std::byte{0x31}, 1}} : std::map{}; + const std::map memory = changed ? std::map{{17, std::byte{0x31}}} : std::map{}; + const std::string state = collected ? "collected" : "uncollected"; + surface.CheckImage(std::byte{128}, texels, "an alias lost " + state + " CPU edge writes", true); + surface.CheckMemory(std::byte{128}, memory, "an alias refreshed from stale " + state + " edge bytes"); + } + } +} + +void aliasBorrowWriteTests(const Context& context, Recorder& recorder) { + Surface surface(context, recorder, 0, 64, 64); + surface.CreateAlias(); + surface.Draw(128); + surface.RefreshAlias(); + surface.CheckImage(std::byte{128}, {}, "an alias did not borrow pending GPU results", true); + surface.Draw(64, false, true); + surface.CpuStore(32, Initial, true); + surface.WriteBack(true); + surface.CheckMemory(std::byte{64}, {}, "a write after an alias borrow lost the physical edge baseline"); +} + +void cpuFallbackTests(const Context& context, Recorder& recorder) { + for (const bool shortSurface : {false, true}) { + Surface surface(context, recorder, shortSurface ? 0 : 8192, shortSurface ? 64 : 512, shortSurface ? 64 : 512, false, true); + surface.Draw(128); + surface.CpuStore(32, Initial); + surface.CpuStore(17, std::byte{0x31}); + surface.CpuStore(surface.Size() - 23, std::byte{0x32}); + surface.WriteBack(); + surface.CheckMemory(std::byte{128}, {{17, std::byte{0x31}}, {surface.Size() - 23, std::byte{0x32}}}, "CPU fallback lost mixed edge writes"); + surface.Refresh(); + surface.CheckImage(std::byte{128}, {{std::byte{0x31}, 1}, {std::byte{0x32}, 1}}, "CPU fallback edge refresh"); + surface.Draw(64); + surface.CpuStore(17, std::byte{0x31}); + surface.CpuStore(surface.Size() - 23, std::byte{0x32}); + surface.WriteBack(); + surface.CheckMemory(std::byte{64}, {}, "CPU fallback reused a stale merged edge baseline"); + } +} + +void pendingEdgeTests(const Context& context, Recorder& recorder) { + for (const bool shortSurface : {false, true}) { + Surface surface(context, recorder, shortSurface ? 0 : 8192, shortSurface ? 64 : 512, shortSurface ? 64 : 512, false, false, true); + surface.Draw(128, true); + surface.CpuStoreUncollected(17, std::byte{0x31}); + surface.CpuStoreUncollected(32, Initial); + surface.CpuStoreUncollected(surface.Size() - 32, Initial); + surface.WriteBack(); + surface.CheckMemory(std::byte{128}, {{17, std::byte{0x31}}}, "a pending upload captured a later CPU edit as its baseline"); + } +} + +void fillEdgeTests(const Context& context, Recorder& recorder) { + for (const bool shortSurface : {false, true}) { + for (const bool draw : {false, true}) { + Surface surface(context, recorder, shortSurface ? 0 : 8192, shortSurface ? 64 : 512, shortSurface ? 64 : 512); + surface.Fill(128); + surface.CheckImage(std::byte{128}, {}, "a GPU fill did not clear the edge image"); + if (draw) surface.Draw(64); + surface.CpuStore(32, Initial); + surface.WriteBack(); + surface.CheckMemory(draw ? std::byte{64} : std::byte{128}, {}, "a GPU fill lost the physical edge baseline"); + } + } +} + +void copyEdgeTests(const Context& context, Recorder& recorder) { + for (const bool draw : {false, true}) { + Surface source(context, recorder, 0, 64, 64); + Surface destination(context, recorder, 0, 64, 64); + source.Draw(128); + destination.CopyFrom(source); + destination.CheckImage(std::byte{128}, {}, "a GPU copy did not copy the edge image"); + if (draw) destination.Draw(64); + destination.CpuStore(32, Initial); + destination.WriteBack(); + destination.CheckMemory(draw ? std::byte{64} : std::byte{128}, {}, "a GPU copy lost the destination's physical edge baseline"); + } +} + +void pendingFillEdgeTests(const Context& context, Recorder& recorder) { + for (const bool shortSurface : {false, true}) { + for (const bool priorWrite : {false, true}) { + Surface surface(context, recorder, shortSurface ? 0 : 8192, shortSurface ? 64 : 512, shortSurface ? 64 : 512); + if (priorWrite) surface.DriverStore(32, std::byte{0x77}); + surface.Fill(128, true); + surface.CpuStoreUncollected(17, std::byte{0x31}); + surface.CpuStoreUncollected(32, priorWrite ? std::byte{0x77} : Initial); + surface.WriteBack(); + surface.CheckMemory(std::byte{128}, {{17, std::byte{0x31}}}, "a pending fill captured a later CPU write as its baseline"); + } + } +} + +void pendingCopyEdgeTests(const Context& context, Recorder& recorder) { + for (const bool shortSurface : {false, true}) { + for (const bool priorWrite : {false, true}) { + Surface source(context, recorder, shortSurface ? 0 : 8192, shortSurface ? 64 : 512, shortSurface ? 64 : 512); + Surface destination(context, recorder, shortSurface ? 0 : 8192, shortSurface ? 64 : 512, shortSurface ? 64 : 512); + source.Draw(128); + if (priorWrite) destination.DriverStore(32, std::byte{0x77}); + destination.CopyFrom(source, true); + destination.CpuStoreUncollected(17, std::byte{0x31}); + destination.CpuStoreUncollected(32, priorWrite ? std::byte{0x77} : Initial); + destination.WriteBack(); + destination.CheckMemory(std::byte{128}, {{17, std::byte{0x31}}}, "a pending copy captured a later CPU write as its baseline"); + } + } +} + +} + +int main() { + try { + std::unique_ptr device; + try { + device = std::make_unique(); + Require(device->GetContext().hostImportAlignment != 0, "edge write-back tests require Vulkan host imports"); + Require(Memory::WriteWatched(), "edge write-back tests require page write tracking"); + } catch (const std::exception& error) { + if (std::getenv("ANYPS5_REQUIRE_VULKAN") != nullptr) throw; + std::cout << "skipped, no usable Vulkan device or write tracking: " << error.what() << '\n'; + return 77; + } + std::lock_guard lock(Memory::GpuMutex()); + const auto& base = device->GetContext(); + SetImportWatch(base, ImportWatch::Watch); + TextureDetiler detiler(base); + auto context = base; + context.detiler = &detiler; + Recorder recorder(context); + recorder.Activate(); + noOpEdgeTests(context, recorder); + mixedEdgeTests(context, recorder); + shortSurfaceTests(context, recorder); + fullBlockTests(context, recorder); + uploadOrderTests(context, recorder); + writeBackBaselineTests(context, recorder); + driverStoreTests(context, recorder); + driverProvenanceTests(context, recorder); + reregisteredEdgeTests(context, recorder); + aliasEdgeTests(context, recorder); + aliasBorrowWriteTests(context, recorder); + cpuFallbackTests(context, recorder); + pendingEdgeTests(context, recorder); + fillEdgeTests(context, recorder); + copyEdgeTests(context, recorder); + pendingFillEdgeTests(context, recorder); + pendingCopyEdgeTests(context, recorder); + recorder.Sync(); + std::cout << "Storage edge write-back tests passed\n"; + } catch (const std::exception& error) { + std::cerr << "Storage edge write-back test failed: " << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteWatch.cpp b/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteWatch.cpp new file mode 100644 index 00000000000..152ba195a1d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteWatch.cpp @@ -0,0 +1,91 @@ +#include "StorageEdgeWriteWatch.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr std::uintptr_t PageBytes = 4096; +std::mutex watchMutex; +std::map watched; +std::set dirty; + +bool covers(std::uintptr_t address, std::size_t bytes) { + if (bytes == 0 || bytes > std::numeric_limits::max() - address) return false; + auto it = watched.upper_bound(address); + if (it == watched.begin()) return false; + --it; + return it->first <= address && address + bytes <= it->second; +} + +void noteWrite(std::uintptr_t address, std::size_t bytes) { + if (!covers(address, bytes)) return; + for (auto page = address & ~(PageBytes - 1); page < address + bytes; page += PageBytes) dirty.insert(page); +} + +} + +void StorageEdgeNoteCpuWrite(std::uintptr_t address, std::size_t bytes) { + std::lock_guard lock(watchMutex); + noteWrite(address, bytes); +} + +namespace GuestWriteWatch { + +extern "C" bool GuestWriteWatchAvailable_nid_postfix() { + return true; +} + +extern "C" void GuestWriteWatchRegister_nid_postfix(const void* pointer, std::size_t bytes) { + const auto address = reinterpret_cast(pointer); + std::lock_guard lock(watchMutex); + watched[address] = address + bytes; + noteWrite(address, bytes); +} + +extern "C" bool GuestWriteWatchUnregister_nid_postfix(const void* pointer, std::size_t bytes) { + const auto address = reinterpret_cast(pointer); + const auto end = address + bytes; + std::lock_guard lock(watchMutex); + std::vector> retained; + bool removed = false; + for (auto it = watched.begin(); it != watched.end();) { + if (it->second <= address || end <= it->first) { + ++it; + continue; + } + removed = true; + if (it->first < address) retained.emplace_back(it->first, address); + if (end < it->second) retained.emplace_back(end, it->second); + it = watched.erase(it); + } + for (const auto& [begin, stop] : retained) watched.emplace(begin, stop); + dirty.erase(dirty.lower_bound(address & ~(PageBytes - 1)), dirty.lower_bound(end)); + return removed; +} + +extern "C" bool GuestWriteWatchCovers_nid_postfix(std::uintptr_t address, std::size_t bytes) { + std::lock_guard lock(watchMutex); + return covers(address, bytes); +} + +extern "C" bool GuestWriteWatchCollect_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) { + std::vector collected; + { + std::lock_guard lock(watchMutex); + if (!covers(address, bytes)) return false; + const auto begin = dirty.lower_bound(address & ~(PageBytes - 1)); + const auto end = dirty.lower_bound(address + bytes); + collected.assign(begin, end); + dirty.erase(begin, end); + } + for (const auto page : collected) written(context, page, page + PageBytes); + return true; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteWatch.hpp b/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteWatch.hpp new file mode 100644 index 00000000000..8799635044d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteWatch.hpp @@ -0,0 +1,9 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_STORAGEEDGEWRITEWATCH_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_STORAGEEDGEWRITEWATCH_HPP + +#include +#include + +void StorageEdgeNoteCpuWrite(std::uintptr_t address, std::size_t bytes); + +#endif From 97655b677e273d93fb8fdfe1393ae7a372904a1b Mon Sep 17 00:00:00 2001 From: Elaine Bell Date: Fri, 9 Oct 2026 03:42:39 +0000 Subject: [PATCH 2/4] test(agc): document and verify partial-edge precedence --- .../tests/StorageEdgeWriteBack.cpp | 64 +++++++++++++++++++ docs/dev/TechnicalDebt.md | 1 + 2 files changed, 65 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp b/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp index 714c0c2f9de..f90ab761640 100644 --- a/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp @@ -557,6 +557,67 @@ void pendingCopyEdgeTests(const Context& context, Recorder& recorder) { } } + +void mergePrecedenceTests(const Context& context, Recorder& recorder) { + Surface surface(context, recorder, 8192); + constexpr std::size_t count = 65536; + std::vector baseline(count), current(count), expected(count); + auto* destination = reinterpret_cast(surface.Address()); + for (unsigned value = 0; value < 256; ++value) { + std::fill(baseline.begin(), baseline.end(), std::byte{static_cast(value)}); + for (std::size_t byte = 0; byte < count; ++byte) { + const auto cpu = static_cast(byte / 256); + const auto gpu = static_cast(byte % 256); + destination[byte] = std::byte{cpu}; + current[byte] = std::byte{gpu}; + expected[byte] = std::byte{cpu == value && gpu != value ? gpu : cpu}; + } + noteWrite(surface.Address(), count); + const auto generation = Memory::CollectWritesUncached(surface.Address(), count); + Require(Memory::MergeChangedCommitted(surface.Address(), current, baseline, generation), "a CPU-only byte merge was refused"); + Require(std::equal(expected.begin(), expected.end(), destination), "a CPU/GPU byte merge violated baseline precedence"); + } +} + +void mergeDriverPrecedenceTests(const Context& context, Recorder& recorder) { + for (const auto value : {Initial, std::byte{0x77}}) { + Surface surface(context, recorder, 8192, 64, 64); + const auto generation = surface.Generation(); + surface.DriverStore(0, value); + surface.WriteBack(); + surface.CpuStore(7, std::byte{0x31}); + std::array baseline, current; + baseline.fill(Initial); + current.fill(std::byte{128}); + Require(!Memory::MergeChangedCommitted(surface.Address(), current, baseline, generation), "a newer driver store permitted a stale byte merge"); + surface.CheckMemory(Initial, {{0, value}, {1, value}, {2, value}, {3, value}, {7, std::byte{0x31}}}, "a refused merge changed driver or CPU bytes"); + } +} + +void edgePrecedenceTests(const Context& context, Recorder& recorder) { + for (const bool cpuFirst : {true, false}) { + Surface surface(context, recorder, 8192, 64, 64); + if (cpuFirst) surface.CpuStore(17, std::byte{0x31}); + surface.Draw(128); + if (!cpuFirst) surface.CpuStore(17, std::byte{0x31}); + surface.CpuStore(19, std::byte{0x33}); + surface.CpuStore(19, Initial); + surface.WriteBack(); + surface.CheckMemory(std::byte{128}, {{17, std::byte{0x31}}}, "mixed CPU/GPU edge result differs"); + surface.CpuStore(18, std::byte{0x32}); + surface.Refresh(); + surface.CheckImage(std::byte{128}, {{std::byte{0x31}, 1}, {std::byte{0x32}, 1}}, "a CPU edit after the edge merge was lost"); + } + Surface surface(context, recorder, 8192, 64, 64); + surface.Draw(205); + surface.CpuStore(17, std::byte{0x31}); + surface.CpuStore(18, Initial); + surface.WriteBack(); + surface.CheckMemory(Initial, {{17, std::byte{0x31}}}, "an unchanged GPU result overwrote a CPU edit"); + surface.Refresh(); + surface.CheckImage(Initial, {{std::byte{0x31}, 1}}, "an unchanged GPU result lost a CPU edit at refresh"); +} + } int main() { @@ -587,6 +648,9 @@ int main() { writeBackBaselineTests(context, recorder); driverStoreTests(context, recorder); driverProvenanceTests(context, recorder); + mergePrecedenceTests(context, recorder); + mergeDriverPrecedenceTests(context, recorder); + edgePrecedenceTests(context, recorder); reregisteredEdgeTests(context, recorder); aliasEdgeTests(context, recorder); aliasBorrowWriteTests(context, recorder); diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 81b67436620..9f2cfd8318b 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -240,6 +240,7 @@ Throughout the project, every function at every stage either **does exactly what - [sceKernelWriteThrottlingStatus](../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) reports no write throttling: the host SSD has no write budget, so the first qword of the 32-byte status is set to `UINT32_MAX` and the rest is zeroed. PPSA17221 reads only that qword: it multiplies it by 64 KiB and compares the product with byte thresholds up to 0x4AF00000 to pick one of five tiers. When the call fails the title assumes 0x4B000001 bytes, which selects the same top tier as `UINT32_MAX`. What the value measures on hardware (a remaining budget or a rate) and the meaning of the other three qwords are unknown - [Wide-character conversions](../../core/libs/prx/libc/src/Strings.cpp) (libc) - `wcsrtombs` and [`swprintf`](../../core/libs/prx/libc/src/Formatting.cpp) still pass guest strings to the host's `wchar_t` functions. The guest's `wchar_t` is 16 bits, so on Linux, where the host's is 32 bits, they misread and miswrite guest strings. The wide `wcsto*` functions parse the ASCII prefix of the guest string, as the C locale accepts only ASCII white space and digits - [SPIRV-Tools optimization](../../core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp) skips CCP and Simplification for modules with storage-only 8- or 16-bit types because the pinned folder produces illegal narrow constants. This also disables unrelated constant folding; the restricted full pipeline mirrors the pinned performance recipe. +- [Storage-image partial-edge writeback](../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) uses an emulator policy whose CPU/GPU precedence has not been measured on console hardware. For each byte, an unchanged GPU result keeps the destination; a changed GPU result replaces it only while it equals the immutable baseline. A same-value CPU write or a change back to the baseline permits the GPU result. A newer ordered driver store vetoes the affected partial-edge merge, even when it writes the same value. The baseline owner must match a current readable and writable allocation range, leased through the merge. Full 64 KiB blocks retain their existing ownership rule. These checks were exercised on Linux llvmpipe with the test-only page-watch backend; native write-watch races remain unverified. The allocation check applies to edge merges: ordinary writeback without a newer CPU stamp can still write into re-registered memory. - Windows host-pointer imports are compared instead of write-watched: a driver can mark imported pages dirty after the synchronous import window. [GuestMemory](../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. This preserves CPU edits through byte comparisons but disables snapshot reuse and unit shadows for excluded blocks; restoring tracking on import retirement without replacing the backing and measuring the comparison cost remain unimplemented. - [sceNetSendmsg and sceNetRecvmsg](../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) copy the scatter/gather list through one host buffer and accept only the flags `sceNetSend`/`sceNetRecv` accept. Sending control data throws; a receive never returns control data (`msg_controllen` 0), since no socket option that queues it is supported - [Additional content](../../core/libs/prx/libSceAppContent/Export.cpp) (libSceAppContent) is not installed or mounted: `sceAppContentAddcontMount` answers `NOT_FOUND` for every entitlement label and `sceAppContentAddcontUnmount` for every mount point, as no add-content mount is ever handed out; `sceAppContentGetAddcontInfo` answers `DRM_NO_ENTITLEMENT` for every label and `sceAppContentGetAddcontInfoList` reports 0 entries From 690ebaded1e266df03954a0816236845c8b24376 Mon Sep 17 00:00:00 2001 From: ToyuAU Date: Fri, 9 Oct 2026 14:18:55 +1000 Subject: [PATCH 3/4] fix(agc): merge CPU-stamped full image blocks by byte --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 5 +- .../Graphics/include/Texture.hpp | 2 + .../libSceAgcDriver/Graphics/src/Texture.cpp | 95 ++++++++---- .../tests/StorageEdgeWriteBack.cpp | 135 +++++++++++++++++- docs/dev/TechnicalDebt.md | 2 +- 5 files changed, 203 insertions(+), 36 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 5fb6d8f518f..ed756c77040 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -480,9 +480,10 @@ endif() if(BUILD_TESTING) set_target_properties(agc_storage_edge_writeback_tests PROPERTIES EXCLUDE_FROM_ALL OFF) add_test(NAME agc_storage_edge_writeback COMMAND agc_storage_edge_writeback_tests) - set_tests_properties(agc_storage_edge_writeback PROPERTIES SKIP_RETURN_CODE 77) + add_test(NAME agc_storage_full_block_writeback COMMAND agc_storage_edge_writeback_tests --full-blocks) + set_tests_properties(agc_storage_edge_writeback agc_storage_full_block_writeback PROPERTIES SKIP_RETURN_CODE 77) if(WIN32) - set_tests_properties(agc_storage_edge_writeback PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(agc_storage_edge_writeback agc_storage_full_block_writeback PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 93a59f0de39..d42b8441aad 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -314,11 +314,13 @@ class StorageTexture : public std::enable_shared_from_this { std::uint64_t end; std::shared_ptr bytes; std::weak_ptr allocation; + std::size_t offset = 0; }; struct EdgeMerge { EdgeSnapshot baseline; std::shared_ptr result; std::uint64_t generation; + std::size_t offset = 0; }; const EdgeSnapshot* edgeSnapshot(std::uint64_t begin, std::uint64_t end) const; void prepareEdgeCapture(std::uint64_t begin, std::uint64_t end); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index a111bd0ba8d..388a672f076 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -704,8 +704,11 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, layerGeneration.assign(trackedLayers, 0); layerPending.assign(trackedLayers, false); const auto surfaceEnd = descriptor.baseAddress + guestBytes; - if (descriptor.baseAddress % 65536 != 0) edgeSnapshots.push_back({descriptor.baseAddress, std::min(surfaceEnd, (descriptor.baseAddress + 65535) & ~std::uint64_t{65535}), {}, {}}); - if (surfaceEnd % 65536 != 0 && (edgeSnapshots.empty() || edgeSnapshots.back().end < surfaceEnd)) edgeSnapshots.push_back({std::max(descriptor.baseAddress, surfaceEnd & ~std::uint64_t{65535}), surfaceEnd, {}, {}}); + for (auto begin = descriptor.baseAddress; begin < surfaceEnd;) { + const auto end = std::min(surfaceEnd, (begin & ~std::uint64_t{65535}) + 65536); + edgeSnapshots.push_back({begin, end, {}, {}}); + begin = end; + } sliceLinearBytes = geometry.sliceLinearBytes; const auto linearBytes = sliceLinearBytes * arrayLayers; @@ -1369,7 +1372,7 @@ bool StorageTexture::Refresh() { const auto begin = layerBegin(unit); const auto bytes = layerBytes(unit); const bool edge = begin % 65536 != 0 || bytes != 65536; - if (edge && edgeSnapshot(begin, begin + bytes) != nullptr) return false; + if (edgeSnapshot(begin, begin + bytes) != nullptr) return false; return !edge || GuestMemory::StoredOver(begin, static_cast(bytes), layerGeneration[unit]); }; // A clear code -> uncompressed flip on an image with results pending: an unstamped pending @@ -1514,28 +1517,34 @@ void StorageTexture::prepareEdgeCapture(std::uint64_t begin, std::uint64_t end) } void StorageTexture::captureEdges(VkCommandBuffer commands, const HostImport& import, VkBuffer source, std::span copies, std::span> ranges) { - bool barrier = false; - auto* recorder = Recorder::Active(); + std::vector selected; + std::size_t total = 0; for (auto& edge : edgeSnapshots) { if (std::none_of(ranges.begin(), ranges.end(), [&](const auto& range) { return range.first <= edge.begin && edge.end <= range.second; })) continue; edge.bytes.reset(); if (!GuestMemory::Watched(edge.begin, static_cast(edge.end - edge.begin))) continue; - auto snapshot = std::make_shared(context, static_cast(edge.end - edge.begin), VK_BUFFER_USAGE_TRANSFER_DST_BIT); - if (!barrier) { - RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); - barrier = true; - } - if (recorder != nullptr) recorder->Keep(snapshot, static_cast(edge.end - edge.begin)); + selected.push_back(&edge); + total += static_cast(edge.end - edge.begin); + } + if (selected.empty()) return; + auto snapshot = std::make_shared(context, total, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + if (auto* recorder = Recorder::Active()) recorder->Keep(snapshot, total); + std::size_t offset = 0; + for (auto* entry : selected) { + auto& edge = *entry; for (const auto& copy : copies) { const auto begin = import.base + copy.dstOffset; const auto from = std::max(begin, edge.begin); const auto to = std::min(begin + copy.size, edge.end); - if (from < to) CopyBuffer(context, commands, source, copy.srcOffset + from - begin, snapshot->Handle(), from - edge.begin, to - from); + if (from < to) CopyBuffer(context, commands, source, copy.srcOffset + from - begin, snapshot->Handle(), offset + from - edge.begin, to - from); } - edge.bytes = std::move(snapshot); + edge.bytes = snapshot; edge.allocation = import.range; + edge.offset = offset; + offset += static_cast(edge.end - edge.begin); } - if (barrier) RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); } void StorageTexture::captureCpuEdges(std::span> ranges) { @@ -1545,21 +1554,35 @@ void StorageTexture::captureCpuEdges(std::span> ranges, std::span source, std::span> allocations) { + std::vector>> selected; + std::size_t total = 0; for (auto& edge : edgeSnapshots) { if (std::none_of(ranges.begin(), ranges.end(), [&](const auto& range) { return range.first <= edge.begin && edge.end <= range.second; })) continue; edge.bytes.reset(); if (!GuestMemory::Watched(edge.begin, static_cast(edge.end - edge.begin))) continue; const auto owner = std::find_if(allocations.begin(), allocations.end(), [&](const auto& range) { return range->readable && range->writable && range->address <= edge.begin && edge.end <= range->address + range->bytes; }); if (owner == allocations.end()) continue; - auto snapshot = std::make_shared(context, static_cast(edge.end - edge.begin), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + selected.emplace_back(&edge, *owner); + total += static_cast(edge.end - edge.begin); + } + if (selected.empty()) return; + auto snapshot = std::make_shared(context, total, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + std::size_t offset = 0; + for (const auto& [entry, owner] : selected) { + auto& edge = *entry; + const auto begin = offset; + const auto length = static_cast(edge.end - edge.begin); + offset += length; + auto destination = snapshot->Bytes().subspan(begin, length); if (source.empty()) { - if (GuestMemory::CopyMapped(edge.begin, snapshot->Bytes()) != GuestMemory::Compare::Equal) continue; + if (GuestMemory::CopyMapped(edge.begin, destination) != GuestMemory::Compare::Equal) continue; } else { - const auto bytes = source.subspan(static_cast(edge.begin - descriptor.baseAddress), static_cast(edge.end - edge.begin)); - std::memcpy(snapshot->Bytes().data(), bytes.data(), bytes.size()); + const auto bytes = source.subspan(static_cast(edge.begin - descriptor.baseAddress), length); + std::memcpy(destination.data(), bytes.data(), bytes.size()); } - edge.bytes = std::move(snapshot); - edge.allocation = *owner; + edge.bytes = snapshot; + edge.allocation = owner; + edge.offset = begin; } } @@ -1594,8 +1617,9 @@ void StorageTexture::finishEdgeMerges(std::span merges) { } edge.bytes->Invalidate(); merge.result->Invalidate(); - const auto baseline = edge.bytes->Bytes(); - const auto result = merge.result->Bytes(); + const auto length = static_cast(edge.end - edge.begin); + const auto baseline = edge.bytes->Bytes().subspan(edge.offset, length); + const auto result = merge.result->Bytes().subspan(merge.offset, length); GuestMemory::MergeChangedCommitted(edge.begin, result, baseline, merge.generation); invalidateEdges(edge.begin, edge.end); } @@ -2255,7 +2279,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); recorder->Keep(linear, linear->Size()); recorder->Keep(tiledScratch, tiledScratch->Size()); - for (const auto& merge : merges) recorder->Keep(merge.result, static_cast(merge.baseline.end - merge.baseline.begin)); + if (!merges.empty()) recorder->Keep(merges.front().result, merges.front().result->Bytes().size()); for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab); for (const auto& slab : padding.slabs) recorder->Keep(slab); if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); @@ -2330,7 +2354,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp for (const auto& merge : merges) { const auto begin = std::max(merge.baseline.begin - descriptor.baseAddress, window.tiledBegin); const auto end = std::min(merge.baseline.end - descriptor.baseAddress, window.tiledEnd); - if (begin < end) CopyBuffer(context, commands, tiledScratch->Handle(), scratchPositions[i] + begin - window.tiledBegin, merge.result->Handle(), descriptor.baseAddress + begin - merge.baseline.begin, end - begin); + if (begin < end) CopyBuffer(context, commands, tiledScratch->Handle(), scratchPositions[i] + begin - window.tiledBegin, merge.result->Handle(), merge.offset + descriptor.baseAddress + begin - merge.baseline.begin, end - begin); } } } @@ -3623,17 +3647,30 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { skippedAny = true; skippedLayer[layer] = true; ++skipped; - const auto* baseline = edge ? edgeSnapshot(from, to) : nullptr; + const auto* baseline = edgeSnapshot(from, to); if (baseline == nullptr || GuestMemory::DriverStoredOver(from, static_cast(to - from), layerGeneration[layer])) { invalidateEdges(from, to); continue; } - merges.push_back({*baseline, std::make_shared(context, static_cast(to - from), VK_BUFFER_USAGE_TRANSFER_DST_BIT), layerGeneration[layer]}); + auto clipped = *baseline; + clipped.offset += static_cast(from - clipped.begin); + clipped.begin = from; + clipped.end = to; + merges.push_back({std::move(clipped), {}, layerGeneration[layer]}); } if (!keep.empty() && keep.back().second == from) keep.back().second = to; else keep.emplace_back(from, to); } } + if (!merges.empty()) { + std::size_t total = 0; + for (auto& merge : merges) { + merge.offset = total; + total += static_cast(merge.baseline.end - merge.baseline.begin); + } + auto result = std::make_shared(context, total, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + for (auto& merge : merges) merge.result = result; + } // Debug aid: APS5_TRACE_FLUSH also names the blocks a write-back leaves to the CPU. static const bool traceKept = std::getenv("APS5_TRACE_FLUSH") != nullptr; if (traceKept && skippedAny) { @@ -3723,7 +3760,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); recorder->Keep(linear, linear->Size()); recorder->Keep(tiledScratch, tiledScratch->Size()); - for (const auto& merge : merges) recorder->Keep(merge.result, static_cast(merge.baseline.end - merge.baseline.begin)); + if (!merges.empty()) recorder->Keep(merges.front().result, merges.front().result->Bytes().size()); for (const auto& slab : padding.slabs) recorder->Keep(slab); // The image itself must outlive the recorded retile: the cache may evict it right after. if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); @@ -3776,7 +3813,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { std::vector copies; for (const auto& [from, to] : directRanges) copies.push_back({from - descriptor.baseAddress, importOffset + (from - descriptor.baseAddress), to - from}); if (!copies.empty()) context.Function("vkCmdCopyBuffer")(commands, tiledScratch->Handle(), import->buffer, static_cast(copies.size()), copies.data()); - for (const auto& merge : merges) CopyBuffer(context, commands, tiledScratch->Handle(), merge.baseline.begin - descriptor.baseAddress, merge.result->Handle(), 0, merge.baseline.end - merge.baseline.begin); + for (const auto& merge : merges) CopyBuffer(context, commands, tiledScratch->Handle(), merge.baseline.begin - descriptor.baseAddress, merge.result->Handle(), merge.offset, merge.baseline.end - merge.baseline.begin); captureEdges(commands, *import, tiledScratch->Handle(), copies, directRanges); } VkImageMemoryBarrier backToGeneral = toSource; @@ -3910,7 +3947,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { GuestMemory::WriteChangedCommitted(from, current.subspan(offset, length), std::span(original).subspan(offset, length)); std::memcpy(original.data() + offset, current.data() + offset, length); } - for (const auto& merge : merges) std::memcpy(merge.result->Bytes().data(), current.data() + (merge.baseline.begin - descriptor.baseAddress), static_cast(merge.baseline.end - merge.baseline.begin)); + for (const auto& merge : merges) std::memcpy(merge.result->Bytes().data() + merge.offset, current.data() + (merge.baseline.begin - descriptor.baseAddress), static_cast(merge.baseline.end - merge.baseline.begin)); finishEdgeMerges(merges); captureCpuEdges(directRanges); // The texels now hold the whole image, so later reads must see them rather than a fast clear diff --git a/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp b/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp index f90ab761640..863d1874296 100644 --- a/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/StorageEdgeWriteBack.cpp @@ -352,9 +352,126 @@ void fullBlockTests(const Context& context, Recorder& recorder) { surface.Draw(128); surface.CpuStore(32, Initial); surface.WriteBack(); - std::map kept; - for (std::size_t at = 0; at < UnitBytes; ++at) kept.emplace(at, Initial); - surface.CheckMemory(std::byte{128}, kept, "an aligned CPU-stamped block changed ownership"); + surface.CheckMemory(std::byte{128}, {}, "a same-value CPU store discarded a full block of GPU results"); +} + +void fullBlockMergeTests(const Context& context, Recorder& recorder) { + for (const bool cpu : {false, true}) { + for (const bool refresh : {false, true}) { + for (const bool collected : {false, true}) { + Surface surface(context, recorder, 0, 512, 512, false, cpu); + surface.Draw(128); + const auto edited = UnitBytes + 17; + if (collected) { + surface.CpuStore(32, Initial); + surface.CpuStore(edited, std::byte{0x31}); + } else { + surface.CpuStoreUncollected(32, Initial); + surface.CpuStoreUncollected(edited, std::byte{0x31}); + } + if (refresh) { + surface.Refresh(); + surface.CheckImage(std::byte{128}, {{std::byte{0x31}, 1}}, "full-block refresh lost a CPU edit or GPU results"); + } else { + surface.WriteBack(); + } + surface.CheckMemory(std::byte{128}, {{edited, std::byte{0x31}}}, "full-block write-back lost mixed CPU/GPU bytes"); + surface.Refresh(); + surface.Draw(64); + surface.CpuStore(32, std::byte{128}); + surface.CpuStore(edited, std::byte{0x31}); + surface.WriteBack(); + surface.CheckMemory(std::byte{64}, {}, "a second full-block merge reused an old baseline"); + } + } + } +} + +void fullBlockGuardTests(const Context& context, Recorder& recorder) { + for (const bool cpu : {false, true}) { + Surface surface(context, recorder, 0, 256, 256); + Require(surface.Size() == UnitBytes, "full-block ownership fixture is not one block"); + surface.Draw(128); + if (cpu) surface.CpuStore(17, std::byte{0x31}); + surface.DriverStore(64, Initial); + surface.WriteBack(); + const std::map edits = cpu ? std::map{{17, std::byte{0x31}}} : std::map{}; + surface.CheckMemory(Initial, edits, "a newer same-value driver store lost full-block ownership"); + } + Surface surface(context, recorder, 0, 256, 256); + surface.Draw(128); + surface.Reregister(); + surface.CpuStore(32, Initial); + surface.WriteBack(); + surface.CheckMemory(Initial, {}, "a full-block baseline overwrote a replacement allocation"); + Surface unchanged(context, recorder, 0, 256, 256); + unchanged.Draw(205); + unchanged.CpuStore(17, std::byte{0x31}); + unchanged.WriteBack(); + unchanged.CheckMemory(Initial, {{17, std::byte{0x31}}}, "unchanged full-block GPU bytes overwrote a CPU edit"); +} + +void fullBlockSourceTests(const Context& context, Recorder& recorder) { + { + Surface surface(context, recorder, 0, 512, 512, true); + surface.Draw(128); + surface.CpuStore(UnitBytes + 32, std::byte{0x77}); + surface.WriteBack(); + surface.CheckMemory(std::byte{128}, {}, "a full-block baseline preceded an ordered upload-source write"); + } + { + Surface surface(context, recorder, 0, 512, 512, false, false, true); + surface.Draw(128, true); + surface.CpuStoreUncollected(UnitBytes + 17, std::byte{0x31}); + surface.WriteBack(); + surface.CheckMemory(std::byte{128}, {{UnitBytes + 17, std::byte{0x31}}}, "a pending upload lost a full-block CPU edit"); + } + { + Surface surface(context, recorder, 0); + surface.Fill(128, true); + surface.CpuStoreUncollected(UnitBytes + 17, std::byte{0x31}); + surface.WriteBack(); + surface.CheckMemory(std::byte{128}, {{UnitBytes + 17, std::byte{0x31}}}, "a pending clear lost a full-block CPU edit"); + } + { + Surface source(context, recorder, 0); + Surface destination(context, recorder, 0); + source.Draw(128); + destination.CopyFrom(source, true); + destination.CpuStoreUncollected(UnitBytes + 17, std::byte{0x31}); + destination.WriteBack(); + destination.CheckMemory(std::byte{128}, {{UnitBytes + 17, std::byte{0x31}}}, "a pending copy lost a full-block CPU edit"); + } + { + Surface surface(context, recorder, 0); + surface.CreateAlias(); + surface.Draw(128); + surface.CpuStore(UnitBytes + 17, std::byte{0x31}); + surface.RefreshAlias(); + surface.CheckImage(std::byte{128}, {{std::byte{0x31}, 1}}, "an alias discarded mixed full-block results", true); + surface.CheckMemory(std::byte{128}, {{UnitBytes + 17, std::byte{0x31}}}, "an alias uploaded stale full-block bytes"); + } + { + Surface surface(context, recorder, 0); + surface.CreateAlias(); + surface.Draw(128); + surface.RefreshAlias(); + surface.Draw(64, false, true); + surface.CpuStore(UnitBytes + 32, Initial, true); + surface.WriteBack(true); + surface.CheckMemory(std::byte{64}, {}, "an alias borrow lost a full-block physical baseline"); + } +} + +void largeFullBlockTests(const Context& context, Recorder& recorder) { + Surface surface(context, recorder, 0, 2048, 4096); + Require(surface.Size() == 8 * 1024 * 1024, "large full-block fixture is not 8 MiB"); + surface.Draw(128); + for (std::size_t at = 0; at < surface.Size(); at += UnitBytes) surface.CpuStore(at + 32, Initial); + const auto edited = surface.Size() / 2 + 17; + surface.CpuStore(edited, std::byte{0x31}); + surface.WriteBack(); + surface.CheckMemory(std::byte{128}, {{edited, std::byte{0x31}}}, "a large image lost GPU results in CPU-stamped full blocks"); } void uploadOrderTests(const Context& context, Recorder& recorder) { @@ -620,8 +737,9 @@ void edgePrecedenceTests(const Context& context, Recorder& recorder) { } -int main() { +int main(int argc, char** argv) { try { + Require(argc == 1 || (argc == 2 && std::string(argv[1]) == "--full-blocks"), "unknown storage write-back test selection"); std::unique_ptr device; try { device = std::make_unique(); @@ -640,6 +758,15 @@ int main() { context.detiler = &detiler; Recorder recorder(context); recorder.Activate(); + if (argc == 2) { + fullBlockMergeTests(context, recorder); + fullBlockGuardTests(context, recorder); + fullBlockSourceTests(context, recorder); + largeFullBlockTests(context, recorder); + recorder.Sync(); + std::cout << "Storage full-block write-back tests passed\n"; + return 0; + } noOpEdgeTests(context, recorder); mixedEdgeTests(context, recorder); shortSurfaceTests(context, recorder); diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 9f2cfd8318b..b5b1e129cfa 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -240,7 +240,7 @@ Throughout the project, every function at every stage either **does exactly what - [sceKernelWriteThrottlingStatus](../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) reports no write throttling: the host SSD has no write budget, so the first qword of the 32-byte status is set to `UINT32_MAX` and the rest is zeroed. PPSA17221 reads only that qword: it multiplies it by 64 KiB and compares the product with byte thresholds up to 0x4AF00000 to pick one of five tiers. When the call fails the title assumes 0x4B000001 bytes, which selects the same top tier as `UINT32_MAX`. What the value measures on hardware (a remaining budget or a rate) and the meaning of the other three qwords are unknown - [Wide-character conversions](../../core/libs/prx/libc/src/Strings.cpp) (libc) - `wcsrtombs` and [`swprintf`](../../core/libs/prx/libc/src/Formatting.cpp) still pass guest strings to the host's `wchar_t` functions. The guest's `wchar_t` is 16 bits, so on Linux, where the host's is 32 bits, they misread and miswrite guest strings. The wide `wcsto*` functions parse the ASCII prefix of the guest string, as the C locale accepts only ASCII white space and digits - [SPIRV-Tools optimization](../../core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp) skips CCP and Simplification for modules with storage-only 8- or 16-bit types because the pinned folder produces illegal narrow constants. This also disables unrelated constant folding; the restricted full pipeline mirrors the pinned performance recipe. -- [Storage-image partial-edge writeback](../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) uses an emulator policy whose CPU/GPU precedence has not been measured on console hardware. For each byte, an unchanged GPU result keeps the destination; a changed GPU result replaces it only while it equals the immutable baseline. A same-value CPU write or a change back to the baseline permits the GPU result. A newer ordered driver store vetoes the affected partial-edge merge, even when it writes the same value. The baseline owner must match a current readable and writable allocation range, leased through the merge. Full 64 KiB blocks retain their existing ownership rule. These checks were exercised on Linux llvmpipe with the test-only page-watch backend; native write-watch races remain unverified. The allocation check applies to edge merges: ordinary writeback without a newer CPU stamp can still write into re-registered memory. +- [Storage-image writeback](../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) uses an emulator policy whose CPU/GPU precedence has not been measured on console hardware. Immutable baselines cover every watched 64 KiB block, including partial edges. For each byte, an unchanged GPU result keeps the destination; a changed GPU result replaces it only while it equals the baseline. A same-value CPU write or a change back to the baseline permits the GPU result. A newer ordered driver store vetoes the affected block merge, even when it writes the same value. The baseline owner must match a current readable and writable allocation range, leased through the merge. Blocks share packed snapshot and result buffers; baseline memory scales with the watched image size, and capture copies guest bytes on upload. Direct writeback bypasses unit shadows for snapshot-covered blocks. The full-block extension needs native Linux and Windows validation, write-watch race checks, and memory/performance measurements. Interactive Silksong rendering remains unverified. The allocation check applies to stamped block merges: ordinary writeback without a newer CPU stamp can still write into re-registered memory. - Windows host-pointer imports are compared instead of write-watched: a driver can mark imported pages dirty after the synchronous import window. [GuestMemory](../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. This preserves CPU edits through byte comparisons but disables snapshot reuse and unit shadows for excluded blocks; restoring tracking on import retirement without replacing the backing and measuring the comparison cost remain unimplemented. - [sceNetSendmsg and sceNetRecvmsg](../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) copy the scatter/gather list through one host buffer and accept only the flags `sceNetSend`/`sceNetRecv` accept. Sending control data throws; a receive never returns control data (`msg_controllen` 0), since no socket option that queues it is supported - [Additional content](../../core/libs/prx/libSceAppContent/Export.cpp) (libSceAppContent) is not installed or mounted: `sceAppContentAddcontMount` answers `NOT_FOUND` for every entitlement label and `sceAppContentAddcontUnmount` for every mount point, as no add-content mount is ever handed out; `sceAppContentGetAddcontInfo` answers `DRM_NO_ENTITLEMENT` for every label and `sceAppContentGetAddcontInfoList` reports 0 entries From d7dee4229bc58b0cfc67739b2d75a4b8be3bc471 Mon Sep 17 00:00:00 2001 From: ToyuAU Date: Fri, 9 Oct 2026 14:33:58 +1000 Subject: [PATCH 4/4] docs(agc): record full-block validation and ownership limits --- docs/dev/TechnicalDebt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index b5b1e129cfa..2c07cda6f11 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -240,7 +240,7 @@ Throughout the project, every function at every stage either **does exactly what - [sceKernelWriteThrottlingStatus](../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) reports no write throttling: the host SSD has no write budget, so the first qword of the 32-byte status is set to `UINT32_MAX` and the rest is zeroed. PPSA17221 reads only that qword: it multiplies it by 64 KiB and compares the product with byte thresholds up to 0x4AF00000 to pick one of five tiers. When the call fails the title assumes 0x4B000001 bytes, which selects the same top tier as `UINT32_MAX`. What the value measures on hardware (a remaining budget or a rate) and the meaning of the other three qwords are unknown - [Wide-character conversions](../../core/libs/prx/libc/src/Strings.cpp) (libc) - `wcsrtombs` and [`swprintf`](../../core/libs/prx/libc/src/Formatting.cpp) still pass guest strings to the host's `wchar_t` functions. The guest's `wchar_t` is 16 bits, so on Linux, where the host's is 32 bits, they misread and miswrite guest strings. The wide `wcsto*` functions parse the ASCII prefix of the guest string, as the C locale accepts only ASCII white space and digits - [SPIRV-Tools optimization](../../core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp) skips CCP and Simplification for modules with storage-only 8- or 16-bit types because the pinned folder produces illegal narrow constants. This also disables unrelated constant folding; the restricted full pipeline mirrors the pinned performance recipe. -- [Storage-image writeback](../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) uses an emulator policy whose CPU/GPU precedence has not been measured on console hardware. Immutable baselines cover every watched 64 KiB block, including partial edges. For each byte, an unchanged GPU result keeps the destination; a changed GPU result replaces it only while it equals the baseline. A same-value CPU write or a change back to the baseline permits the GPU result. A newer ordered driver store vetoes the affected block merge, even when it writes the same value. The baseline owner must match a current readable and writable allocation range, leased through the merge. Blocks share packed snapshot and result buffers; baseline memory scales with the watched image size, and capture copies guest bytes on upload. Direct writeback bypasses unit shadows for snapshot-covered blocks. The full-block extension needs native Linux and Windows validation, write-watch race checks, and memory/performance measurements. Interactive Silksong rendering remains unverified. The allocation check applies to stamped block merges: ordinary writeback without a newer CPU stamp can still write into re-registered memory. +- [Storage-image writeback](../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) uses an emulator policy whose CPU/GPU precedence has not been measured on console hardware. Immutable baselines cover every watched 64 KiB block, including partial edges. For each byte, an unchanged GPU result keeps the destination; a changed GPU result replaces it only while it equals the baseline. A same-value CPU write or a change back to the baseline permits the GPU result. A newer ordered driver store vetoes the affected block merge, even when it writes the same value. The baseline owner must match a current readable and writable allocation range, leased through the merge. Baseline comparisons cannot identify guest allocator reuse inside one registered allocation; storing results before guest GPU completion becomes visible remains necessary for that lifetime case. Blocks share packed snapshot and result buffers; baseline memory scales with the watched image size, and capture copies guest bytes on upload. Direct writeback bypasses unit shadows for snapshot-covered blocks. Edge and full-block regressions execute on Linux Mesa llvmpipe with a test-only page-watch backend. Native write-watch races, Windows GPU execution, and memory/performance measurements remain unverified. Interactive Silksong rendering remains unverified. The allocation check applies to stamped block merges: ordinary writeback without a newer CPU stamp can still write into re-registered memory. - Windows host-pointer imports are compared instead of write-watched: a driver can mark imported pages dirty after the synchronous import window. [GuestMemory](../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. This preserves CPU edits through byte comparisons but disables snapshot reuse and unit shadows for excluded blocks; restoring tracking on import retirement without replacing the backing and measuring the comparison cost remain unimplemented. - [sceNetSendmsg and sceNetRecvmsg](../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) copy the scatter/gather list through one host buffer and accept only the flags `sceNetSend`/`sceNetRecv` accept. Sending control data throws; a receive never returns control data (`msg_controllen` 0), since no socket option that queues it is supported - [Additional content](../../core/libs/prx/libSceAppContent/Export.cpp) (libSceAppContent) is not installed or mounted: `sceAppContentAddcontMount` answers `NOT_FOUND` for every entitlement label and `sceAppContentAddcontUnmount` for every mount point, as no add-content mount is ever handed out; `sceAppContentGetAddcontInfo` answers `DRM_NO_ENTITLEMENT` for every label and `sceAppContentGetAddcontInfoList` reports 0 entries