diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 1df29ca2bc2..5ad209e46ba 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -738,7 +738,8 @@ if(BUILD_TESTING AND WIN32) configure_windows_unwind(agc_page_state_tests) add_test(NAME agc_page_states COMMAND agc_page_state_tests) add_test(NAME agc_page_state_metadata COMMAND agc_page_state_tests --metadata) - set_tests_properties(agc_page_states agc_page_state_metadata PROPERTIES + add_test(NAME agc_page_state_runs COMMAND agc_page_state_tests --runs) + set_tests_properties(agc_page_states agc_page_state_metadata agc_page_state_runs PROPERTIES TIMEOUT 30 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index 960cf4a11cb..651c2e42b84 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -364,6 +364,7 @@ namespace { // callback), and address-based builds verify every image section on each build, so without the table // each of those verifications was a VirtualQuery. APS5_NO_IMAGE_PAGE_CACHE=1 leaves the image uncached. constexpr std::size_t PageBytes = 4096; +constexpr std::size_t BlockPages = 64; constexpr std::uint8_t PageReadable = 1; constexpr std::uint8_t PageWritable = 2; constexpr std::uint8_t PageInaccessible = 4; @@ -373,36 +374,48 @@ struct PageSpan { std::size_t size = 0; // Per page: access flags, zero while unknown. std::uint8_t* pages = nullptr; + std::uint8_t* blocks = nullptr; static constexpr std::size_t ChunkEntries = 65536; - struct alignas(8) Chunk { std::array entries{}; }; + struct alignas(8) Chunk { + std::array entries{}; + std::array blocks{}; + }; std::atomic* chunks = nullptr; - std::uint8_t* chunkData(std::size_t index, bool create) const { + Chunk* chunkAt(std::size_t index, bool create) const { auto* chunk = chunks[index].load(std::memory_order_acquire); if (chunk == nullptr && create) { auto fresh = std::unique_ptr(new (std::nothrow) Chunk{}); if (fresh != nullptr && chunks[index].compare_exchange_strong(chunk, fresh.get(), std::memory_order_acq_rel, std::memory_order_acquire)) chunk = fresh.release(); } - return chunk == nullptr ? nullptr : chunk->entries.data(); + return chunk; } std::uint8_t* entry(std::size_t index, bool create) const { if (chunks == nullptr) return pages + index; - auto* data = chunkData(index / ChunkEntries, create); - return data == nullptr ? nullptr : data + index % ChunkEntries; + auto* chunk = chunkAt(index / ChunkEntries, create); + return chunk == nullptr ? nullptr : chunk->entries.data() + index % ChunkEntries; + } + + std::uint8_t* blockEntry(std::size_t block, bool create) const { + if (chunks == nullptr) return blocks + block; + auto* chunk = chunkAt(block * BlockPages / ChunkEntries, create); + return chunk == nullptr ? nullptr : chunk->blocks.data() + block % (ChunkEntries / BlockPages); } bool allocate() { if (size == 0) return false; - const auto count = (size / PageBytes + 8) & ~static_cast(7); + const auto count = (size / PageBytes + BlockPages) & ~(BlockPages - 1); #ifdef _WIN32 const auto chunkCount = (count + ChunkEntries - 1) / ChunkEntries; chunks = new (std::nothrow) std::atomic[chunkCount]{}; if (chunks == nullptr) size = 0; return chunks != nullptr; #else - pages = static_cast(std::calloc(count, 1)); + const auto blockCount = (count / BlockPages + 8) & ~static_cast(7); + pages = static_cast(std::calloc(count + blockCount, 1)); if (pages == nullptr) size = 0; + else blocks = pages + count; return pages != nullptr; #endif } @@ -411,24 +424,63 @@ struct PageSpan { return size != 0 && address >= base && address - base < size; } + static std::uint8_t loadByte(const std::uint8_t* at) { + return std::atomic_ref(*const_cast(at)).load(std::memory_order_relaxed); + } + + static void storeByte(std::uint8_t* at, std::uint8_t value) { + std::atomic_ref(*at).store(value, std::memory_order_relaxed); + } + + static std::uint64_t loadWide(const std::uint8_t* at) { + return std::atomic_ref(*reinterpret_cast(const_cast(at))).load(std::memory_order_relaxed); + } + std::uint8_t load(std::uintptr_t address) const { auto* value = entry((address - base) / PageBytes, false); - return value == nullptr ? 0 : std::atomic_ref(*value).load(std::memory_order_relaxed); + return value == nullptr ? 0 : loadByte(value); + } + + std::uint8_t common(std::size_t block) const { + const auto* at = entry(block * BlockPages, false); + if (at == nullptr) return 0; + const auto value = loadByte(at); + const std::uint64_t wide = 0x0101010101010101ull * value; + for (std::size_t offset = 0; offset < BlockPages; offset += 8) { + if (loadWide(at + offset) != wide) return 0; + } + return value; + } + + void settle(std::size_t first, std::size_t last) { + std::atomic_thread_fence(std::memory_order_seq_cst); + for (auto block = first / BlockPages; block <= (last - 1) / BlockPages; ++block) { + const auto value = common(block); + auto* summary = blockEntry(block, false); + if (summary == nullptr || loadByte(summary) == value) continue; + storeByte(summary, value); + if (value == 0) continue; + std::atomic_thread_fence(std::memory_order_seq_cst); + if (common(block) != value) storeByte(summary, 0); + } } void store(std::uintptr_t address, std::uint8_t value) { - if (auto* at = entry((address - base) / PageBytes, value != 0 && value != PageInaccessible)) std::atomic_ref(*at).store(value, std::memory_order_relaxed); + store(address, address + PageBytes, value); } void store(std::uintptr_t first, std::uintptr_t last, std::uint8_t value) { - auto index = (first - base) / PageBytes; - const auto stop = (last - 1 - base) / PageBytes + 1; - while (index < stop) { - const auto next = std::min(stop, (index / ChunkEntries + 1) * ChunkEntries); + if (first >= last) return; + const auto begin = (first - base) / PageBytes; + const auto end = (last - 1 - base) / PageBytes + 1; + auto index = begin; + while (index < end) { + const auto next = std::min(end, (index / ChunkEntries + 1) * ChunkEntries); if (auto* at = entry(index, value != 0 && value != PageInaccessible)) { - for (; index < next; ++index, ++at) std::atomic_ref(*at).store(value, std::memory_order_relaxed); + for (; index < next; ++index, ++at) storeByte(at, value); } else index = next; } + settle(begin, end); } // End of the run of pages equal to `value` that starts at `address`, at most `limit`. @@ -443,11 +495,23 @@ struct PageSpan { index = std::min(stop, (index / ChunkEntries + 1) * ChunkEntries); continue; } - if (index % 8 == 0 && index + 8 <= stop && std::atomic_ref(*reinterpret_cast(at)).load(std::memory_order_relaxed) == wide) { + if (index % BlockPages == 0) { + const auto block = index / BlockPages; + const auto* summary = blockEntry(block, false); + if (block % 8 == 0 && index + 8 * BlockPages <= stop && loadWide(summary) == wide) { + index += 8 * BlockPages; + continue; + } + if (index + BlockPages <= stop && loadByte(summary) == value) { + index += BlockPages; + continue; + } + } + if (index % 8 == 0 && index + 8 <= stop && loadWide(at) == wide) { index += 8; continue; } - if (std::atomic_ref(*at).load(std::memory_order_relaxed) != value) break; + if (loadByte(at) != value) break; ++index; } return std::min(limit, base + index * PageBytes); @@ -498,7 +562,7 @@ struct PageStates { if (span->size == 0 || bytes == 0 || address >= span->base + span->size || address + bytes <= span->base) continue; const auto last = std::min(address + bytes, span->base + span->size); for (auto at = std::max(address, span->base) & ~static_cast(PageBytes - 1); at < last; at += PageBytes) { - if (span->load(at) == PageInaccessible) span->store(at, 0); + if (span->load(at) == PageInaccessible) span->store(at, at + PageBytes, 0); } } } diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp index 49cd994d6e4..cb61b1577b5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp @@ -24,6 +24,7 @@ void RunHeapMirrorRunUnwatchTests(const AgcDriver::Graphics::Context& context, c void RunGuestAllocationTests(); void RunGuestLeaseWaitTests(); void RunUnmappedGapTests(); +void RunPageBlockTests(); void RunColorTargetLayoutTests(); void RunLiveStackAccessTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index f94e25095f6..e6f721c7289 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -11715,6 +11715,7 @@ int main(int argc, char** argv) { Require(mock.live == 0, "BDA resources leaked Vulkan objects"); RunGuestAllocationTests(); RunUnmappedGapTests(); + RunPageBlockTests(); RunColorTargetLayoutTests(); RunLiveStackAccessTests(); RunTextureFormatTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp index 89d83afd29e..3a411f0e42a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp @@ -328,3 +328,94 @@ void RunUnmappedGapTests() { GuestArena::GuestArenaRelease_nid_postfix(spare, page); #endif } + +void RunPageBlockTests() { +#if defined(__linux__) + namespace GuestMemory = AgcDriver::GuestMemory; + using Ranges = std::vector>; + if (!GuestArena::GuestArenaAvailable_nid_postfix()) return; + const auto page = static_cast(sysconf(_SC_PAGESIZE)); + if (page != 4096) return; + constexpr std::size_t count = 1536; + const auto bytes = count * page; + auto* guest = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, 512 * page)); + Require(guest != nullptr && mmap(guest, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == guest, "cannot map the page block test pages"); + const auto base = reinterpret_cast(guest); + { + GuestAllocations::Mutation mutation; + mutation.Add(guest, bytes, true, true, true); + } + const auto protect = [&](std::size_t index, bool writable) { + GuestAllocations::Mutation mutation; + mutation.Protect(guest + index * page, page, true, writable, true, [&] { Require(mprotect(guest + index * page, page, writable ? PROT_READ | PROT_WRITE : PROT_READ) == 0, "cannot change a page block test page"); }); + }; + Require(GuestMemory::Accessible(guest, bytes, true), "mapped arena pages are not writable"); + Require(mprotect(guest + 700 * page, page, PROT_READ) == 0, "cannot protect a page block test page behind the registry"); + Require(GuestMemory::Accessible(guest, bytes, true), "a long run of checked arena pages was queried again instead of read from the page table"); + Require(mprotect(guest + 700 * page, page, PROT_READ | PROT_WRITE) == 0, "cannot restore a page block test page behind the registry"); + for (const std::size_t index : {std::size_t{700}, std::size_t{512}, std::size_t{1535}}) { + protect(index, false); + Require(!GuestMemory::Accessible(guest, bytes, true) && GuestMemory::Accessible(guest, bytes), "a page protected inside a long run of cached pages kept its old access"); + Ranges expected{{base, base + index * page}}; + if (index + 1 < count) expected.emplace_back(base + (index + 1) * page, base + bytes); + Require(GuestMemory::CommittedRanges(base, bytes, true) == expected, "the writable pages around a protected page inside a long run were misdescribed"); + Require(GuestMemory::CommittedRanges(base, bytes) == Ranges{{base, base + bytes}}, "the readable pages of a long run with one read-only page were misdescribed"); + protect(index, true); + Require(GuestMemory::DescribeCommitted(base, bytes, true).whole, "a long run of pages made writable again was not described whole"); + } + std::atomic done{false}; + std::atomic described{0}; + std::vector readers; + for (int reader = 0; reader < 2; ++reader) { + readers.emplace_back([&] { + while (!done.load(std::memory_order_relaxed)) { + static_cast(GuestMemory::DescribeCommitted(base, bytes, true)); + described.fetch_add(1, std::memory_order_relaxed); + } + }); + } + for (std::size_t round = 0; round < 2000 || described.load(std::memory_order_relaxed) < 2000; ++round) { + const auto index = 640 + round % 128; + protect(index, false); + protect(index, true); + } + protect(700, false); + done = true; + for (auto& reader : readers) reader.join(); + Require(GuestMemory::CommittedRanges(base, bytes, true) == Ranges{{base, base + 700 * page}, {base + 701 * page, base + bytes}}, "a page protected while other threads described its run is still reported writable"); + protect(700, true); + for (std::size_t round = 0; round < 256; ++round) { + const auto index = (1 + round % 22) * 64 + (round * 37) % 64; + Require(GuestMemory::DescribeCommitted(base, bytes, true).whole, "the pages of a block were not described whole before one of them flipped"); + std::atomic stop{false}; + std::array, 2> scans{}; + std::vector scanners; + for (auto& scanned : scans) { + scanners.emplace_back([&] { + while (!stop.load(std::memory_order_relaxed)) { + static_cast(GuestMemory::DescribeCommitted(base, bytes, true)); + scanned.fetch_add(1, std::memory_order_release); + } + }); + } + for (const auto& scanned : scans) { + while (scanned.load(std::memory_order_acquire) == 0) std::this_thread::yield(); + } + protect(index, false); + for (const auto& scanned : scans) { + const auto flipped = scanned.load(std::memory_order_acquire); + while (scanned.load(std::memory_order_acquire) < flipped + 2) std::this_thread::yield(); + } + stop = true; + for (auto& scanner : scanners) scanner.join(); + Require(GuestMemory::CommittedRanges(base, bytes, true) == Ranges{{base, base + index * page}, {base + (index + 1) * page, base + bytes}}, "a page flipped in a marked block while other threads ran over the block is still reported writable"); + protect(index, true); + } + { + GuestAllocations::Mutation mutation; + mutation.Unmap(guest, bytes, [](const void* pointer, std::size_t length, const void*, bool) { Require(munmap(const_cast(pointer), length) == 0, "cannot unmap the page block test pages"); }); + } + Require(!GuestMemory::Accessible(guest, page) && GuestMemory::CommittedRanges(base, bytes).empty(), "unmapped pages of a long run are still accessible"); + GuestArena::GuestArenaRelease_nid_postfix(guest, bytes); +#endif +} diff --git a/core/libs/prx/libSceAgcDriver/tests/PageStates.cpp b/core/libs/prx/libSceAgcDriver/tests/PageStates.cpp index fb92db485c9..d4e6add733e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/PageStates.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/PageStates.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #ifndef NOMINMAX #define NOMINMAX @@ -41,12 +42,12 @@ class Range { GuestArena::GuestArenaRelease_nid_postfix(data, bytes); } - std::byte* Commit(std::size_t offset) { + std::byte* Commit(std::size_t offset, std::size_t length = Unit) { auto* at = data + offset; - GuestArena::GuestArenaCommit_nid_postfix(at, Unit, PAGE_READWRITE, Unit); - std::memset(at, 0x5b, Unit); + GuestArena::GuestArenaCommit_nid_postfix(at, length, PAGE_READWRITE, Unit); + std::memset(at, 0x5b, length); GuestAllocations::Mutation mutation; - mutation.Add(at, Unit, true, true, true); + mutation.Add(at, length, true, true, true); committed.push_back(offset); return at; } @@ -130,12 +131,115 @@ void CheckBoundaries() { Require(Accessible(block, Unit, true), "access observer did not invalidate inaccessible state after commit"); std::puts("page-state boundaries, publication, holes and invalidation passed"); } + +void CheckRuns() { + constexpr std::size_t page = 4096; + constexpr std::size_t count = 1536; + constexpr auto bytes = count * page; + using Ranges = std::vector>; + Range range(3 * ChunkGuestBytes); + auto* guest = range.Commit(ChunkGuestBytes - 512 * page, bytes); + const auto base = reinterpret_cast(guest); + const auto expected = [&](std::size_t index, std::size_t length = 1) { + Ranges result; + if (index != 0) result.emplace_back(base, base + index * page); + if (index + length < count) result.emplace_back(base + (index + length) * page, base + bytes); + return result; + }; + const auto protect = [&](std::size_t index, bool writable) { + auto* at = guest + index * page; + GuestAllocations::Mutation mutation; + mutation.Protect(at, page, true, writable, true, [&] { + const DWORD protection = writable ? PAGE_READWRITE : PAGE_READONLY; + DWORD previous = 0; + Require(VirtualProtect(at, page, protection, &previous) != 0, "cannot protect run fixture page"); + GuestArena::GuestArenaSetProtection_nid_postfix(reinterpret_cast(at), page, protection); + }); + }; + const Ranges whole{{base, base + bytes}}; + Require(DescribeCommitted(base, bytes, true).whole, "run fixture was not fully writable"); + for (const std::size_t index : {0, 63, 64, 511, 512, 513, 700, 1023, 1024, 1535}) { + protect(index, false); + Require(!Accessible(guest, bytes, true) && Accessible(guest, bytes), "a protected page retained writable access in a summarized run"); + Require(CommittedRanges(base, bytes, true) == expected(index), "a block or chunk boundary retained a stale summary"); + Require(CommittedRanges(base, bytes) == whole, "readable ranges changed around a read-only page"); + Require(CommittedRanges(base + 1, bytes - 2, true) == [&] { + auto clipped = expected(index); + if (!clipped.empty() && clipped.front().first == base) ++clipped.front().first; + if (!clipped.empty() && clipped.back().second == base + bytes) --clipped.back().second; + return clipped; + }(), "an unaligned run crossed a protected page"); + protect(index, true); + Require(DescribeCommitted(base, bytes, true).whole, "restored run was not writable"); + } + auto* hole = guest + 512 * page; + GuestArena::GuestArenaReset_nid_postfix(hole, Unit); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast(hole), Unit); + Require(!Accessible(hole, page), "a decommitted page retained cached access"); + Require(CommittedRanges(base, bytes) == expected(512, Unit / page), "decommitted pages retained a block summary"); + GuestArena::GuestArenaCommit_nid_postfix(hole, Unit, PAGE_READWRITE, Unit); + Require(DescribeCommitted(base, bytes, true).whole, "recommit did not clear inaccessible state and its summary"); + std::atomic done{false}; + std::atomic epoch{0}; + std::atomic changed{0}; + std::array, 2> observed{}; + std::atomic errors{0}; + std::vector readers; + for (std::size_t reader = 0; reader < observed.size(); ++reader) { + readers.emplace_back([&, reader] { + while (!done.load(std::memory_order_acquire)) { + const auto phase = epoch.load(std::memory_order_acquire); + if (phase > observed[reader].load(std::memory_order_relaxed)) { + const auto index = changed.load(std::memory_order_relaxed); + if (CommittedRanges(base, bytes, true) != ((phase & 1) != 0 ? expected(index) : whole)) errors.fetch_add(1, std::memory_order_relaxed); + observed[reader].store(phase, std::memory_order_release); + } else static_cast(DescribeCommitted(base, bytes, true)); + } + }); + } + const auto publish = [&](std::size_t phase, std::size_t index) { + changed.store(index, std::memory_order_relaxed); + epoch.store(phase, std::memory_order_release); + for (auto& seen : observed) { + while (seen.load(std::memory_order_acquire) < phase) std::this_thread::yield(); + } + }; + for (std::size_t round = 0; round < 128; ++round) { + const auto index = (round * 37 + 511) % count; + protect(index, false); + publish(2 * round + 1, index); + protect(index, true); + publish(2 * round + 2, index); + } + for (std::size_t round = 0; round < 2000; ++round) { + const auto index = 448 + round % 128; + protect(index, false); + protect(index, true); + } + protect(512, false); + publish(257, 512); + done.store(true, std::memory_order_release); + for (auto& reader : readers) reader.join(); + Require(errors.load() == 0 && CommittedRanges(base, bytes, true) == expected(512), "concurrent scans retained a stale block summary"); + protect(512, true); + std::array copy{}; + Read(base, copy); + Require(std::all_of(copy.begin(), copy.end(), [](std::byte value) { return value == std::byte{0x5b}; }), "summary scans changed guest data"); + Range edge(2 * ChunkGuestBytes); + constexpr std::size_t edgePages = 512; + const auto edgeFirst = reinterpret_cast(edge.Commit(ChunkGuestBytes - edgePages * page, edgePages * page)); + const Ranges edgeRun{{edgeFirst, edgeFirst + edgePages * page}}; + Require(CommittedRanges(edgeFirst, edgePages * page, true) == edgeRun, "the run before an unallocated chunk was not writable"); + Require(!DescribeCommitted(edgeFirst, edgePages * page + Unit, true).whole && CommittedRanges(edgeFirst, edgePages * page + Unit, true) == edgeRun, "a run into an unallocated chunk was reported past its committed pages"); + std::puts("page-state block scans across 64-page, 512-page and sparse-chunk boundaries passed"); +} } int main(int argc, char** argv) { try { Require(GuestArena::GuestArenaAvailable_nid_postfix(), "page-state fixture needs the guest arena"); if (argc == 2 && std::string_view(argv[1]) == "--metadata") CheckMetadata(); + else if (argc == 2 && std::string_view(argv[1]) == "--runs") CheckRuns(); else CheckBoundaries(); return 0; } catch (const std::exception& error) {