perf(agc): cross uniform runs of cached guest pages 64 pages per load

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -738,7 +738,8 @@ if(BUILD_TESTING AND WIN32)
     configure_windows_unwind(agc_page_state_tests)
     add_test(NAME agc_page_states COMMAND agc_page_state_tests)
     add_test(NAME agc_page_state_metadata COMMAND agc_page_state_tests --metadata)
-    set_tests_properties(agc_page_states agc_page_state_metadata PROPERTIES
+    add_test(NAME agc_page_state_runs COMMAND agc_page_state_tests --runs)
+    set_tests_properties(agc_page_states agc_page_state_metadata agc_page_state_runs PROPERTIES
         TIMEOUT 30 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
 endif()
 

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -364,6 +364,7 @@ namespace {
 // callback), and address-based builds verify every image section on each build, so without the table
 // each of those verifications was a VirtualQuery. APS5_NO_IMAGE_PAGE_CACHE=1 leaves the image uncached.
 constexpr std::size_t PageBytes = 4096;
+constexpr std::size_t BlockPages = 64;
 constexpr std::uint8_t PageReadable = 1;
 constexpr std::uint8_t PageWritable = 2;
 constexpr std::uint8_t PageInaccessible = 4;
@@ -373,36 +374,48 @@ struct PageSpan {
     std::size_t size = 0;
     // Per page: access flags, zero while unknown.
     std::uint8_t* pages = nullptr;
+    std::uint8_t* blocks = nullptr;
     static constexpr std::size_t ChunkEntries = 65536;
-    struct alignas(8) Chunk { std::array<std::uint8_t, ChunkEntries> entries{}; };
+    struct alignas(8) Chunk {
+        std::array<std::uint8_t, ChunkEntries> entries{};
+        std::array<std::uint8_t, ChunkEntries / BlockPages> blocks{};
+    };
     std::atomic<Chunk*>* chunks = nullptr;
 
-    std::uint8_t* chunkData(std::size_t index, bool create) const {
+    Chunk* chunkAt(std::size_t index, bool create) const {
         auto* chunk = chunks[index].load(std::memory_order_acquire);
         if (chunk == nullptr && create) {
             auto fresh = std::unique_ptr<Chunk>(new (std::nothrow) Chunk{});
             if (fresh != nullptr && chunks[index].compare_exchange_strong(chunk, fresh.get(), std::memory_order_acq_rel, std::memory_order_acquire)) chunk = fresh.release();
         }
-        return chunk == nullptr ? nullptr : chunk->entries.data();
+        return chunk;
     }
 
     std::uint8_t* entry(std::size_t index, bool create) const {
         if (chunks == nullptr) return pages + index;
-        auto* data = chunkData(index / ChunkEntries, create);
-        return data == nullptr ? nullptr : data + index % ChunkEntries;
+        auto* chunk = chunkAt(index / ChunkEntries, create);
+        return chunk == nullptr ? nullptr : chunk->entries.data() + index % ChunkEntries;
+    }
+
+    std::uint8_t* blockEntry(std::size_t block, bool create) const {
+        if (chunks == nullptr) return blocks + block;
+        auto* chunk = chunkAt(block * BlockPages / ChunkEntries, create);
+        return chunk == nullptr ? nullptr : chunk->blocks.data() + block % (ChunkEntries / BlockPages);
     }
 
     bool allocate() {
         if (size == 0) return false;
-        const auto count = (size / PageBytes + 8) & ~static_cast<std::size_t>(7);
+        const auto count = (size / PageBytes + BlockPages) & ~(BlockPages - 1);
 #ifdef _WIN32
         const auto chunkCount = (count + ChunkEntries - 1) / ChunkEntries;
         chunks = new (std::nothrow) std::atomic<Chunk*>[chunkCount]{};
         if (chunks == nullptr) size = 0;
         return chunks != nullptr;
 #else
-        pages = static_cast<std::uint8_t*>(std::calloc(count, 1));
+        const auto blockCount = (count / BlockPages + 8) & ~static_cast<std::size_t>(7);
+        pages = static_cast<std::uint8_t*>(std::calloc(count + blockCount, 1));
         if (pages == nullptr) size = 0;
+        else blocks = pages + count;
         return pages != nullptr;
 #endif
     }
@@ -411,24 +424,63 @@ struct PageSpan {
         return size != 0 && address >= base && address - base < size;
     }
 
+    static std::uint8_t loadByte(const std::uint8_t* at) {
+        return std::atomic_ref<std::uint8_t>(*const_cast<std::uint8_t*>(at)).load(std::memory_order_relaxed);
+    }
+
+    static void storeByte(std::uint8_t* at, std::uint8_t value) {
+        std::atomic_ref<std::uint8_t>(*at).store(value, std::memory_order_relaxed);
+    }
+
+    static std::uint64_t loadWide(const std::uint8_t* at) {
+        return std::atomic_ref<std::uint64_t>(*reinterpret_cast<std::uint64_t*>(const_cast<std::uint8_t*>(at))).load(std::memory_order_relaxed);
+    }
+
     std::uint8_t load(std::uintptr_t address) const {
         auto* value = entry((address - base) / PageBytes, false);
-        return value == nullptr ? 0 : std::atomic_ref<std::uint8_t>(*value).load(std::memory_order_relaxed);
+        return value == nullptr ? 0 : loadByte(value);
+    }
+
+    std::uint8_t common(std::size_t block) const {
+        const auto* at = entry(block * BlockPages, false);
+        if (at == nullptr) return 0;
+        const auto value = loadByte(at);
+        const std::uint64_t wide = 0x0101010101010101ull * value;
+        for (std::size_t offset = 0; offset < BlockPages; offset += 8) {
+            if (loadWide(at + offset) != wide) return 0;
+        }
+        return value;
+    }
+
+    void settle(std::size_t first, std::size_t last) {
+        std::atomic_thread_fence(std::memory_order_seq_cst);
+        for (auto block = first / BlockPages; block <= (last - 1) / BlockPages; ++block) {
+            const auto value = common(block);
+            auto* summary = blockEntry(block, false);
+            if (summary == nullptr || loadByte(summary) == value) continue;
+            storeByte(summary, value);
+            if (value == 0) continue;
+            std::atomic_thread_fence(std::memory_order_seq_cst);
+            if (common(block) != value) storeByte(summary, 0);
+        }
     }
 
     void store(std::uintptr_t address, std::uint8_t value) {
-        if (auto* at = entry((address - base) / PageBytes, value != 0 && value != PageInaccessible)) std::atomic_ref<std::uint8_t>(*at).store(value, std::memory_order_relaxed);
+        store(address, address + PageBytes, value);
     }
 
     void store(std::uintptr_t first, std::uintptr_t last, std::uint8_t value) {
-        auto index = (first - base) / PageBytes;
-        const auto stop = (last - 1 - base) / PageBytes + 1;
-        while (index < stop) {
-            const auto next = std::min(stop, (index / ChunkEntries + 1) * ChunkEntries);
+        if (first >= last) return;
+        const auto begin = (first - base) / PageBytes;
+        const auto end = (last - 1 - base) / PageBytes + 1;
+        auto index = begin;
+        while (index < end) {
+            const auto next = std::min(end, (index / ChunkEntries + 1) * ChunkEntries);
             if (auto* at = entry(index, value != 0 && value != PageInaccessible)) {
-                for (; index < next; ++index, ++at) std::atomic_ref<std::uint8_t>(*at).store(value, std::memory_order_relaxed);
+                for (; index < next; ++index, ++at) storeByte(at, value);
             } else index = next;
         }
+        settle(begin, end);
     }
 
     // End of the run of pages equal to `value` that starts at `address`, at most `limit`.
@@ -443,11 +495,23 @@ struct PageSpan {
                 index = std::min(stop, (index / ChunkEntries + 1) * ChunkEntries);
                 continue;
             }
-            if (index % 8 == 0 && index + 8 <= stop && std::atomic_ref<std::uint64_t>(*reinterpret_cast<std::uint64_t*>(at)).load(std::memory_order_relaxed) == wide) {
+            if (index % BlockPages == 0) {
+                const auto block = index / BlockPages;
+                const auto* summary = blockEntry(block, false);
+                if (block % 8 == 0 && index + 8 * BlockPages <= stop && loadWide(summary) == wide) {
+                    index += 8 * BlockPages;
+                    continue;
+                }
+                if (index + BlockPages <= stop && loadByte(summary) == value) {
+                    index += BlockPages;
+                    continue;
+                }
+            }
+            if (index % 8 == 0 && index + 8 <= stop && loadWide(at) == wide) {
                 index += 8;
                 continue;
             }
-            if (std::atomic_ref<std::uint8_t>(*at).load(std::memory_order_relaxed) != value) break;
+            if (loadByte(at) != value) break;
             ++index;
         }
         return std::min(limit, base + index * PageBytes);
@@ -498,7 +562,7 @@ struct PageStates {
             if (span->size == 0 || bytes == 0 || address >= span->base + span->size || address + bytes <= span->base) continue;
             const auto last = std::min(address + bytes, span->base + span->size);
             for (auto at = std::max(address, span->base) & ~static_cast<std::uintptr_t>(PageBytes - 1); at < last; at += PageBytes) {
-                if (span->load(at) == PageInaccessible) span->store(at, 0);
+                if (span->load(at) == PageInaccessible) span->store(at, at + PageBytes, 0);
             }
         }
     }

core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp

@@ -24,6 +24,7 @@ void RunHeapMirrorRunUnwatchTests(const AgcDriver::Graphics::Context& context, c
 void RunGuestAllocationTests();
 void RunGuestLeaseWaitTests();
 void RunUnmappedGapTests();
+void RunPageBlockTests();
 void RunColorTargetLayoutTests();
 void RunLiveStackAccessTests();
 

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -11715,6 +11715,7 @@ int main(int argc, char** argv) {
         Require(mock.live == 0, "BDA resources leaked Vulkan objects");
         RunGuestAllocationTests();
         RunUnmappedGapTests();
+        RunPageBlockTests();
         RunColorTargetLayoutTests();
         RunLiveStackAccessTests();
         RunTextureFormatTests();

core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp

@@ -328,3 +328,94 @@ void RunUnmappedGapTests() {
     GuestArena::GuestArenaRelease_nid_postfix(spare, page);
 #endif
 }
+
+void RunPageBlockTests() {
+#if defined(__linux__)
+    namespace GuestMemory = AgcDriver::GuestMemory;
+    using Ranges = std::vector<std::pair<std::uint64_t, std::uint64_t>>;
+    if (!GuestArena::GuestArenaAvailable_nid_postfix()) return;
+    const auto page = static_cast<std::size_t>(sysconf(_SC_PAGESIZE));
+    if (page != 4096) return;
+    constexpr std::size_t count = 1536;
+    const auto bytes = count * page;
+    auto* guest = static_cast<std::byte*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, 512 * page));
+    Require(guest != nullptr && mmap(guest, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == guest, "cannot map the page block test pages");
+    const auto base = reinterpret_cast<std::uint64_t>(guest);
+    {
+        GuestAllocations::Mutation mutation;
+        mutation.Add(guest, bytes, true, true, true);
+    }
+    const auto protect = [&](std::size_t index, bool writable) {
+        GuestAllocations::Mutation mutation;
+        mutation.Protect(guest + index * page, page, true, writable, true, [&] { Require(mprotect(guest + index * page, page, writable ? PROT_READ | PROT_WRITE : PROT_READ) == 0, "cannot change a page block test page"); });
+    };
+    Require(GuestMemory::Accessible(guest, bytes, true), "mapped arena pages are not writable");
+    Require(mprotect(guest + 700 * page, page, PROT_READ) == 0, "cannot protect a page block test page behind the registry");
+    Require(GuestMemory::Accessible(guest, bytes, true), "a long run of checked arena pages was queried again instead of read from the page table");
+    Require(mprotect(guest + 700 * page, page, PROT_READ | PROT_WRITE) == 0, "cannot restore a page block test page behind the registry");
+    for (const std::size_t index : {std::size_t{700}, std::size_t{512}, std::size_t{1535}}) {
+        protect(index, false);
+        Require(!GuestMemory::Accessible(guest, bytes, true) && GuestMemory::Accessible(guest, bytes), "a page protected inside a long run of cached pages kept its old access");
+        Ranges expected{{base, base + index * page}};
+        if (index + 1 < count) expected.emplace_back(base + (index + 1) * page, base + bytes);
+        Require(GuestMemory::CommittedRanges(base, bytes, true) == expected, "the writable pages around a protected page inside a long run were misdescribed");
+        Require(GuestMemory::CommittedRanges(base, bytes) == Ranges{{base, base + bytes}}, "the readable pages of a long run with one read-only page were misdescribed");
+        protect(index, true);
+        Require(GuestMemory::DescribeCommitted(base, bytes, true).whole, "a long run of pages made writable again was not described whole");
+    }
+    std::atomic<bool> done{false};
+    std::atomic<std::size_t> described{0};
+    std::vector<std::thread> readers;
+    for (int reader = 0; reader < 2; ++reader) {
+        readers.emplace_back([&] {
+            while (!done.load(std::memory_order_relaxed)) {
+                static_cast<void>(GuestMemory::DescribeCommitted(base, bytes, true));
+                described.fetch_add(1, std::memory_order_relaxed);
+            }
+        });
+    }
+    for (std::size_t round = 0; round < 2000 || described.load(std::memory_order_relaxed) < 2000; ++round) {
+        const auto index = 640 + round % 128;
+        protect(index, false);
+        protect(index, true);
+    }
+    protect(700, false);
+    done = true;
+    for (auto& reader : readers) reader.join();
+    Require(GuestMemory::CommittedRanges(base, bytes, true) == Ranges{{base, base + 700 * page}, {base + 701 * page, base + bytes}}, "a page protected while other threads described its run is still reported writable");
+    protect(700, true);
+    for (std::size_t round = 0; round < 256; ++round) {
+        const auto index = (1 + round % 22) * 64 + (round * 37) % 64;
+        Require(GuestMemory::DescribeCommitted(base, bytes, true).whole, "the pages of a block were not described whole before one of them flipped");
+        std::atomic<bool> stop{false};
+        std::array<std::atomic<std::size_t>, 2> scans{};
+        std::vector<std::thread> scanners;
+        for (auto& scanned : scans) {
+            scanners.emplace_back([&] {
+                while (!stop.load(std::memory_order_relaxed)) {
+                    static_cast<void>(GuestMemory::DescribeCommitted(base, bytes, true));
+                    scanned.fetch_add(1, std::memory_order_release);
+                }
+            });
+        }
+        for (const auto& scanned : scans) {
+            while (scanned.load(std::memory_order_acquire) == 0) std::this_thread::yield();
+        }
+        protect(index, false);
+        for (const auto& scanned : scans) {
+            const auto flipped = scanned.load(std::memory_order_acquire);
+            while (scanned.load(std::memory_order_acquire) < flipped + 2) std::this_thread::yield();
+        }
+        stop = true;
+        for (auto& scanner : scanners) scanner.join();
+        Require(GuestMemory::CommittedRanges(base, bytes, true) == Ranges{{base, base + index * page}, {base + (index + 1) * page, base + bytes}}, "a page flipped in a marked block while other threads ran over the block is still reported writable");
+        protect(index, true);
+    }
+    {
+        GuestAllocations::Mutation mutation;
+        mutation.Unmap(guest, bytes, [](const void* pointer, std::size_t length, const void*, bool) { Require(munmap(const_cast<void*>(pointer), length) == 0, "cannot unmap the page block test pages"); });
+    }
+    Require(!GuestMemory::Accessible(guest, page) && GuestMemory::CommittedRanges(base, bytes).empty(), "unmapped pages of a long run are still accessible");
+    GuestArena::GuestArenaRelease_nid_postfix(guest, bytes);
+#endif
+}