fix(threads): run Windows pthread and small fiber stacks in guest VA range

core/libs/prx/libSceFiber/Export.cpp

@@ -281,20 +281,28 @@ static std::uint64_t HostStackGuardBytes() {
 static void AllocateHostStack(Fiber* fiber) {
     const auto page = HostStackGuardBytes();
     const auto bytes = (fiber->contextSize + FIBER_HOST_STACK_MARGIN + page - 1) / page * page + page;
-    auto* mapping = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
-    if (mapping == nullptr) throw std::runtime_error(std::string("sceFiberInitialize: no host stack of ") + std::to_string(bytes) + " bytes for fiber '" + fiber->name + "' (error " + std::to_string(GetLastError()) + ")");
-    DWORD previous = 0;
-    if (!VirtualProtect(mapping, page, PAGE_NOACCESS, &previous)) {
-        const auto error = GetLastError();
-        VirtualFree(mapping, 0, MEM_RELEASE);
-        throw std::runtime_error(std::string("sceFiberInitialize: cannot protect the guard page of the host stack of fiber '") + fiber->name + "' (error " + std::to_string(error) + ")");
+    void* allocation = nullptr;
+    try {
+        allocation = GuestArena::GuestArenaAllocate_nid_postfix(static_cast<std::size_t>(bytes), static_cast<std::size_t>(page));
+        GuestArena::GuestArenaCommit_nid_postfix(static_cast<std::uint8_t*>(allocation) + page, static_cast<std::size_t>(bytes - page), PAGE_READWRITE, 0x1000);
+        GuestArena::GuestArenaPinWritable_nid_postfix(static_cast<std::uint8_t*>(allocation) + page, static_cast<std::size_t>(bytes - page));
+    } catch (...) {
+        if (allocation != nullptr) {
+            GuestArena::GuestArenaReset_nid_postfix(allocation, static_cast<std::size_t>(bytes));
+            GuestArena::GuestArenaRelease_nid_postfix(allocation, static_cast<std::size_t>(bytes));
+        }
+        throw std::runtime_error(std::string("sceFiberInitialize: no host stack of ") + std::to_string(bytes) + " bytes for fiber '" + fiber->name + "'");
     }
-    fiber->hostStack = mapping;
+    fiber->hostStack = static_cast<std::uint8_t*>(allocation);
     fiber->hostStackBytes = bytes;
 }
 
 static void ReleaseHostStack(Fiber* fiber) {
-    if (fiber->hostStack && !VirtualFree(fiber->hostStack, 0, MEM_RELEASE)) throw std::runtime_error(std::string("sceFiberFinalize: cannot release the host stack of fiber '") + fiber->name + "' (error " + std::to_string(GetLastError()) + ")");
+    if (fiber->hostStack == nullptr) return;
+    const auto page = HostStackGuardBytes();
+    GuestArena::GuestArenaUnpinWritable_nid_postfix(fiber->hostStack + page, static_cast<std::size_t>(fiber->hostStackBytes - page));
+    GuestArena::GuestArenaReset_nid_postfix(fiber->hostStack, static_cast<std::size_t>(fiber->hostStackBytes));
+    GuestArena::GuestArenaRelease_nid_postfix(fiber->hostStack, static_cast<std::size_t>(fiber->hostStackBytes));
     fiber->hostStack = nullptr;
     fiber->hostStackBytes = 0;
 }

core/libs/prx/libc/include/GuestStacks.hpp

@@ -11,6 +11,11 @@ extern "C" {
 
 void GuestStackRegister_nid_no_patch(const void* pointer, std::size_t bytes);
 void APS5_VABI GuestStackSwitch_nid_no_patch(std::uintptr_t stackPointer);
+#ifdef _WIN32
+bool GuestStackIsRegistered_nid_no_patch(std::uintptr_t address);
+void GuestStackNoteMainHostBounds_nid_no_patch(std::uintptr_t base, std::uintptr_t limit, std::uintptr_t deallocation);
+bool GuestStackMainHostBounds_nid_no_patch(std::uintptr_t* base, std::uintptr_t* limit, std::uintptr_t* deallocation);
+#endif
 
 }
 

core/libs/prx/libc/src/GuestStacks.cpp

@@ -22,6 +22,8 @@ constexpr std::size_t DeallocationStackOffset = 0x1478;
 
 std::mutex rangesLock;
 std::map<std::uintptr_t, std::uintptr_t> ranges;
+StackBounds mainHostBounds{0, 0, 0};
+bool hasMainHostBounds = false;
 
 StackBounds CurrentBounds() {
     auto* tib = reinterpret_cast<NT_TIB*>(NtCurrentTeb());
@@ -91,4 +93,29 @@ void APS5_VABI GuestStackSwitch_nid_no_patch(std::uintptr_t stackPointer) {
 #endif
 }
 
+#ifdef _WIN32
+bool GuestStackIsRegistered_nid_no_patch(std::uintptr_t address) {
+    std::lock_guard lock(rangesLock);
+    const auto next = ranges.upper_bound(address);
+    if (next == ranges.begin()) return false;
+    const auto& [low, high] = *std::prev(next);
+    return address >= low && address < high;
+}
+
+void GuestStackNoteMainHostBounds_nid_no_patch(std::uintptr_t base, std::uintptr_t limit, std::uintptr_t deallocation) {
+    std::lock_guard lock(rangesLock);
+    mainHostBounds = {base, limit, deallocation};
+    hasMainHostBounds = true;
+}
+
+bool GuestStackMainHostBounds_nid_no_patch(std::uintptr_t* base, std::uintptr_t* limit, std::uintptr_t* deallocation) {
+    std::lock_guard lock(rangesLock);
+    if (!hasMainHostBounds) return false;
+    if (base) *base = mainHostBounds.base;
+    if (limit) *limit = mainHostBounds.limit;
+    if (deallocation) *deallocation = mainHostBounds.deallocation;
+    return true;
+}
+#endif
+
 }

core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp

@@ -64,16 +64,16 @@ struct KernelArena {
     void Release(const void* pointer, size_t len) { GuestArena::GuestArenaRelease_nid_postfix(pointer, len); }
 };
 
-static void CommitArenaRange(void* addr, size_t len, DWORD winProt) {
-    GuestArena::GuestArenaCommit_nid_postfix(addr, len, winProt, PS5_PAGE_SIZE);
+static void CommitArenaRange(void* addr, size_t len, DWORD winProt, size_t granule = 0) {
+    GuestArena::GuestArenaCommit_nid_postfix(addr, len, winProt, granule != 0 ? granule : PS5_PAGE_SIZE);
 }
 
-static void* mmap_aligned(size_t len, int prot, size_t alignment, std::uintptr_t hint = 0) {
+static void* mmap_aligned(size_t len, int prot, size_t alignment, size_t granule, std::uintptr_t hint = 0) {
     auto& arena = KernelArena::Get();
     void* result = GuestArena::GuestArenaAllocateFromHint_nid_postfix(hint, len, alignment);
     if (prot != PROT_NONE) {
         try {
-            CommitArenaRange(result, len, WinProtFromPosix(prot));
+            CommitArenaRange(result, len, WinProtFromPosix(prot), granule);
         } catch (...) {
             arena.Release(result, len);
             throw;
@@ -82,15 +82,15 @@ static void* mmap_aligned(size_t len, int prot, size_t alignment, std::uintptr_t
     return result;
 }
 
-static void* mmap(void* addr, size_t len, int prot, int flags, int, int) {
+static void* mmap(void* addr, size_t len, int prot, int flags, int, int, size_t granule) {
     DWORD winProt = WinProtFromPosix(prot);
     if (flags & MAP_FIXED) {
         auto& arena = KernelArena::Get();
         if (arena.Contains(addr, len)) {
             arena.MarkUsed(addr, len);
             if (prot != PROT_NONE) {
                 try {
-                    CommitArenaRange(addr, len, winProt);
+                    CommitArenaRange(addr, len, winProt, granule);
                 } catch (...) {
                     arena.Release(addr, len);
                     throw;
@@ -102,7 +102,7 @@ static void* mmap(void* addr, size_t len, int prot, int flags, int, int) {
         if (!result) throw std::system_error(static_cast<int>(GetLastError()), std::system_category(), "VirtualAlloc fixed failed");
         return result;
     }
-    return mmap_aligned(len, prot, PS5_PAGE_SIZE);
+    return mmap_aligned(len, prot, PS5_PAGE_SIZE, granule);
 }
 
 static int munmap(void* addr, size_t len) {
@@ -684,7 +684,7 @@ void Unmap(void* addr, size_t len) {
 #endif
 }
 
-void* MapPlaced(void* addr, size_t len, int prot, int flags, size_t alignment) {
+void* MapPlaced(void* addr, size_t len, int prot, int flags, size_t alignment, size_t commitGranule = 0) {
     ValidateLength(len);
     alignment = ValidateAlignment(alignment);
     constexpr int GuestMapFixed = 0x10;
@@ -737,7 +737,7 @@ void* MapPlaced(void* addr, size_t len, int prot, int flags, size_t alignment) {
 #else
         const int placement = MAP_FIXED;
 #endif
-        void* result = mmap(addr, len, prot, MAP_PRIVATE | MAP_ANONYMOUS | placement, -1, 0);
+        void* result = mmap(addr, len, prot, MAP_PRIVATE | MAP_ANONYMOUS | placement, -1, 0, commitGranule);
         if (result == MAP_FAILED) {
             // return SCE_KERNEL_ERROR_ENOMEM;
             throw std::system_error(errno, std::generic_category(), "Fixed mmap failed");
@@ -755,11 +755,11 @@ void* MapPlaced(void* addr, size_t len, int prot, int flags, size_t alignment) {
         }
         return MapAtOrAbove(reinterpret_cast<std::uintptr_t>(addr), len, prot, alignment);
 #else
-        return mmap_aligned(len, prot, alignment, reinterpret_cast<std::uintptr_t>(addr));
+        return mmap_aligned(len, prot, alignment, commitGranule, reinterpret_cast<std::uintptr_t>(addr));
 #endif
     }
 #ifdef _WIN32
-    return mmap_aligned(len, prot, alignment);
+    return mmap_aligned(len, prot, alignment, commitGranule);
 #endif
 #if defined(__linux__)
     if (ArenaAvailable()) {
@@ -773,7 +773,11 @@ void* MapPlaced(void* addr, size_t len, int prot, int flags, size_t alignment) {
         throw std::overflow_error("Aligned mapping size overflow");
     }
     const size_t allocLen = len + alignment;
+#ifdef _WIN32
+    void* result = mmap(nullptr, allocLen, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, 0);
+#else
     void* result = mmap(nullptr, allocLen, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+#endif
     if (result == MAP_FAILED) {
         // return SCE_KERNEL_ERROR_ENOMEM;
         throw std::system_error(errno, std::generic_category(), "Aligned mmap failed");
@@ -836,8 +840,8 @@ void RecordProtection(const void* addr, size_t len, int prot) {
     if (prot >= 0) g_protections.emplace(start, ProtectedRange{start + len, prot});
 }
 
-void* MapAligned(void* addr, size_t len, int prot, int flags, size_t alignment) {
-    void* mapped = MapPlaced(addr, len, prot, flags, alignment);
+void* MapAligned(void* addr, size_t len, int prot, int flags, size_t alignment, size_t commitGranule = 0) {
+    void* mapped = MapPlaced(addr, len, prot, flags, alignment, commitGranule);
 #if defined(__linux__)
     GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(mapped, len);
 #endif
@@ -869,18 +873,42 @@ bool Reserved(const void* addr, size_t len) {
 
 void UnmapRegistered(GuestAllocations::Mutation& mutation, void* addr, size_t len) {
     RequireResidentUnpinned(mutation, reinterpret_cast<std::uintptr_t>(addr), len);
-    mutation.Unmap(addr, len, [&](const void* piece, std::size_t pieceBytes, const void* allocation, bool last) {
-        auto* pieceAddress = const_cast<void*>(piece);
+    // Registry ranges split on protection changes while a host allocation spans them: release one
+    // host allocation at a time so a partial reset never truncates it.
+    const void* pendingBase = nullptr;
+    std::size_t pendingBytes = 0;
+    const void* pendingAllocation = nullptr;
+    bool pendingArena = false;
+    const auto flushPending = [&] {
+        if (pendingBase == nullptr) return;
+        auto* pieceAddress = const_cast<void*>(pendingBase);
         std::lock_guard lock(g_directLock);
-        EraseMappings(reinterpret_cast<std::uintptr_t>(piece), reinterpret_cast<std::uintptr_t>(piece) + pieceBytes);
 #if defined(__linux__)
-        Unmap(pieceAddress, pieceBytes);
+        Unmap(pieceAddress, pendingBytes);
 #else
-        if (KernelArena::Get().Contains(pieceAddress, pieceBytes)) munmap(pieceAddress, pieceBytes);
-        else if (last) munmap_release(const_cast<void*>(allocation));
-        else munmap(pieceAddress, pieceBytes);
+        if (pendingArena) munmap(pieceAddress, pendingBytes);
+        else munmap_release(const_cast<void*>(pendingAllocation));
 #endif
+        pendingBase = nullptr;
+        pendingBytes = 0;
+        pendingAllocation = nullptr;
+    };
+    mutation.Unmap(addr, len, [&](const void* piece, std::size_t pieceBytes, const void* allocation, bool last) {
+        {
+            std::lock_guard lock(g_directLock);
+            EraseMappings(reinterpret_cast<std::uintptr_t>(piece), reinterpret_cast<std::uintptr_t>(piece) + pieceBytes);
+        }
+        const bool arena = KernelArena::Get().Contains(piece, pieceBytes);
+        if (pendingBase != nullptr && (allocation != pendingAllocation || arena != pendingArena || reinterpret_cast<std::uintptr_t>(piece) != reinterpret_cast<std::uintptr_t>(pendingBase) + pendingBytes)) flushPending();
+        if (pendingBase == nullptr) {
+            pendingBase = piece;
+            pendingAllocation = allocation;
+            pendingArena = arena;
+        }
+        pendingBytes += pieceBytes;
+        if (last) flushPending();
     });
+    flushPending();
     EraseReservations(addr, len);
     RecordProtection(addr, len, -1);
 }
@@ -938,7 +966,7 @@ int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart,
     return 0;
 }
 
-int DoMapAnon(void** addr, size_t len, int prot, int flags, size_t alignment, bool exclusive) {
+int DoMapAnon(void** addr, size_t len, int prot, int flags, size_t alignment, bool exclusive, size_t commitGranule) {
     ValidateOutput(addr);
     if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0) return SCE_KERNEL_ERROR_EINVAL;
     GuestAllocations::Mutation mutation;
@@ -951,7 +979,7 @@ int DoMapAnon(void** addr, size_t len, int prot, int flags, size_t alignment, bo
         return 0;
     }
     ReplaceFixedOverlap(mutation, *addr, len, flags);
-    void* mapped = MapAligned(*addr, len, LinuxProtFromSce(prot), flags, alignment);
+    void* mapped = MapAligned(*addr, len, LinuxProtFromSce(prot), flags, alignment, commitGranule);
     try {
         mutation.Add(mapped, len, (prot & 3) != 0, (prot & 2) != 0, (prot & GuestProtGpuReadWrite) != 0);
         if ((prot & GuestProtGpuReadWrite) != 0) mutation.NoteGpuMapping(mapped, len);

core/libs/prx/libkernel/DirectMemory/DirectMemory.hpp

@@ -35,7 +35,7 @@ void DirectMemoryRetype(int64_t start, size_t len, int memoryType);
 bool DirectMemoryLargestFree(int64_t searchStart, int64_t searchEnd, size_t alignment, int64_t* startOut, size_t* sizeOut);
 size_t DirectMemoryFreeRun(uint64_t offset, uint64_t limit);
 int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart, size_t alignment);
-int DoMapAnon(void** addr, size_t len, int prot, int flags, size_t alignment = PS5_PAGE_SIZE, bool exclusive = false);
+int DoMapAnon(void** addr, size_t len, int prot, int flags, size_t alignment = PS5_PAGE_SIZE, bool exclusive = false, size_t commitGranule = 0);
 int DoMapFileBacked_nid_no_patch(void** addr, size_t len, int prot, int flags, int descriptor, std::uint64_t offset, size_t alignment);
 void ReleaseFlexibleMappingRange(std::uintptr_t address, size_t len);
 int DoMapShared(void** addr, size_t len, int prot, HostSharedMemory object);

core/libs/prx/libkernel/Pthread/include/Pthread.hpp

@@ -131,10 +131,10 @@ struct PthreadPrivate {
 #ifndef _WIN32
     clockid_t hostCpuClock{};
     bool hostCpuClockBound = false;
-    void* guestStack = nullptr;
-    std::size_t guestStackBytes = 0;
     int hostTid = 0;
 #endif
+    void* guestStack = nullptr;
+    std::size_t guestStackBytes = 0;
     std::mutex nameLock;
     std::string name;
     std::atomic<bool> _finished;