fix(libkernel): answer a second scePthreadJoin with ESRCH

core/libs/prx/libkernel/CMakeLists.txt

@@ -594,6 +594,16 @@ if(BUILD_TESTING)
         set_tests_properties(guest_pthread_join_self PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
+    add_test_executable(guest_pthread_second_join_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadSecondJoin.cpp)
+    target_include_directories(guest_pthread_second_join_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_pthread_second_join_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_pthread_second_join_tests)
+    add_test(NAME guest_pthread_second_join COMMAND guest_pthread_second_join_tests)
+    set_tests_properties(guest_pthread_second_join PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_pthread_second_join PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+    endif()
     add_test_executable(guest_pthread_key_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadKey.cpp)
     target_include_directories(guest_pthread_key_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_pthread_key_tests PRIVATE libkernel libc)

core/libs/prx/libkernel/Pthread/include/Pthread.hpp

@@ -139,6 +139,8 @@ struct PthreadPrivate {
     void* _retval;
     bool _detached;
     bool _adopted;
+    Pthread* _handleStorage;
+    std::atomic<bool> _joined;
     std::mutex _join_mtx;
     TimedWait::Condition _join_cv;
     std::atomic<bool> cancelPending{false};

core/libs/prx/libkernel/Pthread/src/Thread.cpp

@@ -303,7 +303,7 @@ static CpuClockRegistry& CpuClocks() {
     return *registry;
 }
 
-PthreadPrivate::PthreadPrivate(const PthreadAttrPrivate* attr) : _finished(false), _retval(nullptr), _detached(false), _adopted(false) {
+PthreadPrivate::PthreadPrivate(const PthreadAttrPrivate* attr) : _finished(false), _retval(nullptr), _detached(false), _adopted(false), _handleStorage(nullptr), _joined(false) {
     auto& registry = CpuClocks();
     std::lock_guard lock(registry.lock);
     const auto* parent = CurrentGuestState().current;
@@ -554,6 +554,26 @@ static void ReleaseThread(PthreadPrivate* thread) {
     delete thread;
 }
 
+static std::mutex joinedLock;
+static std::map<Pthread*, PthreadPrivate*> joinedThreads;
+
+static void RetainJoinedThread(PthreadPrivate* thread) {
+    std::lock_guard lock(joinedLock);
+    Pthread* const storage = thread->_handleStorage;
+    const auto found = joinedThreads.find(storage);
+    if (found != joinedThreads.end() && found->second != thread) ReleaseThread(found->second);
+    joinedThreads[storage] = thread;
+}
+
+static void RecycleJoinedThread(Pthread* storage) {
+    std::lock_guard lock(joinedLock);
+    const auto found = joinedThreads.find(storage);
+    if (found == joinedThreads.end()) return;
+    PthreadPrivate* joined = found->second;
+    joinedThreads.erase(found);
+    ReleaseThread(joined);
+}
+
 #ifdef _WIN32
 // The title's job workers ("BPE JobWorkerThread CPU0".."CPU12") spin at full load on one core each.
 // Pinning them to the efficiency cores of a hybrid CPU (APS5_JOB_AFFINITY=1, with the driver's
@@ -716,8 +736,10 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread
         throw;
     }
     auto* published = p.release();
+    published->_handleStorage = thread;
     *thread = published;
     start.set_value(true);
+    RecycleJoinedThread(thread);
     if (detached)
         ReleaseThread(published);
 #else
@@ -772,8 +794,10 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread
         throw;
     }
     auto* published = p.release();
+    published->_handleStorage = thread;
     *thread = published;
     start.set_value(true);
+    RecycleJoinedThread(thread);
     if (detached)
         ReleaseThread(published);
 #endif
@@ -783,30 +807,32 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread
 int APS5_VABI scePthreadJoin(Pthread thread, void** retval) {
     if (!thread) return SCE_KERNEL_ERROR_EINVAL;
     if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL;
+    if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH;
     if (thread == CurrentGuestState().current) return SCE_KERNEL_ERROR_EDEADLK;
     ThreadCancel::Check();
     {
         std::unique_lock lock(thread->_join_mtx);
         ThreadCancel::WaitUntil(thread->_join_cv, lock, std::nullopt, [&] { return thread->_finished.load(); });
     }
+    if (thread->_joined.exchange(true, std::memory_order_acq_rel)) return SCE_KERNEL_ERROR_ESRCH;
 #ifdef _WIN32
     if (WaitForSingleObject(thread->nativeHandle, INFINITE) != WAIT_OBJECT_0)
         throw std::system_error(GetLastError(), std::system_category(), "Joining guest thread");
     if (retval) *retval = thread->_retval;
-    ReleaseThread(thread);
 #else
     if (const int result = pthread_join(thread->hostThread, nullptr); result != 0)
         throw std::system_error(result, std::generic_category(), "Joining guest thread");
     FreeGuestStack(thread);
     if (retval) *retval = thread->_retval;
-    ReleaseThread(thread);
 #endif
+    RetainJoinedThread(thread);
     return SCE_OK;
 }
 
 int APS5_VABI scePthreadDetach(Pthread thread) {
     if (!thread) return SCE_KERNEL_ERROR_EINVAL;
     if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL;
+    if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_EINVAL;
     thread->_detached = true;
 #ifdef _WIN32
     ReleaseThread(thread);

core/libs/tests/GuestPthreadSecondJoin.cpp

@@ -0,0 +1,65 @@
+#include "SceTypes.hpp"
+#include <atomic>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <thread>
+
+extern "C" {
+int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name);
+int APS5_VABI scePthreadJoin(Pthread thread, void** retval);
+int APS5_VABI scePthreadDetach(Pthread thread);
+int APS5_VABI pthread_join_nid_postfix(Pthread thread, void** retval);
+}
+
+static constexpr int SCE_OK = 0;
+static constexpr int SCE_KERNEL_ERROR_ESRCH = static_cast<int>(0x80020003u);
+static constexpr int SCE_KERNEL_ERROR_EINVAL = static_cast<int>(0x80020016u);
+static constexpr int GUEST_ESRCH = 3;
+static constexpr std::size_t REUSED_SLOT_ROUNDS = 64;
+
+static void Require(bool value) { if (!value) std::abort(); }
+
+static void* APS5_VABI Finish(void*) {
+    return reinterpret_cast<void*>(std::uintptr_t{0x1234});
+}
+
+static void* APS5_VABI WaitForRelease(void* arg) {
+    auto* release = static_cast<std::atomic<bool>*>(arg);
+    while (!release->load()) std::this_thread::yield();
+    return nullptr;
+}
+
+int main() {
+    Pthread thread = nullptr;
+    void* result = nullptr;
+    Require(scePthreadCreate(&thread, nullptr, Finish, nullptr, "second-join") == SCE_OK);
+    Require(scePthreadJoin(thread, &result) == SCE_OK);
+    Require(result == reinterpret_cast<void*>(std::uintptr_t{0x1234}));
+
+    result = reinterpret_cast<void*>(std::uintptr_t{0x55});
+    Require(scePthreadJoin(thread, &result) == SCE_KERNEL_ERROR_ESRCH);
+    Require(result == reinterpret_cast<void*>(std::uintptr_t{0x55}));
+    Require(scePthreadDetach(thread) == SCE_KERNEL_ERROR_EINVAL);
+    Require(pthread_join_nid_postfix(thread, &result) == GUEST_ESRCH);
+    Require(result == reinterpret_cast<void*>(std::uintptr_t{0x55}));
+
+    for (std::size_t round = 0; round < REUSED_SLOT_ROUNDS; ++round) {
+        thread = nullptr;
+        result = nullptr;
+        Require(scePthreadCreate(&thread, nullptr, Finish, nullptr, "reused-slot") == SCE_OK);
+        Require(scePthreadJoin(thread, &result) == SCE_OK);
+        Require(result == reinterpret_cast<void*>(std::uintptr_t{0x1234}));
+        Require(scePthreadJoin(thread, nullptr) == SCE_KERNEL_ERROR_ESRCH);
+    }
+
+    static std::atomic<bool> release{false};
+    Pthread detached = nullptr;
+    Require(scePthreadCreate(&detached, nullptr, WaitForRelease, &release, "detached") == SCE_OK);
+    Require(scePthreadDetach(detached) == SCE_OK);
+    result = reinterpret_cast<void*>(std::uintptr_t{0x55});
+    Require(scePthreadJoin(detached, &result) == SCE_KERNEL_ERROR_EINVAL);
+    Require(result == reinterpret_cast<void*>(std::uintptr_t{0x55}));
+    Require(scePthreadDetach(detached) == SCE_KERNEL_ERROR_EINVAL);
+    release = true;
+}

docs/dev/debt/libkernel.md

@@ -131,7 +131,7 @@
 - [sceKernelRaiseException](../../../core/libs/prx/libkernel/System/src/Exception.cpp) on Windows runs the handler wherever the target thread was interrupted. It waits only while the thread is inside winpthreads or restoring a context through `NtContinue`, and throws if that lasts 1 s, so a handler that calls into other host code the thread was interrupted in, such as the C runtime's heap or a lock of the emulator, can deadlock on it or reenter it
 - [sigaltstack](../../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) records each thread's alternate signal stack and reports it back, following FreeBSD's `kern_sigaltstack` (`EINVAL` for flags other than `SS_DISABLE`, `ENOMEM` below `MINSIGSTKSZ` 2048), but handlers never run on it, so `SS_ONSTACK` and `EPERM` are never reported
 - [sigpending](../../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) reports the intersection of the process-wide pending set of `sigtimedwait` and the blocked mask of `_sigprocmask`, as FreeBSD's `sys_sigpending` intersects the pending list with `td_sigmask`, and returns -1 with `EFAULT` for a null set. Because `_sigprocmask` keeps only the first 32 signals, signals 33 to 128 are never blocked and never appear in the set, and setting `SIG_IGN` (or `SIG_DFL` on a signal whose default action is to ignore) does not clear a signal that was already marked pending. PPSA99420 imports it
-- [scePthreadJoin, scePthreadCreate](../../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) - on a PS5 (firmware 13.60) joining a thread a second time returns `ESRCH` (`0x80020003`); here a joined thread is released, so a second join reads freed memory. A null entry crashed the payload and still throws. Joining the calling thread returns `EDEADLK` and a zeroed attribute creates a default thread, as on the console
+- [scePthreadJoin, scePthreadCreate](../../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) - on a PS5 (firmware 13.60) joining a thread a second time returns `ESRCH` (`0x80020003`), as it does here, but a joined thread is kept until `scePthreadCreate` publishes over the same handle storage, so one host object and its handle are retained per distinct guest handle storage location for the lifetime of the process; a guest that joins into a fresh storage location every time keeps every one of them. A null entry crashed the payload and still throws. Joining the calling thread returns `EDEADLK` and a zeroed attribute creates a default thread, as on the console. `scePthreadDetach` on a joined thread answers `EINVAL`, as FreeBSD does on a thread that is not joinable; the console answer for a detach after a join is unverified. A detached thread is still released when it ends, so joining it after it ended reads freed memory. A handle value copied to a second storage and joined there retains the object under the original storage key, so a later `scePthreadCreate` over that original storage releases an object the copy still points at, and the copy reads freed memory
 - [Thread cancellation](../../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) follows FreeBSD libthr (states and types 0/1 and 0/2, `PTHREAD_CANCELED` 1, the `_pthread_cleanup_info` node layout), not measured on the console. A request is acted on only in `scePthreadTestcancel`, the condition variable, `sem` and `scePthreadSem` waits, `scePthreadJoin` and the sleeps, and their POSIX names; file and socket calls are not cancellation points. A sleep is not interrupted: the thread exits when it ends. The asynchronous type acts like the deferred one, except at the two set calls.
 - [cpuset_setaffinity](../../../core/libs/prx/libkernel/Pthread/Posix/Thread.cpp) (libkernel) - only the calling thread or the calling process is handled; other levels (ROOT/CPUSET), other scopes (CPUSET/JAIL) and other threads throw as unmodelled, invalid levels or scopes return `EINVAL` and unknown thread or process IDs return `ESRCH`, following FreeBSD's `cpuset_setaffinity`. Bits outside `sceKernelGetAvailableCpumask` are dropped, and a mask with none of those bits fails with `EDEADLK`, as FreeBSD does when the new set has no CPU in common with the parent set
 - [cpuset_getaffinity](../../../core/libs/prx/libkernel/Pthread/Posix/Thread.cpp) (libkernel) - only the calling thread or the calling process is handled; other levels (ROOT/CPUSET), other scopes (CPUSET/JAIL) and other threads throw as unmodelled, invalid levels or scopes return `EINVAL` and unknown thread or process IDs return `ESRCH`, following FreeBSD's `cpuset_getaffinity`. For the process the mask is the union of the affinities of the live guest threads, as [FreeBSD 11.4](https://github.com/freebsd/freebsd-src/blob/d106002967eff712894ab8b5337bbbff106ce5e0/sys/kern/kern_cpuset.c#L1150-L1156) ORs the mask of every thread in the process; threads the guest did not create through libkernel are not included, and the console result is unverified