fix(libkernel): keep host standard streams open on guest close

core/libs/prx/libkernel/CMakeLists.txt

@@ -14,6 +14,7 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL
         ${CMAKE_CURRENT_SOURCE_DIR}/File/src/OpenFileLimit.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/File/src/SharedMemory.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/File/src/RandomDevice.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/File/src/HostStreams.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/File/src/Stdio.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../libc/src/Rename.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Semaphore/src/Semaphore.cpp
@@ -1148,6 +1149,16 @@ if(BUILD_TESTING)
         set_tests_properties(guest_socket_descriptors PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
+    add_test_executable(guest_host_stream_tests ${LIBS_INCLUDE_DIR}/tests/GuestHostStreams.cpp)
+    target_include_directories(guest_host_stream_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_host_stream_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_host_stream_tests)
+    add_test(NAME guest_host_streams COMMAND guest_host_stream_tests)
+    set_tests_properties(guest_host_streams PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_host_streams PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+    endif()
     add_test_executable(guest_socket_event_tests ${LIBS_INCLUDE_DIR}/tests/GuestSocketEvents.cpp)
     target_include_directories(guest_socket_event_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_socket_event_tests PRIVATE libkernel libc)

core/libs/prx/libkernel/File/include/HostStreams.hpp

@@ -0,0 +1,10 @@
+#ifndef CORE_LIBS_PRX_LIBKERNEL_FILE_HOSTSTREAMS_HPP
+#define CORE_LIBS_PRX_LIBKERNEL_FILE_HOSTSTREAMS_HPP
+
+namespace File {
+
+bool IsHostStream(int fd);
+
+}
+
+#endif

core/libs/prx/libkernel/File/src/HostStreams.cpp

@@ -0,0 +1,40 @@
+#include "prx/libkernel/File/include/HostStreams.hpp"
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#ifdef _WIN32
+#include <io.h>
+#include <stdlib.h>
+#else
+#include <fcntl.h>
+#endif
+
+namespace {
+
+#ifdef _WIN32
+extern "C" _invalid_parameter_handler _set_thread_local_invalid_parameter_handler(_invalid_parameter_handler);
+void IgnoreInvalidParameter(const wchar_t*, const wchar_t*, const wchar_t*, unsigned int, std::uintptr_t) {}
+#endif
+
+bool OpenAtStart(int fd) {
+#ifdef _WIN32
+    const auto previous = _set_thread_local_invalid_parameter_handler(IgnoreInvalidParameter);
+    const bool open = _get_osfhandle(fd) != -1;
+    _set_thread_local_invalid_parameter_handler(previous);
+    return open;
+#else
+    return ::fcntl(fd, F_GETFD) != -1;
+#endif
+}
+
+const std::array<bool, 3> g_hostStreams = {OpenAtStart(0), OpenAtStart(1), OpenAtStart(2)};
+
+}
+
+namespace File {
+
+bool IsHostStream(int fd) {
+    return fd >= 0 && fd < static_cast<int>(g_hostStreams.size()) && g_hostStreams[static_cast<std::size_t>(fd)];
+}
+
+}

core/libs/prx/libkernel/File/src/Open.cpp

@@ -11,6 +11,7 @@
 #include "prx/libkernel/File/include/FileLock.hpp"
 #include "prx/libkernel/Socket/include/SocketRuntime.hpp"
 #include "prx/libkernel/File/include/RandomDevice.hpp"
+#include "prx/libkernel/File/include/HostStreams.hpp"
 #include "prx/libkernel/KernelErrors.hpp"
 #include "prx/libkernel/Equeue/Equeue.hpp"
 #include "SceTypes.hpp"
@@ -233,6 +234,7 @@ int OpenInDirectory_nid_no_patch(int directory, const char* path, int flags, std
 }
 
 int APS5_VABI sceKernelClose(int d) {
+    if (File::IsHostStream(d)) return SCE_KERNEL_ERROR_EBADF;
     if (GuestSockets::IsOpen(d)) return GuestSockets::Close(d) == 0 ? 0 : SceKernelError(*__error_nid_postfix());
     EqueueDescriptorClosed(d);
     File::ForgetRandomDevice(d);

core/libs/prx/libkernel/File/src/Stdio.cpp

@@ -16,6 +16,7 @@
 #include "prx/libkernel/File/include/DescriptorControl.hpp"
 #include "prx/libkernel/File/include/FileLock.hpp"
 #include "prx/libkernel/File/include/RandomDevice.hpp"
+#include "prx/libkernel/File/include/HostStreams.hpp"
 #include "prx/libkernel/KernelErrors.hpp"
 #include "prx/libkernel/Equeue/Equeue.hpp"
 #include "prx/libkernel/Socket/include/SocketRuntime.hpp"
@@ -565,6 +566,7 @@ int APS5_VABI chflags_nid_postfix(const char* path, std::uint64_t flags) {
 }
 
 int APS5_VABI close_nid_postfix(int d) {
+    if (File::IsHostStream(d)) return PosixFailure(GUEST_EBADF);
     int socketError = 0;
     if (KernelCloseSocket_nid_no_patch(d, &socketError)) {
         if (socketError == 0) return 0;

core/libs/tests/GuestHostStreams.cpp

@@ -0,0 +1,38 @@
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include <cstdlib>
+
+extern "C" {
+int APS5_VABI dup_nid_postfix(int);
+int APS5_VABI pipe_nid_postfix(int*);
+int APS5_VABI close_nid_postfix(int);
+int APS5_VABI sceKernelClose(int);
+int* APS5_VABI __error_nid_postfix();
+}
+
+static void Require(bool value) { if (!value) std::abort(); }
+
+constexpr int GuestEbadf = 9;
+constexpr int SceKernelErrorEbadf = static_cast<int>(0x80020009);
+
+static bool Open(int fd) {
+    const int duplicate = dup_nid_postfix(fd);
+    if (duplicate < 0) return false;
+    Require(close_nid_postfix(duplicate) == 0);
+    return true;
+}
+
+int main() {
+    bool open[3];
+    for (int fd = 0; fd < 3; ++fd) open[fd] = Open(fd);
+    for (int fd = 0; fd < 3; ++fd) {
+        if (!open[fd]) continue;
+        Require(close_nid_postfix(fd) == -1 && *__error_nid_postfix() == GuestEbadf);
+        Require(sceKernelClose(fd) == SceKernelErrorEbadf);
+        Require(Open(fd));
+    }
+    int pipe[2];
+    Require(pipe_nid_postfix(pipe) == 0);
+    for (const int fd : pipe) Require(fd >= 3 || !open[fd]);
+    Require(close_nid_postfix(pipe[0]) == 0 && close_nid_postfix(pipe[1]) == 0);
+    return 0;
+}

docs/dev/debt/libkernel.md

@@ -99,6 +99,7 @@
 - [inet_pton, inet_ntop](../../../core/libs/prx/libkernel/Socket/src/AddressConversion.cpp) (libkernel) - the text forms are assumed to be those of FreeBSD 11.4 and are unverified on a console: `inet_pton` accepts only four decimal parts without leading zeros for `AF_INET` and the RFC 4291 forms for `AF_INET6` (one `::` standing for at least one group, a dotted-quad tail only as the last 32 bits, no zone id), and `inet_ntop` prints lowercase hex with the first longest run of two or more zero groups compressed, and a dotted tail only for `::a.b.c.d` and `::ffff:a.b.c.d`. They no longer depend on the host resolver library
 - [sceKernelIsThreadSanitizerEnabled](../../../core/libs/prx/libkernel/System/src/Sanitizer.cpp) (libkernel, NID `LzoM-wVLJDE`) - no source gives the signature; `int (void)` is assumed from `sceKernelIsAddressSanitizerEnabled` and from PPSA23226's bundled `libSceJobManager_nosubmission.prx`, which passes no arguments and tests only the 32-bit result
 - [getdtablesize](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - throws: FreeBSD returns the smaller of the `RLIMIT_NOFILE` soft limit and `maxfilesperproc` ([`sys_getdtablesize`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/kern_descrip.c#L338-L355)), and neither value is known for the console, where `getrlimit(RLIMIT_NOFILE)` throws too. VLC (PPSA85300) imports it for a `closefrom` loop in the child of its spawn helper ([libc_stubs.c](https://github.com/ZiZc3/VLC/blob/d46be6a68f4244466b041c2dc1b640915ac1a983/ps5/compat/libc_stubs.c#L66-L72)), which never runs because its `fork` fails with `ENOSYS`
+- [close, sceKernelClose](../../../core/libs/prx/libkernel/File/src/HostStreams.cpp) (libkernel) - descriptors 0 to 2 that are open when libkernel loads belong to the host process: closing one fails with `EBADF` and leaves it open. The console's result is unknown. PPSA21703 calls `sceKernelClose(0)` repeatedly; closing the host stream let the host C runtime give descriptor 0 to the next file opened, which the next call then closed
 
 ### Functional