Port open upstream relinker fixes (#4340, #4802, #4501)

core/relinker/CMakeLists.txt

@@ -471,6 +471,8 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_input_magic.py $<TARGET_FILE:relinker>)
         add_test(NAME input_preservation COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_input_preservation.py $<TARGET_FILE:relinker>)
+        add_test(NAME output_collisions COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_output_collisions.py $<TARGET_FILE:relinker>)
         add_test(NAME input_file COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_input_file.py $<TARGET_FILE:relinker>)
         add_test(NAME null_symbol_relocations COMMAND ${Python3_EXECUTABLE}
@@ -498,6 +500,9 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_init_array_roots.py $<TARGET_FILE:relinker>)
         add_test(NAME gnu_hash_analysis COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_gnu_hash_analysis.py $<TARGET_FILE:relinker>)
+        add_test(NAME guest_gnu_hash COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_gnu_hash.py $<TARGET_FILE:relinker>)
+        set_tests_properties(guest_gnu_hash PROPERTIES TIMEOUT 20)
         add_test(NAME to_rosetta COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_to_rosetta.py $<TARGET_FILE:relinker>)
         add_test(NAME syscall_number COMMAND ${Python3_EXECUTABLE}

core/relinker/main.cpp

@@ -28,6 +28,7 @@
 #include <filesystem>
 #include <iostream>
 #include <memory>
+#include <random>
 #include <string>
 #include <vector>
 
@@ -38,6 +39,100 @@
 
 namespace {
 
+#ifdef _WIN32
+
+class OutputIdentityDirectory {
+public:
+    OutputIdentityDirectory(const OutputIdentityDirectory&) = delete;
+    OutputIdentityDirectory& operator=(const OutputIdentityDirectory&) = delete;
+
+    explicit OutputIdentityDirectory(const std::filesystem::path& parent) {
+        std::random_device random;
+        for (unsigned attempt = 0; attempt < 64; ++attempt) {
+            const auto candidate = parent / (".anyps5-output-identity-" + std::to_string(random()) + "-" + std::to_string(random()));
+            std::error_code error;
+            if (std::filesystem::create_directory(candidate, error)) {
+                directory = candidate;
+                return;
+            }
+            if (error && error != std::errc::file_exists)
+                throw std::filesystem::filesystem_error("Cannot create output identity directory", candidate, error);
+        }
+        throw Domain::RelinkerException("Cannot reserve an output identity directory in " + Io::Utf8Path(parent));
+    }
+
+    ~OutputIdentityDirectory() {
+        if (directory.empty()) return;
+        try {
+            Remove();
+        } catch (const std::exception& error) {
+            std::cerr << "FAIL: Output identity cleanup: " << error.what() << '\n';
+        }
+    }
+
+    std::filesystem::path GetPath() const { return directory / "file"; }
+    const std::filesystem::path& GetDirectory() const { return directory; }
+
+    void Remove() {
+        std::filesystem::remove(GetPath());
+        std::filesystem::remove(directory);
+        directory.clear();
+    }
+
+private:
+    std::filesystem::path directory;
+};
+
+#endif
+
+bool SameOutputFile(const std::filesystem::path& first, const std::filesystem::path& second) {
+    if (std::filesystem::equivalent(first, second)) return true;
+#ifdef _WIN32
+    if (std::filesystem::status(first).permissions() != std::filesystem::status(second).permissions() &&
+        std::filesystem::hard_link_count(first) > 1 && std::filesystem::hard_link_count(second) > 1) {
+        for (const auto& path : {first, second}) {
+            if ((std::filesystem::status(path).permissions() & std::filesystem::perms::owner_write) == std::filesystem::perms::none)
+                throw Domain::RelinkerException("Cannot compare read-only output identities: " + Io::Utf8Path(path));
+        }
+        OutputIdentityDirectory firstDirectory(first.parent_path());
+        OutputIdentityDirectory secondDirectory(second.parent_path());
+        std::filesystem::create_hard_link(first, firstDirectory.GetPath());
+        std::filesystem::create_hard_link(second, secondDirectory.GetPath());
+        const bool same = std::filesystem::equivalent(firstDirectory.GetPath(), secondDirectory.GetPath());
+        secondDirectory.Remove();
+        firstDirectory.Remove();
+        return same;
+    }
+#endif
+    return false;
+}
+
+bool SameOutputDestination(const std::filesystem::path& first, const std::filesystem::path& second) {
+    const auto firstPath = std::filesystem::weakly_canonical(first);
+    const auto secondPath = std::filesystem::weakly_canonical(second);
+    if (firstPath == secondPath) return true;
+    if (std::filesystem::exists(first) && std::filesystem::exists(second)) return SameOutputFile(first, second);
+#ifdef _WIN32
+    auto firstName = Io::Utf8Path(firstPath.filename());
+    auto secondName = Io::Utf8Path(secondPath.filename());
+    for (auto* name : {&firstName, &secondName})
+        for (auto& character : *name)
+            if (character >= 'A' && character <= 'Z') character = static_cast<char>(character + ('a' - 'A'));
+    if (firstName == secondName && std::filesystem::exists(firstPath.parent_path()) &&
+        std::filesystem::exists(secondPath.parent_path()) && std::filesystem::equivalent(firstPath.parent_path(), secondPath.parent_path())) {
+        OutputIdentityDirectory probe(firstPath.parent_path());
+        auto alternateName = Io::Utf8Path(probe.GetDirectory().filename());
+        for (auto& character : alternateName)
+            if (character >= 'a' && character <= 'z') character = static_cast<char>(character - ('a' - 'A'));
+        const auto alternate = firstPath.parent_path() / Io::NativePath(alternateName);
+        const bool same = std::filesystem::exists(alternate) && std::filesystem::equivalent(probe.GetDirectory(), alternate);
+        probe.Remove();
+        return same;
+    }
+#endif
+    return false;
+}
+
 int Run(const int argc, char* argv[]) {
     Cli::Args args;
     try {
@@ -140,6 +235,23 @@ int Run(const int argc, char* argv[]) {
             for (const auto& path : registryPaths) checkOutputInputs(path, "Registry");
         }
 
+        std::vector<std::filesystem::path> outputPaths{Io::NativePath(absPath)};
+        for (const auto& artifact : guestArtifacts) outputPaths.push_back(artifact.Path);
+        outputPaths.insert(outputPaths.end(), registryPaths.begin(), registryPaths.end());
+        if (desktopEntry) {
+            outputPaths.push_back(desktopEntry->IconPath);
+            outputPaths.push_back(desktopEntry->EntryPath);
+        }
+        if (args.windowsDiagnostics) outputPaths.push_back(std::filesystem::absolute("windows-diagnostics-imports.txt"));
+        for (std::size_t index = 0; index < outputPaths.size(); ++index) {
+            for (std::size_t previous = 0; previous < index; ++previous) {
+                const auto& first = outputPaths[index];
+                const auto& second = outputPaths[previous];
+                if (SameOutputDestination(first, second))
+                    throw Domain::RelinkerException("Output paths collide: " + Io::Utf8Path(first) + " and " + Io::Utf8Path(second));
+            }
+        }
+
         auto byteWriter = std::make_shared<Io::ByteWriter>();
 
         std::shared_ptr<Elfpatcher::IElfPatcher> patcher;

core/relinker/relinker/src/guest/GuestImageReader.cpp

@@ -148,7 +148,7 @@ GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector
         if (tag == 1) needed.push_back(value);
         else if (tag == 0x61000045) moduleImports.push_back(value);
         else if (tag == 0x6100000d || tag == 0x61000043) moduleExports.push_back(value);
-        else if (tag < 0x60000000 || tag == 0x6100003f || (tag >= 0x61000027 && tag <= 0x6100003b)) {
+        else if (tag < 0x60000000 || tag == 0x6ffffef5 || tag == 0x6100003f || (tag >= 0x61000027 && tag <= 0x6100003b)) {
             if (!tags.emplace(tag, value).second) fail("Duplicate dynamic tag " + std::to_string(tag));
         }
     }
@@ -194,7 +194,35 @@ GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector
     std::uint64_t symSize = 0;
     if (tags.contains(0x6100003f)) symSize = tags.at(0x6100003f);
     else if (tags.contains(4)) symSize = static_cast<std::uint64_t>(Io::ReadU32(bytes, translate(tags.at(4), 8) + 4)) * 24;
-    else fail("Missing dynamic symbol count");
+    else if (tags.contains(0x6ffffef5)) {
+        const auto address = tags.at(0x6ffffef5);
+        const auto header = translate(address, 16);
+        const auto bucketCount = Io::ReadU32(bytes, header);
+        const auto firstSymbol = Io::ReadU32(bytes, header + 4);
+        const auto bloomCount = Io::ReadU32(bytes, header + 8);
+        if (bucketCount == 0 || bloomCount == 0 || firstSymbol == 0) fail("Invalid GNU hash header");
+        const std::uint64_t bucketOffset = 16 + static_cast<std::uint64_t>(bloomCount) * 8;
+        const auto chainOffset = bucketOffset + static_cast<std::uint64_t>(bucketCount) * 4;
+        const auto start = translate(address, chainOffset);
+        std::uint64_t lastBucket = 0;
+        for (std::uint64_t index = 0; index < bucketCount; ++index) {
+            const auto bucket = Io::ReadU32(bytes, start + bucketOffset + index * 4);
+            if (bucket != 0 && bucket < firstSymbol) fail("Invalid GNU hash bucket");
+            lastBucket = std::max(lastBucket, static_cast<std::uint64_t>(bucket));
+        }
+        std::uint64_t symbolCount = firstSymbol;
+        if (lastBucket != 0) {
+            symbolCount = lastBucket;
+            for (;;) {
+                const auto offset = chainOffset + (symbolCount - firstSymbol) * 4;
+                if (offset > std::numeric_limits<std::uint64_t>::max() - address) fail("Invalid GNU hash chain");
+                const auto chain = Io::ReadU32(bytes, translate(address + offset, 4));
+                ++symbolCount;
+                if (chain & 1) break;
+            }
+        }
+        symSize = symbolCount * 24;
+    } else fail("Missing dynamic symbol count");
     if (symSize == 0 || symSize % 24 != 0 || symSize / 24 > std::numeric_limits<std::uint32_t>::max()) fail("Invalid dynamic symbol count");
     const auto symOffset = table(6, 0x61000039, symSize);
     image.Dynamic.DynSymData.assign(bytes.begin() + symOffset, bytes.begin() + symOffset + symSize);

core/relinker/relinker/src/guest/GuestModuleBuilder.cpp

@@ -57,6 +57,8 @@ std::string ModuleStem(std::string name, const bool windows) {
     return {};
 }
 
+enum class GuestModuleFormat { Elf, Self, Other };
+
 }
 
 std::vector<GuestArtifact> GuestModuleBuilder::Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, const bool windows, const bool macos, const bool toIntel, const bool toRosetta, ISyscallScanner& syscallScanner, const bool lazyBinding, const std::string& runPath, const std::set<std::string>& excludedModules, const std::filesystem::path& sceModulePath, const std::vector<std::string>& moduleDirectories) const {
@@ -89,16 +91,20 @@ std::vector<GuestArtifact> GuestModuleBuilder::Build(const std::filesystem::path
     }
     std::vector<std::filesystem::path> paths;
     std::set<std::string> unmatchedExclusions = excludedModules;
-    const auto isElf = [](const std::filesystem::path& path) {
+    const auto classifyGuestModule = [](const std::filesystem::path& path) {
         std::ifstream stream(path, std::ios::binary);
         if (!stream) throw Domain::RelinkerException("Cannot read guest candidate: " + Io::Utf8Path(path));
         char magic[4]{};
         stream.read(magic, 4);
         if (stream.bad()) throw Domain::RelinkerException("Cannot read guest candidate magic: " + Io::Utf8Path(path));
-        if (stream.gcount() != 4) return false;
+        if (stream.gcount() != 4) return GuestModuleFormat::Other;
         const auto byte = [&](const std::size_t index) { return static_cast<unsigned char>(magic[index]); };
-        const bool self = (byte(0) == 0x4f && byte(1) == 0x15 && byte(2) == 0x3d && byte(3) == 0x1d) || (byte(0) == 0x54 && byte(1) == 0x14 && byte(2) == 0xf5 && byte(3) == 0xee);
-        return byte(0) == 0x7f && magic[1] == 'E' && magic[2] == 'L' && magic[3] == 'F';
+        if ((byte(0) == 0x4f && byte(1) == 0x15 && byte(2) == 0x3d && byte(3) == 0x1d) ||
+            (byte(0) == 0x54 && byte(1) == 0x14 && byte(2) == 0xf5 && byte(3) == 0xee))
+            return GuestModuleFormat::Self;
+        if (byte(0) == 0x7f && magic[1] == 'E' && magic[2] == 'L' && magic[3] == 'F')
+            return GuestModuleFormat::Elf;
+        return GuestModuleFormat::Other;
     };
     for (const auto& directory : directories) {
         if (!std::filesystem::is_directory(directory)) throw Domain::RelinkerException("Guest module path is not a directory: " + Io::Utf8Path(directory));
@@ -110,7 +116,11 @@ std::vector<GuestArtifact> GuestModuleBuilder::Build(const std::filesystem::path
             }
             if (!entry.is_regular_file()) continue;
             if (std::filesystem::equivalent(entry.path(), inputPath)) continue;
-            if (isElf(entry.path())) paths.push_back(entry.path());
+            const auto format = classifyGuestModule(entry.path());
+            if (format == GuestModuleFormat::Elf) paths.push_back(entry.path());
+            else if (format == GuestModuleFormat::Self)
+                std::cout << "Skipping SELF container: " << entry.path().lexically_relative(root).generic_string()
+                          << " (unwrap it before relinking)\n";
         }
     }
     const auto neededNames = ReadNeededNames(dynamic);

core/relinker/relinker/tests/test_guest_gnu_hash.py

@@ -0,0 +1,111 @@
+from pathlib import Path
+import os
+import struct
+import subprocess
+import sys
+import tempfile
+
+from test_gnu_hash_analysis import name_hash
+from test_guest_intel_trampolines import main_fixture
+from test_guest_symbol_names import dynamic_tags, file_offset
+from test_windows_import_modules import dynamic, executable, provider
+
+
+HASH = 0xA00
+SYMBOLS = 0xC00
+STRINGS = 0xD00
+
+
+def module(names=("alpha", "bravo", "charlie"), bucket_count=3):
+    image = provider(42)
+    image.extend(bytes(0x1000 - len(image)))
+    struct.pack_into("<QQ", image, 120 + 32, 0xA00, 0xA00)
+    names = sorted(names, key=lambda name: name_hash(name + "#A#B") % bucket_count)
+    strings = bytearray(b"\0")
+    buckets = [0] * bucket_count
+    chains = []
+    bloom = 0
+    for index, name in enumerate(names, 1):
+        code = 0x400 + index * 16
+        image[code:code + 6] = b"\xb8" + struct.pack("<I", 40 + index) + b"\xc3"
+        struct.pack_into("<IBBHQQ", image, SYMBOLS + index * 24,
+                         len(strings), 0x12, 0, 1, 0x1000 + index * 16, 6)
+        qualified = name + "#A#B"
+        strings.extend(qualified.encode() + b"\0")
+        hashed = name_hash(qualified)
+        bucket = hashed % bucket_count
+        if buckets[bucket] == 0:
+            buckets[bucket] = index
+            if chains:
+                chains[-1] |= 1
+        chains.append(hashed & ~1)
+        bloom |= (1 << (hashed % 64)) | (1 << ((hashed >> 5) % 64))
+    if chains:
+        chains[-1] |= 1
+    struct.pack_into("<IIIIQ", image, HASH, bucket_count, 1, 1, 5, bloom)
+    struct.pack_into("<" + "I" * (len(buckets) + len(chains)), image, HASH + 24,
+                     *buckets, *chains)
+    image[STRINGS:STRINGS + len(strings)] = strings
+    tags = [(5, STRINGS + 0x1A00), (10, len(strings)), (6, SYMBOLS + 0x1A00), (11, 24),
+            (0x6FFFFEF5, HASH + 0x1A00), (7, 0x2300), (8, 0), (9, 24)]
+    dynamic(image, 0x600, 176, tags)
+    return image, names
+
+
+def main():
+    relinker = Path(sys.argv[1]).resolve()
+    with tempfile.TemporaryDirectory(prefix="anyps5-guest-gnu-hash-") as directory:
+        work = Path(directory)
+
+        def convert(label, windows, image, names):
+            case = work / label / str(windows)
+            modules = case / "sce_module"
+            modules.mkdir(parents=True)
+            guest = modules / "provider.prx"
+            guest.write_bytes(image)
+            source = case / "input.elf"
+            source.write_bytes(executable("provider.prx", names[-1] + "#A#B") if names else main_fixture())
+            output = case / ("output.exe" if windows else "output.elf")
+            result = subprocess.run([str(relinker), *(["--windows"] if windows else []), str(source), str(output)],
+                                    capture_output=True, text=True, timeout=10)
+            assert guest.read_bytes() == image
+            return result, output, case / "app0/sce_module/provider.prx.guest.prx"
+
+        for label, requested, bucket_count in (("multiple-buckets", ("alpha", "bravo", "charlie"), 3),
+                                                ("one-chain", ("alpha", "bravo", "charlie"), 1),
+                                                ("one-symbol", ("alpha",), 3),
+                                                ("no-hashed-symbols", (), 3)):
+            image, names = module(requested, bucket_count)
+            for windows in (False, True):
+                result, output, artifact = convert(label, windows, image, names)
+                assert result.returncode == 0, (label, windows, result.stdout, result.stderr)
+                if windows and os.name == "nt":
+                    run = subprocess.run([str(output)], capture_output=True, text=True, timeout=10)
+                    assert run.returncode == (40 + len(names) if names else 42), (label, run.returncode, run.stdout, run.stderr)
+                if not windows:
+                    data = artifact.read_bytes()
+                    tags = dynamic_tags(data)
+                    symbol_count, = struct.unpack_from("<I", data, file_offset(data, tags[4]) + 4)
+                    assert symbol_count == len(names) + 1, (label, symbol_count)
+
+        for label, offset, value, error in (("zero-buckets", 0, 0, "Invalid GNU hash header"),
+                                             ("zero-bloom", 8, 0, "Invalid GNU hash header"),
+                                             ("zero-first-symbol", 4, 0, "Invalid GNU hash header"),
+                                             ("large-bloom", 8, 0xFFFFFFFF, "Unmapped ELF address")):
+            image, names = module()
+            struct.pack_into("<I", image, HASH + offset, value)
+            result, output, artifact = convert(label, False, image, names)
+            assert result.returncode == 2 and error in result.stderr, (label, result.stderr)
+            assert not output.exists() and not artifact.exists()
+
+        image, names = module()
+        struct.pack_into("<Q", image, 0x600 + 4 * 16 + 8, 0x29E0)
+        struct.pack_into("<IIIIQII", image, 0xFE0, 1, 1, 1, 5, 0, 1, 2)
+        result, output, artifact = convert("unterminated-chain", False, image, names)
+        assert result.returncode == 2 and "Unmapped ELF address" in result.stderr, result.stderr
+        assert not output.exists() and not artifact.exists()
+    print("GNU hash guest module tests passed")
+
+
+if __name__ == "__main__":
+    main()

core/relinker/relinker/tests/test_guest_module_directories.py

@@ -118,9 +118,12 @@ def convert(case, windows, options=()):
                 (case / "prx" / "provider.prx").write_bytes(module_with_symbol(True))
                 result, output = convert(case, windows)
                 assert result.returncode == 0 and output.exists(), result.stderr
+                warning = "Skipping SELF container: sce_module/libc.prx (unwrap it before relinking)"
+                assert result.stdout.count(warning) == 1, result.stdout
                 assert {path.name for path in (case / "app0").rglob("*.guest.prx")} == {"provider.prx.guest.prx"}
                 result, output = convert(case, windows, ["--exclude-sce-module", "libc.prx"])
                 assert result.returncode == 0 and output.exists(), result.stderr
+                assert warning not in result.stdout, result.stdout
                 assert {path.name for path in (case / "app0").rglob("*.guest.prx")} == {"provider.prx.guest.prx"}
 
             case = work / f"{windows}-ambiguous"