sync(upstream): integrate Batch 33 dialogs, savedata events, reaper thread, and macos tls relinker fixes

core/libs/nid/include/nid/ElfPatcher.hpp

@@ -91,6 +91,7 @@ inline bool IsDynamicNameTag(std::int64_t tag) {
     return tag == kDtNeeded || tag == kDtSoname || tag == kDtRpath || tag == kDtRunpath;
 }
 constexpr std::uint32_t kShtDynamic = 6u;
+constexpr std::uint32_t kShtHash = 5u;
 constexpr std::uint32_t kShtGnuVerdef = 0x6ffffffdu;
 constexpr std::uint32_t kShtGnuVerneed = 0x6ffffffeu;
 constexpr std::uint32_t kShtDynsym = 11u;

core/libs/nid/src/ElfNidPatcher.cpp

@@ -27,6 +27,44 @@ std::uint32_t GnuHash(const std::string& name) {
     return h;
 }
 
+std::uint32_t SysvHash(const std::string& name) {
+    std::uint32_t hash = 0u;
+    for (const unsigned char byte : name) {
+        hash = (hash << 4u) + byte;
+        const auto high = hash & 0xf0000000u;
+        hash ^= high >> 24u;
+        hash &= ~high;
+    }
+    return hash;
+}
+
+void ValidateSysvHash(const std::vector<std::uint8_t>& elf, std::size_t offset, std::size_t size, std::size_t symCount) {
+    using namespace Internal;
+    RequireElfRange(elf, offset, size, "hash");
+    if (size < 8u) throw std::runtime_error("invalid ELF hash header");
+    const auto buckets = Read<std::uint32_t>(elf, offset);
+    const auto chains = Read<std::uint32_t>(elf, offset + 4u);
+    if (!buckets || chains != symCount || 8u + (std::uint64_t{buckets} + chains) * 4u > size)
+        throw std::runtime_error("invalid ELF hash geometry");
+}
+
+void RebuildSysvHash(std::vector<std::uint8_t>& elf, std::size_t offset, const std::vector<std::string>& symbolNames) {
+    using namespace Internal;
+    const auto bucketCount = Read<std::uint32_t>(elf, offset);
+    std::vector<std::uint32_t> buckets(bucketCount, 0u);
+    std::vector<std::uint32_t> chains(symbolNames.size(), 0u);
+    for (std::size_t index = 1u; index < symbolNames.size(); ++index) {
+        if (symbolNames[index].empty()) continue;
+        const auto bucket = SysvHash(symbolNames[index]) % bucketCount;
+        chains[index] = buckets[bucket];
+        buckets[bucket] = static_cast<std::uint32_t>(index);
+    }
+    for (std::size_t index = 0u; index < buckets.size(); ++index)
+        Write(elf, offset + 8u + index * 4u, buckets[index]);
+    for (std::size_t index = 0u; index < chains.size(); ++index)
+        Write(elf, offset + 8u + buckets.size() * 4u + index * 4u, chains[index]);
+}
+
 
 GnuHashLayout ReadGnuHashLayout(const std::vector<std::uint8_t>& elf, std::size_t sectionOffset) {
     using namespace Internal;
@@ -328,6 +366,8 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
 
     std::size_t dynSymOffset = 0u, dynSymSize = 0u;
     std::size_t gnuHashOffset = 0u, gnuHashSize = 0u;
+    std::size_t sysvHashOffset = 0u, sysvHashSize = 0u;
+    std::uint32_t sysvHashLink = 0u;
     std::size_t versymOffset = 0u;
     std::uint32_t dynSymSectionIndex = 0u;
     std::uint32_t gnuHashSymOffset = 0u;
@@ -346,6 +386,11 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
             gnuHashOffset = static_cast<std::size_t>(shdr.sh_offset);
             gnuHashSize = static_cast<std::size_t>(shdr.sh_size);
         }
+        if (shdr.sh_type == kShtHash) {
+            sysvHashOffset = static_cast<std::size_t>(shdr.sh_offset);
+            sysvHashSize = static_cast<std::size_t>(shdr.sh_size);
+            sysvHashLink = shdr.sh_link;
+        }
         if (shdr.sh_type == kShtGnuVersym)
             versymOffset = static_cast<std::size_t>(shdr.sh_offset);
         if (shdr.sh_type == kShtGnuVerdef)
@@ -356,6 +401,10 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
 
     const std::size_t symCount = dynSymSize / sizeof(Elf64_Sym);
     if (symCount == 0u) throw std::runtime_error(".dynsym is empty");
+    if (sysvHashOffset != 0u) {
+        if (sysvHashLink != dynSymSectionIndex) throw std::runtime_error("invalid ELF hash symbol table link");
+        ValidateSysvHash(elf, sysvHashOffset, sysvHashSize, symCount);
+    }
     if (gnuHashOffset != 0u) {
         ValidateGnuHash(elf, gnuHashOffset, gnuHashSize, symCount);
         gnuHashSymOffset = Read<std::uint32_t>(elf, gnuHashOffset + 4u);
@@ -510,6 +559,7 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
         FixupRelocationSymbolIndices(elf, ehdr, dynSymSectionIndex, oldToNewIndex);
         RebuildGnuHash(elf, gnuHashOffset, gnuHashSize, symCount, newSymbolNames);
     }
+    if (sysvHashOffset != 0u) RebuildSysvHash(elf, sysvHashOffset, newSymbolNames);
 }
 
 }

core/libs/nid/src/PeNidPatcher.cpp

@@ -8,6 +8,7 @@
 #include <limits>
 #include <stdexcept>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 namespace Nid {
@@ -116,7 +117,45 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
     if (stringsRegionStart == std::numeric_limits<std::uint32_t>::max())
         throw std::runtime_error("could not determine strings region start");
 
-    const std::uint32_t rawLimit = ((edataSection.SizeOfRawData + fileAlignment - 1u) / fileAlignment) * fileAlignment;
+    std::uint32_t rawLimit = ((edataSection.SizeOfRawData + fileAlignment - 1u) / fileAlignment) * fileAlignment;
+    const auto dllName = ReadCStr(pe, RvaToOffset(pe, exportTable.Name, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
+    const bool repackDllName = exportTable.Name >= edataSection.VirtualAddress &&
+        exportTable.Name - edataSection.VirtualAddress < rawLimit;
+    if (repackDllName) stringsRegionStart = std::min(stringsRegionStart, exportTable.Name - edataSection.VirtualAddress);
+    struct Forwarder {
+        std::size_t EntryOffset;
+        std::string Target;
+    };
+    std::vector<Forwarder> forwarders;
+    std::vector<std::uint32_t> directRvas;
+    const auto functionsArrayOffset = RvaToOffset(pe, exportTable.AddressOfFunctions, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+    for (std::uint32_t index = 0u; index < exportTable.NumberOfFunctions; ++index) {
+        const auto entryOffset = functionsArrayOffset + index * sizeof(std::uint32_t);
+        const auto rva = Read<std::uint32_t>(pe, entryOffset);
+        if (rva < exportDir.VirtualAddress || std::uint64_t{rva} - exportDir.VirtualAddress >= exportDir.Size) {
+            directRvas.push_back(rva);
+            continue;
+        }
+        const auto target = ReadCStr(pe, RvaToOffset(pe, rva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
+        if (target.size() + 1u > exportDir.Size - (rva - exportDir.VirtualAddress))
+            throw std::runtime_error("forwarder string outside export directory");
+        forwarders.push_back({entryOffset, target});
+        stringsRegionStart = std::min(stringsRegionStart, rva - edataSection.VirtualAddress);
+    }
+    for (const auto rva : directRvas) {
+        if (rva >= edataSection.VirtualAddress && rva - edataSection.VirtualAddress >= stringsRegionStart)
+            rawLimit = std::min(rawLimit, rva - edataSection.VirtualAddress);
+    }
+    const auto reserveTable = [&](std::uint32_t rva, std::uint64_t size) {
+        if (rva < edataSection.VirtualAddress || rva - edataSection.VirtualAddress >= rawLimit) return;
+        const auto end = std::uint64_t{rva - edataSection.VirtualAddress} + size;
+        if (end > rawLimit) throw std::runtime_error("export table outside edata section");
+        stringsRegionStart = std::max(stringsRegionStart, static_cast<std::uint32_t>(end));
+    };
+    reserveTable(exportDir.VirtualAddress, sizeof(PeExportDirectory));
+    reserveTable(exportTable.AddressOfFunctions, std::uint64_t{exportTable.NumberOfFunctions} * 4u);
+    reserveTable(exportTable.AddressOfNames, std::uint64_t{exportTable.NumberOfNames} * 4u);
+    reserveTable(exportTable.AddressOfNameOrdinals, std::uint64_t{exportTable.NumberOfNames} * 2u);
 
     std::vector<std::string> finalNames(exportTable.NumberOfNames);
     for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
@@ -125,28 +164,48 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
         finalNames[i] = it->second;
     }
 
-    std::uint32_t requiredSize = stringsRegionStart;
-    for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i)
-        requiredSize += static_cast<std::uint32_t>(finalNames[i].size()) + 1u;
-
+    std::vector<std::uint8_t> strings;
+    std::unordered_map<std::string, std::uint32_t> stringRvas;
+    const auto storeString = [&](const std::string& name) {
+        const auto existing = stringRvas.find(name);
+        if (existing != stringRvas.end()) return existing->second;
+        const auto rva = edataSection.VirtualAddress + stringsRegionStart + static_cast<std::uint32_t>(strings.size());
+        stringRvas.emplace(name, rva);
+        strings.insert(strings.end(), name.begin(), name.end());
+        strings.push_back(0u);
+        return rva;
+    };
+    std::vector<std::uint32_t> forwarderRvas;
+    for (const auto& forwarder : forwarders) forwarderRvas.push_back(storeString(forwarder.Target));
+    if (!forwarders.empty() && (std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart < exportDir.VirtualAddress ||
+        std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart + strings.size() > std::uint64_t{exportDir.VirtualAddress} + exportDir.Size))
+        throw std::runtime_error("not enough export directory space to repack forwarders");
+    const auto dllNameRva = repackDllName ? storeString(dllName) : exportTable.Name;
+    std::vector<std::uint32_t> finalNameRvas;
+    for (const auto& name : finalNames) finalNameRvas.push_back(storeString(name));
+    const auto requiredSize = std::uint64_t{stringsRegionStart} + strings.size();
     if (requiredSize > rawLimit)
         throw std::runtime_error("not enough raw space in edata section to repack export names");
-
-    std::uint32_t writeOffset = stringsRegionStart;
-    for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
-        const std::uint32_t newNameRva = edataSection.VirtualAddress + writeOffset;
-        const std::size_t newNameFileOffset = static_cast<std::size_t>(edataSection.PointerToRawData) + writeOffset;
-
-        if (newNameFileOffset + finalNames[i].size() + 1u > pe.size())
-            throw std::runtime_error("repack write out of file bounds");
-
-        std::memcpy(pe.data() + newNameFileOffset, finalNames[i].data(), finalNames[i].size());
-        pe[newNameFileOffset + finalNames[i].size()] = 0u;
-        Write(pe, namesArrayOffset + i * 4u, newNameRva);
-        writeOffset += static_cast<std::uint32_t>(finalNames[i].size()) + 1u;
+    const auto directoryEnd = std::max(std::uint64_t{exportDir.VirtualAddress} + exportDir.Size,
+        std::uint64_t{edataSection.VirtualAddress} + requiredSize);
+    for (const auto rva : directRvas) {
+        if (rva != 0u && rva >= exportDir.VirtualAddress && std::uint64_t{rva} < directoryEnd)
+            throw std::runtime_error("repacked export directory would cover a direct export");
     }
-
-    const std::uint32_t newVirtualSize = std::max(edataSection.VirtualSize, writeOffset);
+    const auto directorySize = directoryEnd - exportDir.VirtualAddress;
+    if (directorySize > std::numeric_limits<std::uint32_t>::max()) throw std::runtime_error("export directory too large");
+    const auto stringFileOffset = static_cast<std::size_t>(edataSection.PointerToRawData) + stringsRegionStart;
+    if (stringFileOffset > pe.size() || strings.size() > pe.size() - stringFileOffset)
+        throw std::runtime_error("repack write out of file bounds");
+    std::memcpy(pe.data() + stringFileOffset, strings.data(), strings.size());
+    for (std::size_t index = 0u; index < finalNameRvas.size(); ++index)
+        Write(pe, namesArrayOffset + index * 4u, finalNameRvas[index]);
+    for (std::size_t index = 0u; index < forwarders.size(); ++index)
+        Write(pe, forwarders[index].EntryOffset, forwarderRvas[index]);
+    Write(pe, exportDirOffset + offsetof(PeExportDirectory, Name), dllNameRva);
+    Write(pe, dataDirectoryOffset + offsetof(PeDataDirectory, Size), static_cast<std::uint32_t>(directorySize));
+
+    const std::uint32_t newVirtualSize = std::max(edataSection.VirtualSize, static_cast<std::uint32_t>(requiredSize));
     auto patchedSection = edataSection;
     patchedSection.VirtualSize = newVirtualSize;
     Write(pe, edataSectionOffset, patchedSection);

core/libs/nid/tests/test_gnu_hash.py

@@ -3,7 +3,32 @@
 import unittest
 from pathlib import Path
 
-from test_symbol_versions import build_image, symbols, patch, HASH_OFFSET, SHOFF, read_dynsym, gnu_hash
+from test_symbol_versions import build_image, symbols, patch, HASH_OFFSET, SHOFF, read_dynsym, gnu_hash, sections
+
+
+def sysv_hash(name):
+    value = 0
+    for byte in name.encode("utf-8"):
+        value = ((value << 4) + byte) & 0xFFFFFFFF
+        high = value & 0xF0000000
+        value = (value ^ (high >> 24)) & ~high
+    return value
+
+
+def sysv_lookup(image, name):
+    table = next(section for section in sections(image) if section[1] == 5)
+    buckets, chains = struct.unpack_from("<II", image, table[4])
+    entries = read_dynsym(image)
+    index = struct.unpack_from("<I", image, table[4] + 8 + (sysv_hash(name) % buckets) * 4)[0]
+    visited = set()
+    while index:
+        if index >= chains or index in visited:
+            raise AssertionError("invalid SysV hash chain")
+        visited.add(index)
+        if entries[index][0] == name:
+            return index
+        index = struct.unpack_from("<I", image, table[4] + 8 + buckets * 4 + index * 4)[0]
+    return None
 
 
 class GnuHashTests(unittest.TestCase):
@@ -70,6 +95,63 @@ def test_empty_hash_with_no_buckets_is_supported(self):
         self.assertEqual(struct.unpack_from("<I", after, HASH_OFFSET)[0], 0)
 
 
+class SysvHashTests(unittest.TestCase):
+    def image(self, both, buckets):
+        entries = symbols(False)
+        entries.append(("retained_nid_no_patch", 1, 9, 2, 0x1400))
+        entries[3:] = sorted(entries[3:], key=lambda entry: gnu_hash(entry[0]) % 4)
+        image = build_image(entries, False)
+        offset = 0x600
+        size = 8 + (buckets + len(entries)) * 4
+        struct.pack_into("<II", image, offset, buckets, len(entries))
+        for index, entry in enumerate(entries[1:], 1):
+            bucket_offset = offset + 8 + (sysv_hash(entry[0]) % buckets) * 4
+            head = struct.unpack_from("<I", image, bucket_offset)[0]
+            struct.pack_into("<I", image, offset + 8 + buckets * 4 + index * 4, head)
+            struct.pack_into("<I", image, bucket_offset, index)
+        section = (0, 5, 2, 0, offset, size, 1, 0, 4, 4)
+        struct.pack_into("<IIQQQQIIQQ", image, SHOFF + (4 if both else 3) * 64, *section)
+        if both:
+            struct.pack_into("<H", image, 0x3C, 5)
+        return image
+
+    def test_rebuilt_sysv_hash_finds_renamed_and_preserved_symbols(self):
+        for both in (False, True):
+            for buckets in (1, 3, 7):
+                with self.subTest(both=both, buckets=buckets):
+                    image = self.image(both, buckets)
+                    before = read_dynsym(image)
+                    for name, _, _ in before[1:]:
+                        self.assertIsNotNone(sysv_lookup(image, name), name)
+                    result, after = patch(PATCHER, image, "sysv.so")
+                    self.assertEqual(result.returncode, 0, result.stderr)
+                    entries = read_dynsym(after)
+                    if not both:
+                        self.assertEqual([(entry[1], entry[2]) for entry in entries],
+                                         [(entry[1], entry[2]) for entry in before])
+                    else:
+                        self.assertNotEqual([entry[2] for entry in entries], [entry[2] for entry in before])
+                    for index, (name, _, _) in enumerate(entries[1:], 1):
+                        self.assertEqual(sysv_lookup(after, name), index, name)
+                    self.assertEqual(entries[sysv_lookup(after, "retained_nid_no_patch")][2], 0x1400)
+                    for name, _, value in before[3:]:
+                        if name != "retained_nid_no_patch":
+                            self.assertIsNone(sysv_lookup(after, name), name)
+                            renamed = next(entry[0] for entry in entries if entry[2] == value)
+                            self.assertRegex(renamed, r"^[A-Za-z0-9+-]{11}$")
+                    self.assertIsNone(sysv_lookup(after, "missing_nid_no_patch"))
+
+    def test_invalid_sysv_geometry_is_rejected_without_writing(self):
+        for field, value in ((0, 0), (4, 1000), (4, 1)):
+            with self.subTest(field=field, value=value):
+                image = self.image(False, 3)
+                struct.pack_into("<I", image, 0x600 + field, value)
+                result, after = patch(PATCHER, image, "invalid-sysv.so")
+                self.assertEqual(result.returncode, 2, result.stderr)
+                self.assertIn(b"hash", result.stderr)
+                self.assertEqual(after, image)
+
+
 if __name__ == "__main__":
     PATCHER = Path(sys.argv.pop(1)).resolve()
     unittest.main()