From d8bc9e8c1e4f6d602d0763cb35f755015c49dc56 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Sun, 12 Apr 2026 19:44:44 +0300 Subject: [PATCH] Add vendor binary ABI audit tool, replace libhisicompat with uclibc-compat - Add general/scripts/audit-vendor-abi.py: checks vendor imports against actual musl libc.so exports and compiles struct size probes with the platform cross-compiler. Auto-downloads per-platform toolchains. - hi3516cv100: replace libhisicompat (sTaT hack) with uclibc-compat dependency. Restore original vendor libmpi.so that imports standard stat() instead of the custom sTaT workaround. Delete libhisicompat/. - Register uclibc-compat in general/package/Config.in Ref: #1992 Co-Authored-By: Claude Opus 4.6 (1M context) --- Makefile | 3 + general/package/Config.in | 1 + .../files/lib/libmpi.so | Bin 404225 -> 404225 bytes .../hisilicon-osdrv-hi3516cv100.mk | 6 +- .../libhisicompat/Makefile | 14 - .../libhisicompat/libhisicompat.so | Bin 7132 -> 0 bytes .../libhisicompat/sTaT.c | 23 - general/scripts/audit-vendor-abi.allow | 7 + general/scripts/audit-vendor-abi.py | 967 ++++++++++++++++++ 9 files changed, 979 insertions(+), 42 deletions(-) delete mode 100644 general/package/hisilicon-osdrv-hi3516cv100/libhisicompat/Makefile delete mode 100755 general/package/hisilicon-osdrv-hi3516cv100/libhisicompat/libhisicompat.so delete mode 100644 general/package/hisilicon-osdrv-hi3516cv100/libhisicompat/sTaT.c create mode 100644 general/scripts/audit-vendor-abi.allow create mode 100755 general/scripts/audit-vendor-abi.py diff --git a/Makefile b/Makefile index a8070ace93..bb73e607d9 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,9 @@ clean: distclean: @rm -rf $(BR_FILE) $(TARGET) +audit-abi: + @python3 $(PWD)/general/scripts/audit-vendor-abi.py + deps: sudo apt-get install -y automake autotools-dev bc build-essential cpio \ curl file fzf git libncurses-dev libtool lzop make rsync unzip wget libssl-dev diff --git a/general/package/Config.in b/general/package/Config.in index a777be30ad..61d0f53b64 100644 --- a/general/package/Config.in +++ b/general/package/Config.in @@ -105,6 +105,7 @@ source "$BR2_EXTERNAL_GENERAL_PATH/package/ssv615x-openipc/Config.in" source "$BR2_EXTERNAL_GENERAL_PATH/package/ssv635x-openipc/Config.in" source "$BR2_EXTERNAL_GENERAL_PATH/package/ssw101b/Config.in" source "$BR2_EXTERNAL_GENERAL_PATH/package/uacme-openipc/Config.in" +source "$BR2_EXTERNAL_GENERAL_PATH/package/uclibc-compat/Config.in" source "$BR2_EXTERNAL_GENERAL_PATH/package/uqmi-openipc/Config.in" source "$BR2_EXTERNAL_GENERAL_PATH/package/usrsctp/Config.in" source "$BR2_EXTERNAL_GENERAL_PATH/package/vdec-openipc/Config.in" diff --git a/general/package/hisilicon-osdrv-hi3516cv100/files/lib/libmpi.so b/general/package/hisilicon-osdrv-hi3516cv100/files/lib/libmpi.so index 0862f1ed5b50e8ca3eb8a7bc0806382c18cb371b..72b687a4007d4aa4313e30fff260d2147a2bf648 100644 GIT binary patch delta 58 zcmZp?Cee6JV#6h8=90vc>2D?2B%9wkx4&~{1Y#y2W(H!G?eCmfpWOzl*#DYEZ2Mbz HHW6+B46Yf5 delta 58 zcmZp?Cee6JV#6h8=8(jY>2D?2B%9wkx4&~{1Y#y2W(H!G?eCmfpWOzl*#DYEZ2Mbz HHW6+B{ht{I diff --git a/general/package/hisilicon-osdrv-hi3516cv100/hisilicon-osdrv-hi3516cv100.mk b/general/package/hisilicon-osdrv-hi3516cv100/hisilicon-osdrv-hi3516cv100.mk index 6066594e67..a8fbaeba1a 100644 --- a/general/package/hisilicon-osdrv-hi3516cv100/hisilicon-osdrv-hi3516cv100.mk +++ b/general/package/hisilicon-osdrv-hi3516cv100/hisilicon-osdrv-hi3516cv100.mk @@ -8,10 +8,7 @@ HISILICON_OSDRV_HI3516CV100_VERSION = HISILICON_OSDRV_HI3516CV100_SITE = HISILICON_OSDRV_HI3516CV100_LICENSE = MIT HISILICON_OSDRV_HI3516CV100_LICENSE_FILES = LICENSE - -define HISILICON_OSDRV_HI3516CV100_BUILD_CMDS - $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(HISILICON_OSDRV_HI3516CV100_PKGDIR)/libhisicompat all -endef +HISILICON_OSDRV_HI3516CV100_DEPENDENCIES = uclibc-compat define HISILICON_OSDRV_HI3516CV100_INSTALL_TARGET_CMDS $(INSTALL) -m 755 -d $(TARGET_DIR)/etc/sensors @@ -92,7 +89,6 @@ define HISILICON_OSDRV_HI3516CV100_INSTALL_TARGET_CMDS $(INSTALL) -m 644 -t $(TARGET_DIR)/usr/lib $(HISILICON_OSDRV_HI3516CV100_PKGDIR)/files/lib/libVoiceEngine.so $(INSTALL) -m 644 -t $(TARGET_DIR)/usr/lib $(HISILICON_OSDRV_HI3516CV100_PKGDIR)/files/lib/libvqev2.so - $(INSTALL) -D -m 0755 $(HISILICON_OSDRV_HI3516CV100_PKGDIR)/libhisicompat/libhisicompat.so $(TARGET_DIR)/usr/lib endef $(eval $(generic-package)) diff --git a/general/package/hisilicon-osdrv-hi3516cv100/libhisicompat/Makefile b/general/package/hisilicon-osdrv-hi3516cv100/libhisicompat/Makefile deleted file mode 100644 index c3eb1a0d44..0000000000 --- a/general/package/hisilicon-osdrv-hi3516cv100/libhisicompat/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -LIB_NAME := libhisicompat - -SRCS := sTaT.c - -OBJS := sTaT.o - -all: $(LIB_NAME).so - -$(LIB_NAME).so: $(OBJS) - $(CC) -shared -o $@ $(OBJS) - -clean: - @rm -f $(OBJS) $(LIB_NAME).so - diff --git a/general/package/hisilicon-osdrv-hi3516cv100/libhisicompat/libhisicompat.so b/general/package/hisilicon-osdrv-hi3516cv100/libhisicompat/libhisicompat.so deleted file mode 100755 index cc4ff80ddbe1d4f3aafc3141723b69e3c1779496..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7132 zcmeHMZ)_Y#6`%Flt?ef1C2<;4QgW({a3$Z~Cb1)f0y@Wbl7iiu#2;vdv{~P+eOvBs zPrG|1rbsxbh#+d2KUD>&sgH^e{XnG(38_da1R_{{QyEJ;`se$ zZrrKsqVf$SKk40jZ{C|XZ{Fe)`8Z0i~lmqBD3eg0zg zoqhT5Uj6D@Z@j&a?b~694ge|X4f$juq?{I$l2bnqf}A(x4ESdZ`2hHb3^@ay^V5(4 z^9?P4(*6lU&Vc90jED0*fOu9w)0osLDfh_;XCNy*X6**mAQA2&h;SdCpLL6Vnr=c~S8zv5z3! zIPnrU2`c`{QBe*S3U(YKZa!a!K}R~2`K6ozJ%qUqkzzu_q*&w;QY@@PiU(s4=zXwD zlfrI*6ge0q#lv%i6n0iV`)4cv+U<|Ea~s{wY^%Gu-rCn(Yxe&2m#yo2klEj^ZKi+s z%;xnw*IJr?)!<(-_?Ha60X{F*TX_eVS#Jrk)=W8TO(8C9bY<3>7|$AkuJ^|5X3F{A zt!`%(HWzMU#QxqJtD6TA4=LmN3F2q~F9Ju9gBkdpLEkN(>B%O?nAqRg`0>%p&0nWq z`LD;AgYOyOoqO8Q5NgB ze%5ZkbsadqO+SK|8Sl%`4MAsR8aJii%x_xocl$QReF>Ps*w;XZK(7P81KxqXgZa_E z%lW<>6HDHoNw2pu@XI)@G-F$9z5xGetmn_$?Y|7OFMON>y$I^S?s?=}%r!Qwd}HGW zh(V+qH}{Ih#wz;lO*b}rz<-lIoU5(0^K|QZtbv30%;yBIc40RIyBXNcz-|V1Gq9V1 z-3;tz;Qu%Suj77j3dnoZkAS=@@;=3T81GFV0p)$>g=;VtuHNtq902YI@_us%jpVO^@~)Q$9t8FR|JiPb zvM=}#1H6B0aL{o(YMn`W&pJCf`KiqC>3UEuMq!v_vP<6a;^K+oiN$O&h?3=pGUN8B z?TCp_pr_}13xTj60NXP*KmA#8S5L3iXC3Gsunt*=Q-jw1oaXz0`68rb)^Q)}gqo6B zQF&8(HDl&e$;`IOkCK^fl{Y0b>uUX!e7BT2cO^5ELe@Z(%qoIfXC-sBg!qDTi}+N9 zf$L95bt(y5e<~Zcn z4L$AOG~}buYgyZmsAYtu>aX*|H-!E9J}T%FW9YAC-T$YC{&UbXpCG{fG9JA?e2X84 zLdzCp`sX{Zpif+eUd!6Qe2Jw&pIC+d6#TPXrv2;ZfxE>Iq1Unnnfdr+XMOoLzY1A{ zUJur16!eMh`DcHff9@K;GGwlY3H0{|==JkwLH-kD{d`%-=O$#GU&?Po=Ke#VJ@cX8 z5AHM^Kfkqpy)s`@hTLPk(GQ~U0A#cNWWMnk=#Lxra!)g4>K}(})~noY`lk$;{l5;` ztY=*@^zRFwl66Gaiz08?^{Yv=EK~)sSg%x;MY!;UUr5mXq=W8;MiwF8 z(~WgOMCiI_&QHuvyVH+Mxvp@h9(!bB_Tfo1qc*7<-C`WNC9hg!mGbP&qo*fk+(*xx zxiCHN&QF}4nMS{D6-rg|{J=V;S%s7(QC+xvPeD{xF8@=-R97`sX?4fyr?BJYO5!bm zCXu2g%`H{ycFBuN!Y(dXVWwyj2|M!3Y*S3FoCsS^$WHuA;N@_(UsCy1I+a?b(>y$Q zm7pN(r4YS}Ug8ORA&!NOb*^CTggtS7*7lMl3Kr@K9*hqSM`6h=jjtlV?Zk6Hy=`#2 zCGrgQcbsEr$8SE7HO=jIA4eN^I@<9JMdbeo^g&xvo~5!KnRYxQ5MKnZ0o^2w;g?@} z+VN~f%!4<_?S=dqAni^Ac@`rMgQp$8BtAzJq0~;LOXtw_+OA-H#AA#%-_>MPLFo-TZF|NCEo##N&FU2+wmJV z3OlxI(EJ)2=$HQZ?Hh+3&*vJn-Lq(*Kc}Oh9oJU_b;PSc_^Rpli#ym+M_cAt!xh-o rFxVk9d>Y7c(IlTKIalz6e+&rmg3*zF2s{m%cF*Bb@UqmXA8q$ -#include -#include -#include - -struct ugly_fake_stat { - unsigned long long A; - unsigned long long B; - unsigned int st_mode; /* File mode. */ -}; - -int sTaT(const char *pathname, struct stat *statbuf) -{ - struct stat good_struct_stat; - memset(&good_struct_stat, 0, sizeof(struct stat)); - int ret = stat(pathname, &good_struct_stat); - - struct ugly_fake_stat* ugly = (struct ugly_fake_stat*)statbuf; - ugly->st_mode = good_struct_stat.st_mode; - - return ret; -} - diff --git a/general/scripts/audit-vendor-abi.allow b/general/scripts/audit-vendor-abi.allow new file mode 100644 index 0000000000..aef7491a73 --- /dev/null +++ b/general/scripts/audit-vendor-abi.allow @@ -0,0 +1,7 @@ +# Allowlist for audit-vendor-abi.py +# Format: /files/: +# Lines starting with # are comments. +# +# Entries here suppress findings for symbols that are known to be +# handled by compatibility shims or otherwise confirmed safe. +# Only add entries after verifying the shim actually covers the call. diff --git a/general/scripts/audit-vendor-abi.py b/general/scripts/audit-vendor-abi.py new file mode 100755 index 0000000000..7f354a5938 --- /dev/null +++ b/general/scripts/audit-vendor-abi.py @@ -0,0 +1,967 @@ +#!/usr/bin/env python3 +""" +audit-vendor-abi.py - Vendor binary ABI compatibility auditor for OpenIPC +========================================================================== + +WHAT THIS TOOL DOES +------------------- + +OpenIPC ships vendor "osdrv" packages containing prebuilt .so libraries from +chip manufacturers (HiSilicon, Sigmastar, Ingenic, Goke, Rockchip, etc). +These binaries were compiled by vendors against old C libraries -- typically +uclibc from the Linux 3.x era, or older glibc -- but OpenIPC runs them on +modern musl libc. + +This tool performs two definitive checks (no guessing): + + 1. SYMBOL CHECK: extracts every imported symbol from vendor .so files and + checks whether the actual musl libc.so exports it. Any symbol the + vendor needs but musl doesn't provide will fail at dlopen/startup. + + 2. STRUCT PROBE: compiles a small C program with the platform's musl + cross-compiler and runs it under qemu to get exact sizeof/offsetof + for every struct that crosses the vendor-musl boundary (stat, off_t, + pthread_mutex_t, jmp_buf, etc). Reports the actual musl sizes so + they can be compared against known vendor libc layouts. + +REQUIREMENTS +------------ + + - Python 3.10+ + - readelf (from binutils) + - A Buildroot output directory with the musl cross-compiler and sysroot + (i.e., run at least one `make BOARD=...` before using this tool) + - qemu-arm-static or qemu-arm (to run the compiled struct probe) + +USAGE +----- + + # Audit a specific package (MVP) + ./general/scripts/audit-vendor-abi.py --package hisilicon-osdrv-hi3516ev200 + + # Audit all packages + ./general/scripts/audit-vendor-abi.py + + # Just check symbols, skip struct probe (no compiler/qemu needed) + ./general/scripts/audit-vendor-abi.py --symbols-only + + # Via Makefile + make audit-abi + +EXIT CODES +---------- + + 0 No missing symbols or struct mismatches detected + 1 Missing symbols found (vendor .so will fail to load on musl) + 2 Struct size mismatches found (runtime corruption risk) +""" + +import argparse +import os +import re +import subprocess +import sys +import tempfile +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + +@dataclass +class BinaryInfo: + path: str # absolute path + rel_path: str # relative to package dir + imports: list # [(binding, symbol), ...] + needed: list # NEEDED entries from .dynamic + version_reqs: list # GLIBC_2.x etc version tags + + +@dataclass +class PackageResult: + name: str + source_libc: str = "unknown" + arch: str = "" + so_count: int = 0 + binaries: list = field(default_factory=list) + missing_symbols: dict = field(default_factory=lambda: defaultdict(list)) + # maps symbol -> [binary_rel_paths] + struct_mismatches: list = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Struct types that cross the vendor-musl boundary. +# Each entry: (header, type_expr, fields_with_offsets) +# Fields is a list of (field_name, access_expr) for offsetof probing. +# --------------------------------------------------------------------------- + +STRUCT_PROBES = [ + ("sys/stat.h", "struct stat", [ + ("st_dev", "st_dev"), ("st_ino", "st_ino"), ("st_mode", "st_mode"), + ("st_nlink", "st_nlink"), ("st_uid", "st_uid"), ("st_gid", "st_gid"), + ("st_rdev", "st_rdev"), ("st_size", "st_size"), + ]), + ("sys/types.h", "off_t", []), + ("time.h", "time_t", []), + ("time.h", "struct timespec", [ + ("tv_sec", "tv_sec"), ("tv_nsec", "tv_nsec"), + ]), + ("pthread.h", "pthread_mutex_t", []), + ("pthread.h", "pthread_cond_t", []), + ("signal.h", "struct sigaction", []), + ("setjmp.h", "jmp_buf", []), + ("dirent.h", "struct dirent", [ + ("d_ino", "d_ino"), ("d_off", "d_off"), + ("d_reclen", "d_reclen"), ("d_type", "d_type"), + ("d_name", "d_name"), + ]), + ("fcntl.h", "struct flock", [ + ("l_type", "l_type"), ("l_whence", "l_whence"), + ("l_start", "l_start"), ("l_len", "l_len"), + ("l_pid", "l_pid"), + ]), +] + +# Symbols that are known to be inter-vendor-library (not libc). +# We skip these in the "missing from musl" report to reduce noise. +# Patterns are checked with str.startswith or exact match. +VENDOR_SYMBOL_PREFIXES = ( + "HI_", "Hi_", "hi_", "MPI_", "mpi_", "ISP_", "isp_", # HiSilicon + "MI_", "mi_", # Sigmastar + "IMP_", "imp_", # Ingenic + "FDK", "pcmDmx", "aac", # Audio codecs + "FD_", "RK_", "rk_", # Rockchip + "XM_", "xm_", # Xiongmai + "GK_", "gk_", "SC_", # Goke + "AW_", "aw_", # Allwinner + "_Z", # C++ mangled +) + +# These are commonly undefined weak symbols or linker internals that +# never resolve and are expected to be NULL. Not a real problem. +LINKER_INTERNALS = { + "__gmon_start__", "_Jv_RegisterClasses", + "__deregister_frame_info", "__register_frame_info", + "_ITM_deregisterTMCloneTable", "_ITM_registerTMCloneTable", +} + + +# --------------------------------------------------------------------------- +# ELF helpers +# --------------------------------------------------------------------------- + +def run_readelf(args: list[str], path: str) -> str: + try: + r = subprocess.run( + ["readelf", "-W"] + args + [path], + capture_output=True, text=True, timeout=10, + ) + return r.stdout + except (subprocess.TimeoutExpired, FileNotFoundError): + return "" + + +def is_elf(path: str) -> bool: + try: + with open(path, "rb") as f: + return f.read(4) == b"\x7fELF" + except OSError: + return False + + +def get_musl_exports(libc_path: str) -> set[str]: + """Get all symbols exported by musl libc.so.""" + out = run_readelf(["--dyn-syms"], libc_path) + exports = set() + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 8 and parts[6] != "UND" and parts[7]: + exports.add(parts[7].split("@")[0]) + return exports + + +def parse_binary(path: str, pkg_dir: str) -> BinaryInfo | None: + """Parse a single .so file, extract imports, NEEDED, versions.""" + if not is_elf(path): + return None + + rel_path = os.path.relpath(path, pkg_dir) + + # Dynamic section for NEEDED + dyn_out = run_readelf(["-d"], path) + needed = re.findall(r"NEEDED.*\[(.+?)\]", dyn_out) + + # Dynamic symbols -- undefined imports + dynsym_out = run_readelf(["--dyn-syms"], path) + imports = [] + for line in dynsym_out.splitlines(): + parts = line.split() + if len(parts) >= 8 and parts[6] == "UND" and parts[7]: + binding = parts[4] + sym = parts[7].split("@")[0] + imports.append((binding, sym)) + + # GNU version requirements + ver_out = run_readelf(["-V"], path) + version_reqs = sorted(set(re.findall( + r"(G(?:LIBC|LIBCXX)_[\d.]+)", ver_out + ))) + + return BinaryInfo( + path=path, rel_path=rel_path, + imports=imports, needed=needed, version_reqs=version_reqs, + ) + + +def detect_source_libc(binaries: list[BinaryInfo]) -> str: + """Detect source libc from NEEDED entries across all binaries.""" + all_needed = set() + all_imports = set() + for b in binaries: + all_needed.update(b.needed) + all_imports.update(sym for _, sym in b.imports) + + if any("ld-uClibc" in n or n == "libc.so.0" for n in all_needed): + return "uclibc" + if any("libc.so.6" in n or "ld-linux" in n for n in all_needed): + return "glibc" + if any(s.startswith("__uClibc_") for s in all_imports): + return "uclibc" + if any(s in ("__xstat", "__fxstat", "__lxstat") for s in all_imports): + return "glibc" + return "unknown" + + +def detect_arch(path: str) -> str: + hdr = run_readelf(["-h"], path) + machine = bits = "" + for line in hdr.splitlines(): + if "Machine:" in line: + machine = line.split("Machine:", 1)[1].strip() + elif "Class:" in line: + c = line.split("Class:", 1)[1].strip() + bits = "32-bit" if "32" in c else "64-bit" if "64" in c else "" + return f"{machine} {bits}".strip() + + +def is_vendor_symbol(sym: str) -> bool: + """True if symbol looks like a vendor-internal (not libc) symbol.""" + if sym in LINKER_INTERNALS: + return True + if any(sym.startswith(p) for p in VENDOR_SYMBOL_PREFIXES): + return True + # Uppercase-start symbols are usually vendor-defined + if sym[0:1].isupper(): + return True + # g_ prefixed globals + if sym.startswith("g_"): + return True + return False + + +# --------------------------------------------------------------------------- +# Struct probe: compile & run under qemu +# --------------------------------------------------------------------------- + +PROBE_SOURCE_TEMPLATE = """\ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main(void) { +%s + return 0; +} +""" + + +def generate_probe_source() -> str: + """Generate C source that prints sizeof/offsetof for all probed types.""" + lines = [] + for header, type_expr, fields in STRUCT_PROBES: + lines.append(f' printf("sizeof({type_expr})=%zu\\n", sizeof({type_expr}));') + for fname, access in fields: + lines.append( + f' printf(" {fname} offset=%zu size=%zu\\n", ' + f'offsetof({type_expr}, {access}), ' + f'sizeof((({type_expr}*)0)->{access}));' + ) + return PROBE_SOURCE_TEMPLATE % "\n".join(lines) + + +def compile_and_run_probe(cc: str) -> dict[str, str] | None: + """Compile the struct probe, run under qemu, return {line: value} dict.""" + source = generate_probe_source() + + with tempfile.TemporaryDirectory() as tmpdir: + src_path = os.path.join(tmpdir, "probe.c") + bin_path = os.path.join(tmpdir, "probe") + + with open(src_path, "w") as f: + f.write(source) + + # Compile static so qemu can run it without sysroot + r = subprocess.run( + [cc, "-o", bin_path, src_path, "-static"], + capture_output=True, text=True, timeout=30, + ) + if r.returncode != 0: + print(f" WARNING: probe compilation failed: {r.stderr.strip()}", file=sys.stderr) + return None + + # Try qemu-arm-static first, then qemu-arm + for qemu in ("qemu-arm-static", "qemu-arm"): + try: + r = subprocess.run( + [qemu, bin_path], + capture_output=True, text=True, timeout=10, + ) + if r.returncode == 0: + return _parse_probe_output(r.stdout) + except FileNotFoundError: + continue + + print(" WARNING: qemu not found, cannot run struct probe", file=sys.stderr) + return None + + +def _parse_probe_output(output: str) -> dict[str, str]: + """Parse probe output into dict preserving original lines.""" + result = {} + for line in output.strip().splitlines(): + line = line.strip() + if "=" in line: + result[line.split("=")[0].strip()] = line.split("=")[1].strip() + return result + + +# Known uclibc ARM 32-bit struct sizes (from uclibc 0.9.33 / ARM EABI). +# These are what vendor binaries compiled against uclibc expect. +UCLIBC_ARM32 = { + "sizeof(struct stat)": "88", + " st_dev offset": "0", + " st_ino offset": "12", + " st_mode offset": "16", + " st_nlink offset": "20", + " st_uid offset": "24", + " st_gid offset": "28", + " st_rdev offset": "32", + " st_size offset": "44", + "sizeof(off_t)": "4", + "sizeof(time_t)": "4", + "sizeof(struct timespec)": "8", + " tv_sec offset": "0", + " tv_nsec offset": "4", + "sizeof(pthread_mutex_t)": "24", + "sizeof(pthread_cond_t)": "48", + "sizeof(struct sigaction)": "140", + "sizeof(jmp_buf)": "392", + "sizeof(struct dirent)": "280", + "sizeof(struct flock)": "24", + " l_type offset": "0", + " l_whence offset": "2", + " l_start offset": "4", + " l_len offset": "8", + " l_pid offset": "12", +} + +# Known glibc ARM 32-bit struct sizes (glibc 2.31 / ARM EABI). +GLIBC_ARM32 = { + "sizeof(struct stat)": "88", + " st_dev offset": "0", + " st_ino offset": "12", + " st_mode offset": "16", + " st_nlink offset": "20", + " st_uid offset": "24", + " st_gid offset": "28", + " st_rdev offset": "32", + " st_size offset": "44", + "sizeof(off_t)": "4", + "sizeof(time_t)": "4", + "sizeof(struct timespec)": "8", + " tv_sec offset": "0", + " tv_nsec offset": "4", + "sizeof(pthread_mutex_t)": "24", + "sizeof(pthread_cond_t)": "48", + "sizeof(struct sigaction)": "140", + "sizeof(jmp_buf)": "392", + "sizeof(struct dirent)": "280", + "sizeof(struct flock)": "24", + " l_type offset": "0", + " l_whence offset": "2", + " l_start offset": "4", + " l_len offset": "8", + " l_pid offset": "12", +} + +KNOWN_VENDOR_LAYOUTS = { + "uclibc": UCLIBC_ARM32, + "glibc": GLIBC_ARM32, +} + +# Functions that pass these structs by pointer (callers allocate the struct, +# so size and layout MUST match). +STRUCT_FUNCTIONS = { + "struct stat": [ + "stat", "fstat", "lstat", "stat64", "fstat64", "lstat64", + "__xstat", "__fxstat", "__lxstat", + "__xstat64", "__fxstat64", "__lxstat64", + ], + "off_t": [ + "mmap", "mmap64", "lseek", "lseek64", + "ftruncate", "ftruncate64", "truncate", "truncate64", + "pread", "pread64", "pwrite", "pwrite64", + "fseeko", "fseeko64", "ftello", "ftello64", + "sendfile", "sendfile64", + ], + "struct flock": ["fcntl", "fcntl64"], + "struct timespec": ["clock_gettime", "clock_getres", "clock_nanosleep", "nanosleep"], + "time_t": ["time", "stime"], + "pthread_mutex_t": [ + "pthread_mutex_init", "pthread_mutex_destroy", + "pthread_mutex_lock", "pthread_mutex_unlock", + "pthread_mutex_trylock", "pthread_mutex_timedlock", + ], + "pthread_cond_t": [ + "pthread_cond_init", "pthread_cond_destroy", + "pthread_cond_wait", "pthread_cond_signal", + "pthread_cond_broadcast", "pthread_cond_timedwait", + ], + "struct sigaction": ["sigaction"], + "jmp_buf": ["setjmp", "_setjmp", "sigsetjmp", "longjmp", "_longjmp", "siglongjmp"], + "struct dirent": ["readdir", "readdir64", "scandir", "scandir64"], +} + +# Reverse: symbol -> struct name +_SYM_TO_STRUCT = {} +for _stype, _syms in STRUCT_FUNCTIONS.items(): + for _s in _syms: + _SYM_TO_STRUCT[_s] = _stype + + +# --------------------------------------------------------------------------- +# Main audit logic +# --------------------------------------------------------------------------- + +def audit_package(pkg_name: str, pkg_dir: str, musl_exports: set[str], + musl_sizes: dict[str, str] | None) -> PackageResult | None: + """Audit all .so files in a package.""" + files_dir = os.path.join(pkg_dir, "files") + if not os.path.isdir(files_dir): + return None + + # Collect .so files + so_files = [] + for root, _dirs, filenames in os.walk(files_dir): + for fn in filenames: + if ".so" in fn: + so_files.append(os.path.join(root, fn)) + so_files.sort() + if not so_files: + return None + + result = PackageResult(name=pkg_name) + + # Parse all binaries + for so in so_files: + info = parse_binary(so, pkg_dir) + if info: + result.binaries.append(info) + result.so_count += 1 + + if not result.binaries: + return None + + result.source_libc = detect_source_libc(result.binaries) + result.arch = detect_arch(result.binaries[0].path) + + # Phase 1: Missing symbols (skip vendor-internal and linker symbols) + for binfo in result.binaries: + for binding, sym in binfo.imports: + if sym in musl_exports: + continue + if is_vendor_symbol(sym): + continue + if binding == "WEAK" and sym in LINKER_INTERNALS: + continue + result.missing_symbols[sym].append(binfo.rel_path) + + # Phase 2: Struct mismatches + if musl_sizes: + vendor_sizes = KNOWN_VENDOR_LAYOUTS.get(result.source_libc) + if vendor_sizes: + # Find which struct-passing functions are actually imported + imported_syms = set() + for binfo in result.binaries: + for _, sym in binfo.imports: + imported_syms.add(sym) + + # Check each struct type + for stype, func_list in STRUCT_FUNCTIONS.items(): + used_funcs = [f for f in func_list if f in imported_syms] + if not used_funcs: + continue + + # Compare all sizes/offsets for this struct type + mismatches = [] + for key, musl_val in musl_sizes.items(): + if not key.startswith(f"sizeof({stype})") and \ + not (key.startswith(" ") and f"sizeof({stype})" in + _find_parent_sizeof(key, musl_sizes)): + # Check if this key belongs to this struct + if not _key_belongs_to_struct(key, stype, musl_sizes): + continue + vendor_val = vendor_sizes.get(key) + if vendor_val is not None and vendor_val != musl_val: + mismatches.append((key, vendor_val, musl_val)) + + if mismatches: + # Which binaries use these functions? + affected = [] + for binfo in result.binaries: + bsyms = {s for _, s in binfo.imports} + if bsyms & set(used_funcs): + affected.append(binfo.rel_path) + + result.struct_mismatches.append({ + "struct": stype, + "functions": used_funcs, + "binaries": affected, + "mismatches": mismatches, + }) + + return result + + +def _key_belongs_to_struct(key: str, stype: str, sizes: dict) -> bool: + """Check if an offset key belongs to a given struct type.""" + if key.startswith(f"sizeof({stype})"): + return True + # For field-level offsets, we use the STRUCT_PROBES definition + for header, type_expr, fields in STRUCT_PROBES: + if type_expr == stype: + for fname, _ in fields: + if key.strip().startswith(fname): + return True + return False + + +def _find_parent_sizeof(key: str, sizes: dict) -> str: + """Not used currently but kept for future expansion.""" + return "" + + +# --------------------------------------------------------------------------- +# Report +# --------------------------------------------------------------------------- + +def print_report(result: PackageResult, quiet: bool) -> None: + print(f"Package: {result.name}") + print(f" Source libc: {result.source_libc}") + print(f" Architecture: {result.arch}") + print(f" Binaries: {result.so_count} .so files") + print() + + has_issues = False + + # Missing symbols + if result.missing_symbols: + has_issues = True + libc_missing = {} + other_missing = {} + for sym, bins in sorted(result.missing_symbols.items()): + if _looks_like_libc_symbol(sym): + libc_missing[sym] = bins + else: + other_missing[sym] = bins + + if libc_missing: + print(f" MISSING FROM MUSL ({len(libc_missing)} libc symbols):") + print(f" These symbols are imported by vendor .so but musl does not export them.") + print(f" Binaries using them will fail to load.") + print() + for sym, bins in sorted(libc_missing.items()): + print(f" {sym}") + if not quiet: + for b in sorted(set(bins)): + print(f" <- {b}") + print() + + if other_missing and not quiet: + print(f" UNRESOLVED ({len(other_missing)} non-libc symbols):") + print(f" Likely resolved by other vendor .so at runtime.") + print() + for sym, bins in sorted(other_missing.items()): + print(f" {sym}") + print() + + # Struct mismatches + if result.struct_mismatches: + has_issues = True + print(f" STRUCT LAYOUT MISMATCHES ({result.source_libc} vs musl):") + print(f" Vendor binaries were built against {result.source_libc}.") + print(f" Fields at different offsets/sizes will cause silent corruption.") + print() + for sm in result.struct_mismatches: + print(f" {sm['struct']} (used by: {', '.join(sm['functions'])})") + for key, vendor_val, musl_val in sm["mismatches"]: + print(f" {key}: {result.source_libc}={vendor_val} musl={musl_val}") + if not quiet: + print(f" affected binaries:") + for b in sorted(set(sm["binaries"])): + print(f" {b}") + print() + + # Version requirements + ver_issues = defaultdict(list) + for binfo in result.binaries: + for vr in binfo.version_reqs: + if vr.startswith("GLIBC_"): + ver_issues[vr].append(binfo.rel_path) + if ver_issues: + has_issues = True + print(f" GLIBC VERSION REQUIREMENTS:") + print(f" musl does not implement glibc symbol versioning.") + print() + for vr, bins in sorted(ver_issues.items()): + print(f" {vr} ({len(bins)} binaries)") + print() + + if not has_issues: + print(" No issues found.") + print() + + print("-" * 71) + print() + + +def _looks_like_libc_symbol(sym: str) -> bool: + """Heuristic: does this symbol look like it should come from libc?""" + # Starts with __ (libc internals) + if sym.startswith("__"): + return True + # Starts with _ followed by lowercase (libc private) + if sym.startswith("_") and len(sym) > 1 and sym[1].islower(): + return True + # Known libc function patterns + libc_patterns = ( + "memcpy_s", "memmove_s", "memset_s", "snprintf_s", + "strncpy_s", "strncat_s", "sprintf_s", "strcpy_s", + ) + if sym in libc_patterns: + return True + return False + + +# --------------------------------------------------------------------------- +# Toolchain resolution: package -> defconfig -> toolchain download +# --------------------------------------------------------------------------- + +TOOLCHAIN_CACHE_DIR = Path.home() / ".cache" / "openipc-audit-toolchains" +TOOLCHAIN_URL_BASE = "https://github.com/openipc/firmware/releases/download/toolchain" + + +@dataclass +class ToolchainInfo: + """Resolved toolchain for a specific SoC family.""" + soc_vendor: str + soc_family: str + prefix: str # e.g. "arm-openipc-linux-musleabi" + libc_type: str # "musl", "glibc", "uclibc" + cache_dir: Path # where the extracted toolchain lives + + @property + def libc_so(self) -> str: + """Path to this toolchain's libc.so.""" + sysroot = self.cache_dir / self.prefix / "sysroot" / "lib" / "libc.so" + if sysroot.exists(): + return str(sysroot) + # Try alternate layout + sdk = self.cache_dir / f"{self.prefix}_sdk-buildroot" + alt = sdk / self.prefix / "sysroot" / "lib" / "libc.so" + if alt.exists(): + return str(alt) + return "" + + @property + def cc(self) -> str: + """Path to this toolchain's gcc.""" + for base in [self.cache_dir, self.cache_dir / f"{self.prefix}_sdk-buildroot"]: + cc = base / "bin" / f"{self.prefix}-gcc" + if cc.exists(): + return str(cc) + return "" + + +def package_to_soc(pkg_name: str) -> tuple[str, str] | None: + """Extract (vendor, soc_family) from osdrv package name. + e.g. 'hisilicon-osdrv-hi3516ev200' -> ('hisilicon', 'hi3516ev200') + """ + # Pattern: -osdrv- + m = re.match(r"^(.+?)-osdrv-(.+)$", pkg_name.split("/")[-1]) + if not m: + return None + return m.group(1), m.group(2) + + +def find_defconfig(repo_root: Path, soc_vendor: str, soc_family: str) -> Path | None: + """Find the _lite_defconfig for a given SoC family.""" + import glob + patterns = [ + str(repo_root / f"br-ext-chip-{soc_vendor}/configs/{soc_family}_lite_defconfig"), + str(repo_root / f"br-ext-chip-{soc_vendor}/configs/{soc_family}_*_defconfig"), + ] + for pat in patterns: + matches = glob.glob(pat) + if matches: + return Path(matches[0]) + return None + + +def parse_defconfig_toolchain(defconfig: Path) -> tuple[str, str]: + """Extract (prefix, libc_type) from a defconfig file.""" + prefix = "" + libc_type = "" + with open(defconfig) as f: + for line in f: + line = line.strip() + if line.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_PREFIX="): + prefix = line.split("=", 1)[1].strip('"') + elif "CUSTOM_MUSL=y" in line: + libc_type = "musl" + elif "CUSTOM_GLIBC=y" in line or "USES_GLIBC=y" in line: + libc_type = "glibc" + elif "CUSTOM_UCLIBC=y" in line: + libc_type = "uclibc" + return prefix, libc_type + + +def download_toolchain(soc_vendor: str, soc_family: str, prefix: str) -> Path | None: + """Download and extract a toolchain, return the extracted directory.""" + TOOLCHAIN_CACHE_DIR.mkdir(parents=True, exist_ok=True) + + # toolchain.hisilicon-hi3516ev200.tgz + tarball_name = f"toolchain.{soc_vendor}-{soc_family}.tgz" + tarball_path = TOOLCHAIN_CACHE_DIR / tarball_name + extract_dir = TOOLCHAIN_CACHE_DIR / f"{soc_vendor}-{soc_family}" + + # Already extracted? + if extract_dir.is_dir(): + return extract_dir + + # Already downloaded? + if not tarball_path.exists(): + url = f"{TOOLCHAIN_URL_BASE}/{tarball_name}" + print(f" Downloading toolchain: {url}") + try: + r = subprocess.run( + ["wget", "-q", "-O", str(tarball_path), url], + timeout=300, + ) + if r.returncode != 0: + tarball_path.unlink(missing_ok=True) + print(f" ERROR: download failed", file=sys.stderr) + return None + except (subprocess.TimeoutExpired, FileNotFoundError): + tarball_path.unlink(missing_ok=True) + print(f" ERROR: download failed (timeout or wget not found)", file=sys.stderr) + return None + + # Extract + print(f" Extracting toolchain to {extract_dir}") + extract_dir.mkdir(parents=True, exist_ok=True) + r = subprocess.run( + ["tar", "xzf", str(tarball_path), "-C", str(extract_dir), "--strip-components=1"], + capture_output=True, timeout=120, + ) + if r.returncode != 0: + print(f" ERROR: extraction failed: {r.stderr.decode()}", file=sys.stderr) + return None + + return extract_dir + + +def resolve_toolchain(repo_root: Path, pkg_name: str) -> ToolchainInfo | None: + """Resolve the correct toolchain for a given osdrv package.""" + soc = package_to_soc(pkg_name) + if not soc: + return None + soc_vendor, soc_family = soc + + # Find and parse defconfig + defconfig = find_defconfig(repo_root, soc_vendor, soc_family) + if not defconfig: + print(f" WARNING: no defconfig found for {soc_vendor}-{soc_family}", file=sys.stderr) + return None + + prefix, libc_type = parse_defconfig_toolchain(defconfig) + if not prefix: + print(f" WARNING: no toolchain prefix in {defconfig.name}", file=sys.stderr) + return None + + # Check if already available in output/ + output_tc = repo_root / "output" / "host" / "opt" / "ext-toolchain" + if output_tc.is_dir(): + # Verify it matches this package's SoC + config_file = repo_root / "output" / ".config" + if config_file.exists(): + config_text = config_file.read_text() + if f'BR2_OPENIPC_SOC_FAMILY="{soc_family}"' in config_text: + return ToolchainInfo( + soc_vendor=soc_vendor, soc_family=soc_family, + prefix=prefix, libc_type=libc_type, + cache_dir=output_tc, + ) + + # Download from releases + cache_dir = download_toolchain(soc_vendor, soc_family, prefix) + if not cache_dir: + return None + + return ToolchainInfo( + soc_vendor=soc_vendor, soc_family=soc_family, + prefix=prefix, libc_type=libc_type, + cache_dir=cache_dir, + ) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description="Audit vendor .so binaries for ABI incompatibilities with musl.", + ) + parser.add_argument("--package", help="Audit only the named package") + parser.add_argument("--symbols-only", action="store_true", + help="Only check symbol availability, skip struct probe") + parser.add_argument("--quiet", action="store_true", + help="Less detail in output") + args = parser.parse_args() + + script_dir = Path(__file__).resolve().parent + repo_root = script_dir.parent.parent + pkg_dir = repo_root / "general" / "package" + + if not pkg_dir.is_dir(): + print(f"ERROR: Package directory not found: {pkg_dir}", file=sys.stderr) + sys.exit(1) + + # Build package list + if args.package: + p = pkg_dir / args.package + if not p.is_dir(): + print(f"ERROR: Package not found: {args.package}", file=sys.stderr) + sys.exit(1) + packages = [args.package] + else: + packages = sorted( + d.name for d in pkg_dir.iterdir() + if d.is_dir() and "-osdrv-" in d.name + ) + legacy_dir = pkg_dir / "legacy" + if legacy_dir.is_dir(): + for d in sorted(legacy_dir.iterdir()): + if d.is_dir() and "-osdrv-" in d.name: + packages.append(f"legacy/{d.name}") + + # Header + from datetime import date + print("=" * 71) + print(" OpenIPC Vendor Binary ABI Compatibility Audit") + print(f" Date: {date.today()}") + print("=" * 71) + print() + + total_missing = 0 + total_struct_issues = 0 + total_binaries = 0 + total_packages = 0 + + # Cache: toolchain key -> (musl_exports, musl_sizes) + # Multiple packages may share the same toolchain (same SoC family) + tc_cache: dict[str, tuple[set[str], dict[str, str] | None]] = {} + + for pkg_name in packages: + # Resolve per-package toolchain + tc = resolve_toolchain(repo_root, pkg_name) + if not tc: + print(f"Skipping {pkg_name}: no toolchain found") + print("-" * 71) + print() + continue + + tc_key = f"{tc.soc_vendor}-{tc.soc_family}" + if tc_key not in tc_cache: + libc_path = tc.libc_so + if not libc_path: + print(f"Skipping {pkg_name}: libc.so not found in toolchain") + print("-" * 71) + print() + continue + + print(f"Toolchain: {tc_key} ({tc.libc_type}, prefix={tc.prefix})") + print(f" libc.so: {libc_path}") + musl_exports = get_musl_exports(libc_path) + print(f" {len(musl_exports)} exported symbols") + + musl_sizes = None + if not args.symbols_only: + cc = tc.cc + if cc: + musl_sizes = compile_and_run_probe(cc) + if musl_sizes: + print(f" Probed {len(musl_sizes)} struct size/offset values") + else: + print(" WARNING: struct probe failed, skipping struct checks") + else: + print(" WARNING: gcc not found in toolchain, skipping struct probe") + print() + + tc_cache[tc_key] = (musl_exports, musl_sizes) + + musl_exports, musl_sizes = tc_cache[tc_key] + + result = audit_package( + pkg_name, str(pkg_dir / pkg_name), musl_exports, musl_sizes, + ) + if not result: + continue + + total_packages += 1 + total_binaries += result.so_count + total_missing += len(result.missing_symbols) + total_struct_issues += len(result.struct_mismatches) + + print_report(result, args.quiet) + + # Summary + print("=" * 71) + print(" OVERALL SUMMARY") + print("=" * 71) + print(f" Packages scanned: {total_packages}") + print(f" Binaries scanned: {total_binaries}") + print(f" Missing symbols: {total_missing}") + print(f" Struct mismatches: {total_struct_issues}") + print("=" * 71) + + if total_missing > 0: + sys.exit(1) + elif total_struct_issues > 0: + sys.exit(2) + else: + sys.exit(0) + + +if __name__ == "__main__": + main()