Convert more stdlibs to LazyLibraries. Add tests for dep tree.

This converts more JLL stdlibs to use lazily-loaded libraries, and as a
useful side-effect, causes them to be loaded by absolute path, isolating
them all from `LD_LIBRARY_PATH`-like effects.

Adds tests that audits the expressed library dependencies in our
LazyLibrary JLL definitions, to ensure that we don't get out of sync
with the actual binaries.

Co-Authored-By: Mosè Giordano <[email protected]>
Co-Authored-By: Elliot Saba <[email protected]>

Author: 2 people

stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl

@@ -64,13 +64,18 @@ if @isdefined(_libatomic_path)
     const libatomic = LazyLibrary(_libatomic_path)
 end
 const libgcc_s = LazyLibrary(_libgcc_s_path)
-libgfortran_deps = [libgcc_s]
+
+_libgfortran_deps = [libgcc_s]
 if @isdefined _libquadmath_path
     const libquadmath = LazyLibrary(_libquadmath_path)
-    push!(libgfortran_deps, libquadmath)
+    push!(_libgfortran_deps, libquadmath)
 end
-const libgfortran = LazyLibrary(_libgfortran_path, dependencies=libgfortran_deps)
-const libstdcxx = LazyLibrary(_libstdcxx_path, dependencies=[libgcc_s])
+
+const libgfortran = LazyLibrary(_libgfortran_path, dependencies=_libgfortran_deps)
+
+_libstdcxx_dependencies = LazyLibrary[libgcc_s]
+const libstdcxx = LazyLibrary(_libstdcxx_path, dependencies=_libstdcxx_dependencies)
+
 const libgomp = LazyLibrary(_libgomp_path)
 
 # Some installations (such as those from-source) may not have `libssp`
@@ -116,4 +121,9 @@ function __init__()
     push!(LIBPATH_list, LIBPATH[])
 end
 
+if Base.generating_output()
+    precompile(eager_mode, ())
+    precompile(is_available, ())
+end
+
 end  # module CompilerSupportLibraries_jll

stdlib/GMP_jll/Project.toml

@@ -4,6 +4,7 @@ version = "6.3.0+2"
 
 [deps]
 Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
 [compat]

stdlib/GMP_jll/src/GMP_jll.jl

@@ -2,51 +2,62 @@
 
 ## dummy stub for https://github.com/JuliaBinaryWrappers/GMP_jll.jl
 baremodule GMP_jll
-using Base, Libdl
-
-const PATH_list = String[]
-const LIBPATH_list = String[]
+using Base, Libdl, CompilerSupportLibraries_jll
 
 export libgmp, libgmpxx
 
 # These get calculated in __init__()
 const PATH = Ref("")
+const PATH_list = String[]
 const LIBPATH = Ref("")
+const LIBPATH_list = String[]
 artifact_dir::String = ""
-libgmp_handle::Ptr{Cvoid} = C_NULL
 libgmp_path::String = ""
-libgmpxx_handle::Ptr{Cvoid} = C_NULL
 libgmpxx_path::String = ""
 
 if Sys.iswindows()
-    const libgmp = "libgmp-10.dll"
-    const libgmpxx = "libgmpxx-4.dll"
+    const _libgmp_path = BundledLazyLibraryPath("libgmp-10.dll")
+    const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx-4.dll")
+elseif Sys.isapple()
+    const _libgmp_path = BundledLazyLibraryPath("libgmp.10.dylib")
+    const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx.4.dylib")
+else
+    const _libgmp_path = BundledLazyLibraryPath("libgmp.so.10")
+    const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx.so.4")
+end
+
+const libgmp = LazyLibrary(_libgmp_path)
+
+if Sys.isfreebsd()
+    _libgmpxx_dependencies = LazyLibrary[libgmp, libgcc_s]
 elseif Sys.isapple()
-    const libgmp = "@rpath/libgmp.10.dylib"
-    const libgmpxx = "@rpath/libgmpxx.4.dylib"
+    _libgmpxx_dependencies = LazyLibrary[libgmp]
 else
-    const libgmp = "libgmp.so.10"
-    const libgmpxx = "libgmpxx.so.4"
+    _libgmpxx_dependencies = LazyLibrary[libgmp, libstdcxx, libgcc_s]
+end
+const libgmpxx = LazyLibrary(
+    _libgmpxx_path,
+    dependencies=_libgmpxx_dependencies,
+)
+
+function eager_mode()
+    CompilerSupportLibraries_jll.eager_mode()
+    dlopen(libgmp)
+    dlopen(libgmpxx)
 end
+is_available() = true
 
 function __init__()
-    global libgmp_handle = dlopen(libgmp)
-    global libgmp_path = dlpath(libgmp_handle)
-    global libgmpxx_handle = dlopen(libgmpxx)
-    global libgmpxx_path = dlpath(libgmpxx_handle)
+    global libgmp_path = string(_libgmp_path)
+    global libgmpxx_path = string(_libgmpxx_path)
     global artifact_dir = dirname(Sys.BINDIR)
     LIBPATH[] = dirname(libgmp_path)
     push!(LIBPATH_list, LIBPATH[])
 end
 
-# JLLWrappers API compatibility shims.  Note that not all of these will really make sense.
-# For instance, `find_artifact_dir()` won't actually be the artifact directory, because
-# there isn't one.  It instead returns the overall Julia prefix.
-is_available() = true
-find_artifact_dir() = artifact_dir
-dev_jll() = error("stdlib JLLs cannot be dev'ed")
-best_wrapper = nothing
-get_libgmp_path() = libgmp_path
-get_libgmpxx_path() = libgmpxx_path
+if Base.generating_output()
+    precompile(eager_mode, ())
+    precompile(is_available, ())
+end
 
 end  # module GMP_jll

stdlib/GMP_jll/test/runtests.jl

@@ -3,6 +3,6 @@
 using Test, Libdl, GMP_jll
 
 @testset "GMP_jll" begin
-    vn = VersionNumber(unsafe_string(unsafe_load(cglobal((:__gmp_version, libgmp), Ptr{Cchar}))))
+    vn = VersionNumber(unsafe_string(unsafe_load(cglobal(dlsym(libgmp, :__gmp_version), Ptr{Cchar}))))
     @test vn == v"6.3.0"
 end

stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl

@@ -5,38 +5,33 @@
 baremodule LLVMLibUnwind_jll
 using Base, Libdl
 
-const PATH_list = String[]
-const LIBPATH_list = String[]
-
 export llvmlibunwind
 
 # These get calculated in __init__()
 const PATH = Ref("")
+const PATH_list = String[]
 const LIBPATH = Ref("")
+const LIBPATH_list = String[]
 artifact_dir::String = ""
-llvmlibunwind_handle::Ptr{Cvoid} = C_NULL
 llvmlibunwind_path::String = ""
 
-const llvmlibunwind = "libunwind"
+const _llvmlibunwind_path = BundledLazyLibraryPath("libunwind")
+const llvmlibunwind = LazyLibrary(_llvmlibunwind_path)
+function eager_mode()
+    dlopen(llvmlibunwind)
+end
+is_available() = @static Sys.isapple() ? true : false
 
 function __init__()
-    # We only dlopen something on MacOS
-    @static if Sys.isapple()
-        global llvmlibunwind_handle = dlopen(llvmlibunwind)
-        global llvmlibunwind_path = dlpath(llvmlibunwind_handle)
-        global artifact_dir = dirname(Sys.BINDIR)
-        LIBPATH[] = dirname(llvmlibunwind_path)
-        push!(LIBPATH_list, LIBPATH[])
-    end
+    global llvmlibunwind_path = string(_llvmlibunwind_path)
+    global artifact_dir = dirname(Sys.BINDIR)
+    LIBPATH[] = dirname(llvmlibunwind_path)
+    push!(LIBPATH_list, LIBPATH[])
 end
 
-# JLLWrappers API compatibility shims.  Note that not all of these will really make sense.
-# For instance, `find_artifact_dir()` won't actually be the artifact directory, because
-# there isn't one.  It instead returns the overall Julia prefix.
-is_available() = @static Sys.isapple() ? true : false
-find_artifact_dir() = artifact_dir
-dev_jll() = error("stdlib JLLs cannot be dev'ed")
-best_wrapper = nothing
-get_llvmlibunwind_path() = llvmlibunwind_path
+if Base.generating_output()
+    precompile(eager_mode, ())
+    precompile(is_available, ())
+end
 
 end  # module LLVMLibUnwind_jll

stdlib/LLVMLibUnwind_jll/test/runtests.jl

@@ -1,16 +1,14 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-using Test, Libdl
-using LLVMLibUnwind_jll: llvmlibunwind_handle
-
+using Test, Libdl, LLVMLibUnwind_jll
 @testset "LLVMLibUnwind_jll" begin
     if Sys.isapple()
-        @test dlsym(llvmlibunwind_handle, :unw_getcontext; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_init_local; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_init_local_dwarf; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_step; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_get_reg; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_set_reg; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_resume; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_getcontext; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_init_local; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_init_local_dwarf; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_step; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_get_reg; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_set_reg; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_resume; throw_error=false) !== nothing
     end
 end

stdlib/LibCURL_jll/src/LibCURL_jll.jl

@@ -9,41 +9,52 @@ if !(Sys.iswindows() || Sys.isapple())
     using OpenSSL_jll
 end
 
-const PATH_list = String[]
-const LIBPATH_list = String[]
-
 export libcurl
 
 # These get calculated in __init__()
 const PATH = Ref("")
+const PATH_list = String[]
 const LIBPATH = Ref("")
+const LIBPATH_list = String[]
 artifact_dir::String = ""
-libcurl_handle::Ptr{Cvoid} = C_NULL
 libcurl_path::String = ""
 
+_libcurl_dependencies = LazyLibrary[libz, libnghttp2, libssh2]
+if !(Sys.iswindows() || Sys.isapple())
+    append!(_libcurl_dependencies, [libssl, libcrypto])
+end
+
 if Sys.iswindows()
-    const libcurl = "libcurl-4.dll"
+    const _libcurl_path = BundledLazyLibraryPath("libcurl-4.dll")
 elseif Sys.isapple()
-    const libcurl = "@rpath/libcurl.4.dylib"
+    const _libcurl_path = BundledLazyLibraryPath("libcurl.4.dylib")
 else
-    const libcurl = "libcurl.so.4"
+    const _libcurl_path = BundledLazyLibraryPath("libcurl.so.4")
 end
 
+const libcurl = LazyLibrary(
+    _libcurl_path,
+    dependencies=_libcurl_dependencies,
+)
+
+function eager_mode()
+    Zlib_jll.eager_mode()
+    nghttp2_jll.eager_mode()
+    LibSSH2_jll.eager_mode()
+    dlopen(libcurl)
+end
+is_available() = true
+
 function __init__()
-    global libcurl_handle = dlopen(libcurl)
-    global libcurl_path = dlpath(libcurl_handle)
+    global libcurl_path = string(_libcurl_path)
     global artifact_dir = dirname(Sys.BINDIR)
     LIBPATH[] = dirname(libcurl_path)
     push!(LIBPATH_list, LIBPATH[])
 end
 
-# JLLWrappers API compatibility shims.  Note that not all of these will really make sense.
-# For instance, `find_artifact_dir()` won't actually be the artifact directory, because
-# there isn't one.  It instead returns the overall Julia prefix.
-is_available() = true
-find_artifact_dir() = artifact_dir
-dev_jll() = error("stdlib JLLs cannot be dev'ed")
-best_wrapper = nothing
-get_libcurl_path() = libcurl_path
+if Base.generating_output()
+    precompile(eager_mode, ())
+    precompile(is_available, ())
+end
 
 end  # module LibCURL_jll

stdlib/LibGit2_jll/src/LibGit2_jll.jl

@@ -9,41 +9,50 @@ if !(Sys.iswindows() || Sys.isapple())
     using OpenSSL_jll
 end
 
-const PATH_list = String[]
-const LIBPATH_list = String[]
-
 export libgit2
 
 # These get calculated in __init__()
 const PATH = Ref("")
+const PATH_list = String[]
 const LIBPATH = Ref("")
+const LIBPATH_list = String[]
 artifact_dir::String = ""
-libgit2_handle::Ptr{Cvoid} = C_NULL
 libgit2_path::String = ""
 
 if Sys.iswindows()
-    const libgit2 = "libgit2.dll"
+    const _libgit2_path = BundledLazyLibraryPath("libgit2.dll")
 elseif Sys.isapple()
-    const libgit2 = "@rpath/libgit2.1.9.dylib"
+    const _libgit2_path = BundledLazyLibraryPath("libgit2.1.9.dylib")
+else
+    const _libgit2_path = BundledLazyLibraryPath("libgit2.so.1.9")
+end
+
+if Sys.isfreebsd()
+    _libgit2_dependencies = LazyLibrary[libssh2, libssl, libcrypto]
 else
-    const libgit2 = "libgit2.so.1.9"
+    _libgit2_dependencies = LazyLibrary[libssh2]
+end
+const libgit2 = LazyLibrary(_libgit2_path, dependencies=_libgit2_dependencies)
+
+function eager_mode()
+    LibSSH2_jll.eager_mode()
+    @static if !(Sys.iswindows() || Sys.isapple())
+        OpenSSL_jll.eager_mode()
+    end
+    dlopen(libgit2)
 end
+is_available() = true
 
 function __init__()
-    global libgit2_handle = dlopen(libgit2)
-    global libgit2_path = dlpath(libgit2_handle)
+    global libgit2_path = string(_libgit2_path)
     global artifact_dir = dirname(Sys.BINDIR)
     LIBPATH[] = dirname(libgit2_path)
     push!(LIBPATH_list, LIBPATH[])
 end
 
-# JLLWrappers API compatibility shims.  Note that not all of these will really make sense.
-# For instance, `find_artifact_dir()` won't actually be the artifact directory, because
-# there isn't one.  It instead returns the overall Julia prefix.
-is_available() = true
-find_artifact_dir() = artifact_dir
-dev_jll() = error("stdlib JLLs cannot be dev'ed")
-best_wrapper = nothing
-get_libgit2_path() = libgit2_path
+if Base.generating_output()
+    precompile(eager_mode, ())
+    precompile(is_available, ())
+end
 
 end  # module LibGit2_jll

stdlib/LibSSH2_jll/Project.toml

@@ -6,6 +6,7 @@ version = "1.11.3+1"
 OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a"
 
 [compat]
 julia = "1.8"