Skip to content

Move download-ci-llvm out of bootstrap.py #95170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 27, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock
Original file line number Diff line number Diff line change
@@ -225,8 +225,10 @@ dependencies = [
"pretty_assertions",
"serde",
"serde_json",
"tar",
"toml",
"winapi",
"xz2",
]

[[package]]
2 changes: 2 additions & 0 deletions src/bootstrap/Cargo.toml
Original file line number Diff line number Diff line change
@@ -42,10 +42,12 @@ cc = "1.0.69"
libc = "0.2"
serde = { version = "1.0.8", features = ["derive"] }
serde_json = "1.0.2"
tar = "0.4"
toml = "0.5"
ignore = "0.4.10"
opener = "0.5"
once_cell = "1.7.2"
xz2 = "0.1"

[target.'cfg(windows)'.dependencies.winapi]
version = "0.3"
149 changes: 0 additions & 149 deletions src/bootstrap/bootstrap.py
Original file line number Diff line number Diff line change
@@ -500,81 +500,6 @@ def download_toolchain(self, stage0=True, rustc_channel=None):
with output(self.rustfmt_stamp()) as rustfmt_stamp:
rustfmt_stamp.write(self.stage0_rustfmt.channel())

# Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
if self.downloading_llvm() and stage0:
# We want the most recent LLVM submodule update to avoid downloading
# LLVM more often than necessary.
#
# This git command finds that commit SHA, looking for bors-authored
# commits that modified src/llvm-project or other relevant version
# stamp files.
#
# This works even in a repository that has not yet initialized
# submodules.
top_level = subprocess.check_output([
"git", "rev-parse", "--show-toplevel",
]).decode(sys.getdefaultencoding()).strip()
llvm_sha = subprocess.check_output([
"git", "rev-list", "[email protected]", "-n1",
"--first-parent", "HEAD",
"--",
"{}/src/llvm-project".format(top_level),
"{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
# the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
"{}/src/version".format(top_level)
]).decode(sys.getdefaultencoding()).strip()
llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
llvm_root = self.llvm_root()
llvm_lib = os.path.join(llvm_root, "lib")
if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
self._download_ci_llvm(llvm_sha, llvm_assertions)
for binary in ["llvm-config", "FileCheck"]:
self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
for lib in os.listdir(llvm_lib):
if lib.endswith(".so"):
self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
with output(self.llvm_stamp()) as llvm_stamp:
llvm_stamp.write(llvm_sha + str(llvm_assertions))

def downloading_llvm(self):
opt = self.get_toml('download-ci-llvm', 'llvm')
# This is currently all tier 1 targets and tier 2 targets with host tools
# (since others may not have CI artifacts)
# https://doc.rust-lang.org/rustc/platform-support.html#tier-1
supported_platforms = [
# tier 1
"aarch64-unknown-linux-gnu",
"i686-pc-windows-gnu",
"i686-pc-windows-msvc",
"i686-unknown-linux-gnu",
"x86_64-unknown-linux-gnu",
"x86_64-apple-darwin",
"x86_64-pc-windows-gnu",
"x86_64-pc-windows-msvc",
# tier 2 with host tools
"aarch64-apple-darwin",
"aarch64-pc-windows-msvc",
"aarch64-unknown-linux-musl",
"arm-unknown-linux-gnueabi",
"arm-unknown-linux-gnueabihf",
"armv7-unknown-linux-gnueabihf",
"mips-unknown-linux-gnu",
"mips64-unknown-linux-gnuabi64",
"mips64el-unknown-linux-gnuabi64",
"mipsel-unknown-linux-gnu",
"powerpc-unknown-linux-gnu",
"powerpc64-unknown-linux-gnu",
"powerpc64le-unknown-linux-gnu",
"riscv64gc-unknown-linux-gnu",
"s390x-unknown-linux-gnu",
"x86_64-unknown-freebsd",
"x86_64-unknown-illumos",
"x86_64-unknown-linux-musl",
"x86_64-unknown-netbsd",
]
return opt == "true" \
or (opt == "if-available" and self.build in supported_platforms)

def _download_component_helper(
self, filename, pattern, tarball_suffix, stage0=True, key=None
):
@@ -606,53 +531,6 @@ def _download_component_helper(
)
unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)

def _download_ci_llvm(self, llvm_sha, llvm_assertions):
if not llvm_sha:
print("error: could not find commit hash for downloading LLVM")
print("help: maybe your repository history is too shallow?")
print("help: consider disabling `download-ci-llvm`")
print("help: or fetch enough history to include one upstream commit")
exit(1)
cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, cache_prefix)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)

base = "https://ci-artifacts.rust-lang.org"
url = "rustc-builds/{}".format(llvm_sha)
if llvm_assertions:
url = url.replace('rustc-builds', 'rustc-builds-alt')
# ci-artifacts are only stored as .xz, not .gz
if not support_xz():
print("error: XZ support is required to download LLVM")
print("help: consider disabling `download-ci-llvm` or using python3")
exit(1)
tarball_suffix = '.tar.xz'
filename = "rust-dev-nightly-" + self.build + tarball_suffix
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
help_on_error = "error: failed to download llvm from ci"
help_on_error += "\nhelp: old builds get deleted after a certain time"
help_on_error += "\nhelp: if trying to compile an old commit of rustc,"
help_on_error += " disable `download-ci-llvm` in config.toml:"
help_on_error += "\n"
help_on_error += "\n[llvm]"
help_on_error += "\ndownload-ci-llvm = false"
help_on_error += "\n"
get(
base,
"{}/{}".format(url, filename),
tarball,
self.checksums_sha256,
verbose=self.verbose,
do_verify=False,
help_on_error=help_on_error,
)
unpack(tarball, tarball_suffix, self.llvm_root(),
match="rust-dev",
verbose=self.verbose)

def fix_bin_or_dylib(self, fname):
"""Modifies the interpreter section of 'fname' to fix the dynamic linker,
or the RPATH section, to fix the dynamic library search path
@@ -816,17 +694,6 @@ def rustfmt_stamp(self):
"""
return os.path.join(self.bin_root(True), '.rustfmt-stamp')

def llvm_stamp(self):
"""Return the path for .llvm-stamp

>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
True
"""
return os.path.join(self.llvm_root(), '.llvm-stamp')


def program_out_of_date(self, stamp_path, key):
"""Check if the given program stamp is out of date"""
if not os.path.exists(stamp_path) or self.clean:
@@ -856,22 +723,6 @@ def bin_root(self, stage0):
subdir = "ci-rustc"
return os.path.join(self.build_dir, self.build, subdir)

def llvm_root(self):
"""Return the CI LLVM root directory

>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.llvm_root() == os.path.join("build", "ci-llvm")
True

When the 'build' property is given should be a nested directory:

>>> rb.build = "devel"
>>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
True
"""
return os.path.join(self.build_dir, self.build, "ci-llvm")

def get_toml(self, key, section=None):
"""Returns the value of the given key in config.toml, otherwise returns None

7 changes: 6 additions & 1 deletion src/bootstrap/builder.rs
Original file line number Diff line number Diff line change
@@ -12,7 +12,6 @@ use std::process::Command;
use std::time::{Duration, Instant};

use crate::cache::{Cache, Interned, INTERNER};
use crate::check;
use crate::compile;
use crate::config::{SplitDebuginfo, TargetSelection};
use crate::dist;
@@ -25,6 +24,7 @@ use crate::test;
use crate::tool::{self, SourceType};
use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t};
use crate::EXTRA_CHECK_CFGS;
use crate::{check, Config};
use crate::{Build, CLang, DocTests, GitRepo, Mode};

pub use crate::Compiler;
@@ -960,6 +960,11 @@ impl<'a> Builder<'a> {
None
}

/// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`.
pub(crate) fn llvm_link_shared(&self) -> bool {
Config::llvm_link_shared(self)
}

/// Prepares an invocation of `cargo` to be run.
///
/// This will create a `Command` that represents a pending execution of
2 changes: 1 addition & 1 deletion src/bootstrap/compile.rs
Original file line number Diff line number Diff line change
@@ -737,7 +737,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS
);
cargo.env("LLVM_STATIC_STDCPP", file);
}
if builder.config.llvm_link_shared {
if builder.llvm_link_shared() {
cargo.env("LLVM_LINK_SHARED", "1");
}
if builder.config.llvm_use_libcxx {
70 changes: 52 additions & 18 deletions src/bootstrap/config.rs
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@
//! This module implements parsing `config.toml` configuration files to tweak
//! how the build runs.

use std::cell::Cell;
use std::cmp;
use std::collections::{HashMap, HashSet};
use std::env;
@@ -11,7 +12,7 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::str::FromStr;

use crate::builder::TaskPath;
use crate::builder::{Builder, TaskPath};
use crate::cache::{Interned, INTERNER};
use crate::channel::GitInfo;
pub use crate::flags::Subcommand;
@@ -68,13 +69,14 @@ pub struct Config {
pub test_compare_mode: bool,
pub llvm_libunwind: LlvmLibunwind,
pub color: Color,
pub patch_binaries_for_nix: bool,

pub on_fail: Option<String>,
pub stage: u32,
pub keep_stage: Vec<u32>,
pub keep_stage_std: Vec<u32>,
pub src: PathBuf,
// defaults to `config.toml`
/// defaults to `config.toml`
pub config: PathBuf,
pub jobs: Option<u32>,
pub cmd: Subcommand,
@@ -95,7 +97,11 @@ pub struct Config {
pub llvm_release_debuginfo: bool,
pub llvm_version_check: bool,
pub llvm_static_stdcpp: bool,
pub llvm_link_shared: bool,
/// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm.
#[cfg(not(test))]
llvm_link_shared: Cell<Option<bool>>,
#[cfg(test)]
pub llvm_link_shared: Cell<Option<bool>>,
pub llvm_clang_cl: Option<String>,
pub llvm_targets: Option<String>,
pub llvm_experimental_targets: Option<String>,
@@ -856,6 +862,7 @@ impl Config {
set(&mut config.local_rebuild, build.local_rebuild);
set(&mut config.print_step_timings, build.print_step_timings);
set(&mut config.print_step_rusage, build.print_step_rusage);
set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix);

config.verbose = cmp::max(config.verbose, flags.verbose);

@@ -911,7 +918,9 @@ impl Config {
set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
set(&mut config.llvm_version_check, llvm.version_check);
set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
set(&mut config.llvm_link_shared, llvm.link_shared);
if let Some(v) = llvm.link_shared {
config.llvm_link_shared.set(Some(v));
}
config.llvm_targets = llvm.targets.clone();
config.llvm_experimental_targets = llvm.experimental_targets.clone();
config.llvm_link_jobs = llvm.link_jobs;
@@ -981,6 +990,7 @@ impl Config {
check_ci_llvm!(llvm.optimize);
check_ci_llvm!(llvm.thin_lto);
check_ci_llvm!(llvm.release_debuginfo);
// CI-built LLVM can be either dynamic or static. We won't know until we download it.
check_ci_llvm!(llvm.link_shared);
check_ci_llvm!(llvm.static_libstdcpp);
check_ci_llvm!(llvm.targets);
@@ -998,26 +1008,14 @@ impl Config {
check_ci_llvm!(llvm.clang);
check_ci_llvm!(llvm.build_config);
check_ci_llvm!(llvm.plugins);

// CI-built LLVM can be either dynamic or static.
let ci_llvm = config.out.join(&*config.build.triple).join("ci-llvm");
config.llvm_link_shared = if config.dry_run {
// just assume dynamic for now
true
} else {
let link_type = t!(
std::fs::read_to_string(ci_llvm.join("link-type.txt")),
format!("CI llvm missing: {}", ci_llvm.display())
);
link_type == "dynamic"
};
}

// NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above.
if config.llvm_thin_lto && llvm.link_shared.is_none() {
// If we're building with ThinLTO on, by default we want to link
// to LLVM shared, to avoid re-doing ThinLTO (which happens in
// the link step) with each stage.
config.llvm_link_shared = true;
config.llvm_link_shared.set(Some(true));
}
}

@@ -1272,6 +1270,42 @@ impl Config {
}
}

/// The absolute path to the downloaded LLVM artifacts.
pub(crate) fn ci_llvm_root(&self) -> PathBuf {
assert!(self.llvm_from_ci);
self.out.join(&*self.build.triple).join("ci-llvm")
}

/// Determine whether llvm should be linked dynamically.
///
/// If `false`, llvm should be linked statically.
/// This is computed on demand since LLVM might have to first be downloaded from CI.
pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool {
let mut opt = builder.config.llvm_link_shared.get();
if opt.is_none() && builder.config.dry_run {
// just assume static for now - dynamic linking isn't supported on all platforms
return false;
}

let llvm_link_shared = *opt.get_or_insert_with(|| {
if builder.config.llvm_from_ci {
crate::native::maybe_download_ci_llvm(builder);
let ci_llvm = builder.config.ci_llvm_root();
let link_type = t!(
std::fs::read_to_string(ci_llvm.join("link-type.txt")),
format!("CI llvm missing: {}", ci_llvm.display())
);
link_type == "dynamic"
} else {
// unclear how thought-through this default is, but it maintains compatibility with
// previous behavior
false
}
});
builder.config.llvm_link_shared.set(opt);
llvm_link_shared
}

pub fn verbose(&self) -> bool {
self.verbose > 0
}
8 changes: 4 additions & 4 deletions src/bootstrap/dist.rs
Original file line number Diff line number Diff line change
@@ -1904,7 +1904,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir
// clear why this is the case, though. llvm-config will emit the versioned
// paths and we don't want those in the sysroot (as we're expecting
// unversioned paths).
if target.contains("apple-darwin") && builder.config.llvm_link_shared {
if target.contains("apple-darwin") && builder.llvm_link_shared() {
let src_libdir = builder.llvm_out(target).join("lib");
let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
if llvm_dylib_path.exists() {
@@ -1939,7 +1939,7 @@ pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection,
// We do not need to copy LLVM files into the sysroot if it is not
// dynamically linked; it is already included into librustc_llvm
// statically.
if builder.config.llvm_link_shared {
if builder.llvm_link_shared() {
maybe_install_llvm(builder, target, &dst_libdir);
}
}
@@ -1951,7 +1951,7 @@ pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection
// We do not need to copy LLVM files into the sysroot if it is not
// dynamically linked; it is already included into librustc_llvm
// statically.
if builder.config.llvm_link_shared {
if builder.llvm_link_shared() {
maybe_install_llvm(builder, target, &dst_libdir);
}
}
@@ -2077,7 +2077,7 @@ impl Step for RustDev {
// compiler libraries.
let dst_libdir = tarball.image_dir().join("lib");
maybe_install_llvm(builder, target, &dst_libdir);
let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);

Some(tarball.generate())
10 changes: 10 additions & 0 deletions src/bootstrap/lib.rs
Original file line number Diff line number Diff line change
@@ -1391,6 +1391,16 @@ impl Build {
paths
}

/// Create a temporary directory in `out` and return its path.
///
/// NOTE: this temporary directory is shared between all steps;
/// if you need an empty directory, create a new subdirectory inside it.
fn tempdir(&self) -> PathBuf {
let tmp = self.out.join("tmp");
t!(fs::create_dir_all(&tmp));
tmp
}

/// Copies a file from `src` to `dst`
pub fn copy(&self, src: &Path, dst: &Path) {
if self.config.dry_run {
292 changes: 288 additions & 4 deletions src/bootstrap/native.rs
Original file line number Diff line number Diff line change
@@ -12,9 +12,12 @@ use std::env;
use std::env::consts::EXE_EXTENSION;
use std::ffi::{OsStr, OsString};
use std::fs::{self, File};
use std::io;
use std::io::{self, BufRead, BufReader, ErrorKind};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::process::{Command, Stdio};

use once_cell::sync::OnceCell;
use xz2::bufread::XzDecoder;

use crate::builder::{Builder, RunConfig, ShouldRun, Step};
use crate::config::TargetSelection;
@@ -62,6 +65,8 @@ pub fn prebuilt_llvm_config(
builder: &Builder<'_>,
target: TargetSelection,
) -> Result<PathBuf, Meta> {
maybe_download_ci_llvm(builder);

// If we're using a custom LLVM bail out here, but we can only use a
// custom LLVM for the build triple.
if let Some(config) = builder.config.target_config.get(&target) {
@@ -111,6 +116,285 @@ pub fn prebuilt_llvm_config(
Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() })
}

pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
let config = &builder.config;
if !config.llvm_from_ci {
return;
}
let mut rev_list = Command::new("git");
rev_list.args(&[
PathBuf::from("rev-list"),
"--author=bors@rust-lang.org".into(),
"-n1".into(),
"--first-parent".into(),
"HEAD".into(),
"--".into(),
builder.src.join("src/llvm-project"),
builder.src.join("src/bootstrap/download-ci-llvm-stamp"),
// the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
builder.src.join("src/version"),
]);
let llvm_sha = output(&mut rev_list);
let llvm_sha = llvm_sha.trim();

if llvm_sha == "" {
println!("error: could not find commit hash for downloading LLVM");
println!("help: maybe your repository history is too shallow?");
println!("help: consider disabling `download-ci-llvm`");
println!("help: or fetch enough history to include one upstream commit");
panic!();
}

let llvm_root = config.ci_llvm_root();
let llvm_stamp = llvm_root.join(".llvm-stamp");
let key = format!("{}{}", llvm_sha, config.llvm_assertions);
if program_out_of_date(&llvm_stamp, &key) && !config.dry_run {
download_ci_llvm(builder, &llvm_sha);
for binary in ["llvm-config", "FileCheck"] {
fix_bin_or_dylib(builder, &llvm_root.join("bin").join(binary));
}
let llvm_lib = llvm_root.join("lib");
for entry in t!(fs::read_dir(&llvm_lib)) {
let lib = t!(entry).path();
if lib.ends_with(".so") {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You cannot use this to check for an extension, only whole file names - the docs say "Only considers whole path components to match." - I have no idea how this ever passed NixOS, it just can't ever work AFAICT.

Copy link
Member

@eddyb eddyb May 14, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fix_bin_or_dylib(builder, &lib);
}
}
t!(fs::write(llvm_stamp, key));
}
}

fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
let llvm_assertions = builder.config.llvm_assertions;

let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions);
let cache_dst = builder.out.join("cache");
let rustc_cache = cache_dst.join(cache_prefix);
if !rustc_cache.exists() {
t!(fs::create_dir_all(&rustc_cache));
}
let base = "https://ci-artifacts.rust-lang.org";
let url = if llvm_assertions {
format!("rustc-builds-alt/{}", llvm_sha)
} else {
format!("rustc-builds/{}", llvm_sha)
};
let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
let tarball = rustc_cache.join(&filename);
if !tarball.exists() {
download_component(builder, base, &format!("{}/{}", url, filename), &tarball);
}
let llvm_root = builder.config.ci_llvm_root();
unpack(builder, &tarball, &llvm_root);
}

/// Modifies the interpreter section of 'fname' to fix the dynamic linker,
/// or the RPATH section, to fix the dynamic library search path
///
/// This is only required on NixOS and uses the PatchELF utility to
/// change the interpreter/RPATH of ELF executables.
///
/// Please see https://nixos.org/patchelf.html for more information
fn fix_bin_or_dylib(builder: &Builder<'_>, fname: &Path) {
// FIXME: cache NixOS detection?
match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
Err(_) => return,
Ok(output) if !output.status.success() => return,
Ok(output) => {
let mut s = output.stdout;
if s.last() == Some(&b'\n') {
s.pop();
}
if s != b"Linux" {
return;
}
}
}

// If the user has asked binaries to be patched for Nix, then
// don't check for NixOS or `/lib`, just continue to the patching.
// FIXME: shouldn't this take precedence over the `uname` check above?
if !builder.config.patch_binaries_for_nix {
// Use `/etc/os-release` instead of `/etc/NIXOS`.
// The latter one does not exist on NixOS when using tmpfs as root.
const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
let os_release = match File::open("/etc/os-release") {
Err(e) if e.kind() == ErrorKind::NotFound => return,
Err(e) => panic!("failed to access /etc/os-release: {}", e),
Ok(f) => f,
};
if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
return;
}
if Path::new("/lib").exists() {
return;
}
}

// At this point we're pretty sure the user is running NixOS or using Nix
println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());

// Only build `.nix-deps` once.
static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
let mut nix_build_succeeded = true;
let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
// Run `nix-build` to "build" each dependency (which will likely reuse
// the existing `/nix/store` copy, or at most download a pre-built copy).
//
// Importantly, we create a gc-root called `.nix-deps` in the `build/`
// directory, but still reference the actual `/nix/store` path in the rpath
// as it makes it significantly more robust against changes to the location of
// the `.nix-deps` location.
//
// bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
// zlib: Needed as a system dependency of `libLLVM-*.so`.
// patchelf: Needed for patching ELF binaries (see doc comment above).
let nix_deps_dir = builder.out.join(".nix-deps");
const NIX_EXPR: &str = "
with (import <nixpkgs> {});
symlinkJoin {
name = \"rust-stage0-dependencies\";
paths = [
zlib
patchelf
stdenv.cc.bintools
];
}
";
nix_build_succeeded = builder.try_run(Command::new("nix-build").args(&[
Path::new("-E"),
Path::new(NIX_EXPR),
Path::new("-o"),
&nix_deps_dir,
]));
nix_deps_dir
});
if !nix_build_succeeded {
return;
}

let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
let rpath_entries = {
// ORIGIN is a relative default, all binary and dynamic libraries we ship
// appear to have this (even when `../lib` is redundant).
// NOTE: there are only two paths here, delimited by a `:`
let mut entries = OsString::from("$ORIGIN/../lib:");
entries.push(t!(fs::canonicalize(nix_deps_dir)));
entries.push("/lib");
entries
};
patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
if !fname.ends_with(".so") {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same thing here, this cannot check for an extension (it's equivalent to fname.file_name() == ".so" basically).

// Finally, set the corret .interp for binaries
let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
// FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
}

builder.try_run(patchelf.arg(fname));
}

fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &Path) {
// Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
let tempfile = builder.tempdir().join(dest_path.file_name().unwrap());
// FIXME: support `do_verify` (only really needed for nightly rustfmt)
// FIXME: support non-utf8 paths?
download_with_retries(builder, tempfile.to_str().unwrap(), &format!("{}/{}", base, url));
t!(std::fs::rename(&tempfile, dest_path));
}

fn download_with_retries(builder: &Builder<'_>, tempfile: &str, url: &str) {
println!("downloading {}", url);

// FIXME: check if curl is installed instead of skipping straight to powershell
if builder.build.build.contains("windows-msvc") {
for _ in 0..3 {
if builder.try_run(Command::new("PowerShell.exe").args(&[
"/nologo",
"-Command",
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
&format!(
"(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
url, tempfile
),
])) {
return;
}
println!("\nspurious failure, trying again");
}
} else {
builder.run(Command::new("curl").args(&[
"-#",
"-y",
"30",
"-Y",
"10", // timeout if speed is < 10 bytes/sec for > 30 seconds
"--connect-timeout",
"30", // timeout if cannot connect within 30 seconds
"--retry",
"3",
"-Sf",
"-o",
tempfile,
url,
]));
}
}

fn unpack(builder: &Builder<'_>, tarball: &Path, dst: &Path) {
println!("extracting {} to {}", tarball.display(), dst.display());
if !dst.exists() {
t!(fs::create_dir_all(dst));
}

// FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild
const MATCH: &str = "rust-dev";

// `tarball` ends with `.tar.xz`; strip that suffix
// example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
let uncompressed_filename =
Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());

// decompress the file
let data = t!(File::open(tarball));
let decompressor = XzDecoder::new(BufReader::new(data));

let mut tar = tar::Archive::new(decompressor);
for member in t!(tar.entries()) {
let mut member = t!(member);
let original_path = t!(member.path()).into_owned();
// skip the top-level directory
if original_path == directory_prefix {
continue;
}
let mut short_path = t!(original_path.strip_prefix(directory_prefix));
if !short_path.starts_with(MATCH) {
continue;
}
short_path = t!(short_path.strip_prefix(MATCH));
let dst_path = dst.join(short_path);
builder.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
if !t!(member.unpack_in(dst)) {
panic!("path traversal attack ??");
}
let src_path = dst.join(original_path);
if src_path.is_dir() && dst_path.exists() {
continue;
}
t!(fs::rename(src_path, dst_path));
}
t!(fs::remove_dir_all(dst.join(directory_prefix)));
}

fn program_out_of_date(stamp: &Path, key: &str) -> bool {
if !stamp.exists() {
return true;
}
t!(fs::read_to_string(stamp)) != key
}

#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Llvm {
pub target: TargetSelection,
@@ -153,7 +437,7 @@ impl Step for Llvm {
};

builder.update_submodule(&Path::new("src").join("llvm-project"));
if builder.config.llvm_link_shared
if builder.llvm_link_shared()
&& (target.contains("windows") || target.contains("apple-darwin"))
{
panic!("shared linking to LLVM is not currently supported on {}", target.triple);
@@ -255,7 +539,7 @@ impl Step for Llvm {
//
// If we're not linking rustc to a dynamic LLVM, though, then don't link
// tools to it.
if builder.llvm_link_tools_dynamically(target) && builder.config.llvm_link_shared {
if builder.llvm_link_tools_dynamically(target) && builder.llvm_link_shared() {
cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
}

11 changes: 4 additions & 7 deletions src/bootstrap/test.rs
Original file line number Diff line number Diff line change
@@ -1577,9 +1577,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the
cmd.env("RUSTC_PROFILER_SUPPORT", "1");
}

let tmp = builder.out.join("tmp");
std::fs::create_dir_all(&tmp).unwrap();
cmd.env("RUST_TEST_TMPDIR", tmp);
cmd.env("RUST_TEST_TMPDIR", builder.tempdir());

cmd.arg("--adb-path").arg("adb");
cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR);
@@ -2259,14 +2257,13 @@ impl Step for RemoteCopyLibs {
builder.ensure(compile::Std { compiler, target });

builder.info(&format!("REMOTE copy libs to emulator ({})", target));
t!(fs::create_dir_all(builder.out.join("tmp")));

let server = builder.ensure(tool::RemoteTestServer { compiler, target });

// Spawn the emulator and wait for it to come online
let tool = builder.tool_exe(Tool::RemoteTestClient);
let mut cmd = Command::new(&tool);
cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.out.join("tmp"));
cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir());
if let Some(rootfs) = builder.qemu_rootfs(target) {
cmd.arg(rootfs);
}
@@ -2300,7 +2297,7 @@ impl Step for Distcheck {
/// Runs "distcheck", a 'make check' from a tarball
fn run(self, builder: &Builder<'_>) {
builder.info("Distcheck");
let dir = builder.out.join("tmp").join("distcheck");
let dir = builder.tempdir().join("distcheck");
let _ = fs::remove_dir_all(&dir);
t!(fs::create_dir_all(&dir));

@@ -2326,7 +2323,7 @@ impl Step for Distcheck {

// Now make sure that rust-src has all of libstd's dependencies
builder.info("Distcheck rust-src");
let dir = builder.out.join("tmp").join("distcheck-src");
let dir = builder.tempdir().join("distcheck-src");
let _ = fs::remove_dir_all(&dir);
t!(fs::create_dir_all(&dir));