-
Notifications
You must be signed in to change notification settings - Fork 13.6k
Move download-ci-llvm
out of bootstrap.py
#95170
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -500,81 +500,6 @@ def download_toolchain(self, stage0=True, rustc_channel=None): | |
with output(self.rustfmt_stamp()) as rustfmt_stamp: | ||
rustfmt_stamp.write(self.stage0_rustfmt.channel()) | ||
|
||
# Avoid downloading LLVM twice (once for stage0 and once for the master rustc) | ||
if self.downloading_llvm() and stage0: | ||
# We want the most recent LLVM submodule update to avoid downloading | ||
# LLVM more often than necessary. | ||
# | ||
# This git command finds that commit SHA, looking for bors-authored | ||
# commits that modified src/llvm-project or other relevant version | ||
# stamp files. | ||
# | ||
# This works even in a repository that has not yet initialized | ||
# submodules. | ||
top_level = subprocess.check_output([ | ||
"git", "rev-parse", "--show-toplevel", | ||
]).decode(sys.getdefaultencoding()).strip() | ||
llvm_sha = subprocess.check_output([ | ||
"git", "rev-list", "[email protected]", "-n1", | ||
"--first-parent", "HEAD", | ||
"--", | ||
"{}/src/llvm-project".format(top_level), | ||
"{}/src/bootstrap/download-ci-llvm-stamp".format(top_level), | ||
# the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` | ||
"{}/src/version".format(top_level) | ||
]).decode(sys.getdefaultencoding()).strip() | ||
llvm_assertions = self.get_toml('assertions', 'llvm') == 'true' | ||
llvm_root = self.llvm_root() | ||
llvm_lib = os.path.join(llvm_root, "lib") | ||
if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)): | ||
self._download_ci_llvm(llvm_sha, llvm_assertions) | ||
for binary in ["llvm-config", "FileCheck"]: | ||
self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary)) | ||
for lib in os.listdir(llvm_lib): | ||
if lib.endswith(".so"): | ||
self.fix_bin_or_dylib(os.path.join(llvm_lib, lib)) | ||
with output(self.llvm_stamp()) as llvm_stamp: | ||
llvm_stamp.write(llvm_sha + str(llvm_assertions)) | ||
|
||
def downloading_llvm(self): | ||
opt = self.get_toml('download-ci-llvm', 'llvm') | ||
# This is currently all tier 1 targets and tier 2 targets with host tools | ||
# (since others may not have CI artifacts) | ||
# https://doc.rust-lang.org/rustc/platform-support.html#tier-1 | ||
supported_platforms = [ | ||
# tier 1 | ||
"aarch64-unknown-linux-gnu", | ||
"i686-pc-windows-gnu", | ||
"i686-pc-windows-msvc", | ||
"i686-unknown-linux-gnu", | ||
"x86_64-unknown-linux-gnu", | ||
"x86_64-apple-darwin", | ||
"x86_64-pc-windows-gnu", | ||
"x86_64-pc-windows-msvc", | ||
# tier 2 with host tools | ||
"aarch64-apple-darwin", | ||
"aarch64-pc-windows-msvc", | ||
"aarch64-unknown-linux-musl", | ||
"arm-unknown-linux-gnueabi", | ||
"arm-unknown-linux-gnueabihf", | ||
"armv7-unknown-linux-gnueabihf", | ||
"mips-unknown-linux-gnu", | ||
"mips64-unknown-linux-gnuabi64", | ||
"mips64el-unknown-linux-gnuabi64", | ||
"mipsel-unknown-linux-gnu", | ||
"powerpc-unknown-linux-gnu", | ||
"powerpc64-unknown-linux-gnu", | ||
"powerpc64le-unknown-linux-gnu", | ||
"riscv64gc-unknown-linux-gnu", | ||
"s390x-unknown-linux-gnu", | ||
"x86_64-unknown-freebsd", | ||
"x86_64-unknown-illumos", | ||
"x86_64-unknown-linux-musl", | ||
"x86_64-unknown-netbsd", | ||
] | ||
return opt == "true" \ | ||
or (opt == "if-available" and self.build in supported_platforms) | ||
|
||
def _download_component_helper( | ||
self, filename, pattern, tarball_suffix, stage0=True, key=None | ||
): | ||
|
@@ -606,53 +531,6 @@ def _download_component_helper( | |
) | ||
unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose) | ||
|
||
def _download_ci_llvm(self, llvm_sha, llvm_assertions): | ||
if not llvm_sha: | ||
print("error: could not find commit hash for downloading LLVM") | ||
print("help: maybe your repository history is too shallow?") | ||
print("help: consider disabling `download-ci-llvm`") | ||
print("help: or fetch enough history to include one upstream commit") | ||
exit(1) | ||
cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions) | ||
cache_dst = os.path.join(self.build_dir, "cache") | ||
rustc_cache = os.path.join(cache_dst, cache_prefix) | ||
if not os.path.exists(rustc_cache): | ||
os.makedirs(rustc_cache) | ||
|
||
base = "https://ci-artifacts.rust-lang.org" | ||
url = "rustc-builds/{}".format(llvm_sha) | ||
if llvm_assertions: | ||
url = url.replace('rustc-builds', 'rustc-builds-alt') | ||
# ci-artifacts are only stored as .xz, not .gz | ||
if not support_xz(): | ||
print("error: XZ support is required to download LLVM") | ||
print("help: consider disabling `download-ci-llvm` or using python3") | ||
exit(1) | ||
tarball_suffix = '.tar.xz' | ||
filename = "rust-dev-nightly-" + self.build + tarball_suffix | ||
tarball = os.path.join(rustc_cache, filename) | ||
if not os.path.exists(tarball): | ||
help_on_error = "error: failed to download llvm from ci" | ||
help_on_error += "\nhelp: old builds get deleted after a certain time" | ||
help_on_error += "\nhelp: if trying to compile an old commit of rustc," | ||
help_on_error += " disable `download-ci-llvm` in config.toml:" | ||
help_on_error += "\n" | ||
help_on_error += "\n[llvm]" | ||
help_on_error += "\ndownload-ci-llvm = false" | ||
help_on_error += "\n" | ||
get( | ||
base, | ||
"{}/{}".format(url, filename), | ||
tarball, | ||
self.checksums_sha256, | ||
verbose=self.verbose, | ||
do_verify=False, | ||
help_on_error=help_on_error, | ||
) | ||
unpack(tarball, tarball_suffix, self.llvm_root(), | ||
match="rust-dev", | ||
verbose=self.verbose) | ||
|
||
def fix_bin_or_dylib(self, fname): | ||
"""Modifies the interpreter section of 'fname' to fix the dynamic linker, | ||
or the RPATH section, to fix the dynamic library search path | ||
|
@@ -816,17 +694,6 @@ def rustfmt_stamp(self): | |
""" | ||
return os.path.join(self.bin_root(True), '.rustfmt-stamp') | ||
|
||
def llvm_stamp(self): | ||
"""Return the path for .llvm-stamp | ||
|
||
>>> rb = RustBuild() | ||
>>> rb.build_dir = "build" | ||
>>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp") | ||
True | ||
""" | ||
return os.path.join(self.llvm_root(), '.llvm-stamp') | ||
|
||
|
||
def program_out_of_date(self, stamp_path, key): | ||
"""Check if the given program stamp is out of date""" | ||
if not os.path.exists(stamp_path) or self.clean: | ||
|
@@ -856,22 +723,6 @@ def bin_root(self, stage0): | |
subdir = "ci-rustc" | ||
return os.path.join(self.build_dir, self.build, subdir) | ||
|
||
def llvm_root(self): | ||
"""Return the CI LLVM root directory | ||
|
||
>>> rb = RustBuild() | ||
>>> rb.build_dir = "build" | ||
>>> rb.llvm_root() == os.path.join("build", "ci-llvm") | ||
True | ||
|
||
When the 'build' property is given should be a nested directory: | ||
|
||
>>> rb.build = "devel" | ||
>>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm") | ||
True | ||
""" | ||
return os.path.join(self.build_dir, self.build, "ci-llvm") | ||
|
||
def get_toml(self, key, section=None): | ||
"""Returns the value of the given key in config.toml, otherwise returns None | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,9 +12,12 @@ use std::env; | |
use std::env::consts::EXE_EXTENSION; | ||
use std::ffi::{OsStr, OsString}; | ||
use std::fs::{self, File}; | ||
use std::io; | ||
use std::io::{self, BufRead, BufReader, ErrorKind}; | ||
use std::path::{Path, PathBuf}; | ||
use std::process::Command; | ||
use std::process::{Command, Stdio}; | ||
|
||
use once_cell::sync::OnceCell; | ||
use xz2::bufread::XzDecoder; | ||
|
||
use crate::builder::{Builder, RunConfig, ShouldRun, Step}; | ||
use crate::config::TargetSelection; | ||
|
@@ -62,6 +65,8 @@ pub fn prebuilt_llvm_config( | |
builder: &Builder<'_>, | ||
target: TargetSelection, | ||
) -> Result<PathBuf, Meta> { | ||
maybe_download_ci_llvm(builder); | ||
|
||
// If we're using a custom LLVM bail out here, but we can only use a | ||
// custom LLVM for the build triple. | ||
if let Some(config) = builder.config.target_config.get(&target) { | ||
|
@@ -111,6 +116,285 @@ pub fn prebuilt_llvm_config( | |
Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() }) | ||
} | ||
|
||
pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { | ||
let config = &builder.config; | ||
if !config.llvm_from_ci { | ||
return; | ||
} | ||
let mut rev_list = Command::new("git"); | ||
rev_list.args(&[ | ||
PathBuf::from("rev-list"), | ||
"--author=bors@rust-lang.org".into(), | ||
"-n1".into(), | ||
"--first-parent".into(), | ||
"HEAD".into(), | ||
"--".into(), | ||
builder.src.join("src/llvm-project"), | ||
builder.src.join("src/bootstrap/download-ci-llvm-stamp"), | ||
// the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` | ||
builder.src.join("src/version"), | ||
]); | ||
let llvm_sha = output(&mut rev_list); | ||
let llvm_sha = llvm_sha.trim(); | ||
|
||
if llvm_sha == "" { | ||
println!("error: could not find commit hash for downloading LLVM"); | ||
println!("help: maybe your repository history is too shallow?"); | ||
println!("help: consider disabling `download-ci-llvm`"); | ||
println!("help: or fetch enough history to include one upstream commit"); | ||
panic!(); | ||
} | ||
|
||
let llvm_root = config.ci_llvm_root(); | ||
let llvm_stamp = llvm_root.join(".llvm-stamp"); | ||
let key = format!("{}{}", llvm_sha, config.llvm_assertions); | ||
if program_out_of_date(&llvm_stamp, &key) && !config.dry_run { | ||
download_ci_llvm(builder, &llvm_sha); | ||
for binary in ["llvm-config", "FileCheck"] { | ||
fix_bin_or_dylib(builder, &llvm_root.join("bin").join(binary)); | ||
} | ||
let llvm_lib = llvm_root.join("lib"); | ||
for entry in t!(fs::read_dir(&llvm_lib)) { | ||
let lib = t!(entry).path(); | ||
if lib.ends_with(".so") { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You cannot use this to check for an extension, only whole file names - the docs say "Only considers whole path components to match." - I have no idea how this ever passed NixOS, it just can't ever work AFAICT. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Opened fix PR: |
||
fix_bin_or_dylib(builder, &lib); | ||
} | ||
} | ||
t!(fs::write(llvm_stamp, key)); | ||
} | ||
} | ||
|
||
fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) { | ||
let llvm_assertions = builder.config.llvm_assertions; | ||
|
||
let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); | ||
let cache_dst = builder.out.join("cache"); | ||
let rustc_cache = cache_dst.join(cache_prefix); | ||
if !rustc_cache.exists() { | ||
t!(fs::create_dir_all(&rustc_cache)); | ||
} | ||
let base = "https://ci-artifacts.rust-lang.org"; | ||
let url = if llvm_assertions { | ||
format!("rustc-builds-alt/{}", llvm_sha) | ||
} else { | ||
format!("rustc-builds/{}", llvm_sha) | ||
}; | ||
let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple); | ||
let tarball = rustc_cache.join(&filename); | ||
if !tarball.exists() { | ||
download_component(builder, base, &format!("{}/{}", url, filename), &tarball); | ||
} | ||
let llvm_root = builder.config.ci_llvm_root(); | ||
unpack(builder, &tarball, &llvm_root); | ||
} | ||
|
||
/// Modifies the interpreter section of 'fname' to fix the dynamic linker, | ||
/// or the RPATH section, to fix the dynamic library search path | ||
/// | ||
/// This is only required on NixOS and uses the PatchELF utility to | ||
/// change the interpreter/RPATH of ELF executables. | ||
/// | ||
/// Please see https://nixos.org/patchelf.html for more information | ||
fn fix_bin_or_dylib(builder: &Builder<'_>, fname: &Path) { | ||
// FIXME: cache NixOS detection? | ||
match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { | ||
Err(_) => return, | ||
Ok(output) if !output.status.success() => return, | ||
Ok(output) => { | ||
let mut s = output.stdout; | ||
if s.last() == Some(&b'\n') { | ||
s.pop(); | ||
} | ||
if s != b"Linux" { | ||
return; | ||
} | ||
} | ||
} | ||
|
||
// If the user has asked binaries to be patched for Nix, then | ||
// don't check for NixOS or `/lib`, just continue to the patching. | ||
// FIXME: shouldn't this take precedence over the `uname` check above? | ||
if !builder.config.patch_binaries_for_nix { | ||
Mark-Simulacrum marked this conversation as resolved.
Show resolved
Hide resolved
|
||
// Use `/etc/os-release` instead of `/etc/NIXOS`. | ||
// The latter one does not exist on NixOS when using tmpfs as root. | ||
const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""]; | ||
let os_release = match File::open("/etc/os-release") { | ||
Err(e) if e.kind() == ErrorKind::NotFound => return, | ||
Err(e) => panic!("failed to access /etc/os-release: {}", e), | ||
Ok(f) => f, | ||
}; | ||
if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) { | ||
return; | ||
} | ||
if Path::new("/lib").exists() { | ||
return; | ||
} | ||
} | ||
|
||
// At this point we're pretty sure the user is running NixOS or using Nix | ||
println!("info: you seem to be using Nix. Attempting to patch {}", fname.display()); | ||
|
||
// Only build `.nix-deps` once. | ||
static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new(); | ||
let mut nix_build_succeeded = true; | ||
let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { | ||
// Run `nix-build` to "build" each dependency (which will likely reuse | ||
// the existing `/nix/store` copy, or at most download a pre-built copy). | ||
// | ||
// Importantly, we create a gc-root called `.nix-deps` in the `build/` | ||
// directory, but still reference the actual `/nix/store` path in the rpath | ||
// as it makes it significantly more robust against changes to the location of | ||
// the `.nix-deps` location. | ||
// | ||
// bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). | ||
// zlib: Needed as a system dependency of `libLLVM-*.so`. | ||
// patchelf: Needed for patching ELF binaries (see doc comment above). | ||
let nix_deps_dir = builder.out.join(".nix-deps"); | ||
const NIX_EXPR: &str = " | ||
with (import <nixpkgs> {}); | ||
symlinkJoin { | ||
name = \"rust-stage0-dependencies\"; | ||
paths = [ | ||
zlib | ||
patchelf | ||
stdenv.cc.bintools | ||
]; | ||
} | ||
"; | ||
nix_build_succeeded = builder.try_run(Command::new("nix-build").args(&[ | ||
Path::new("-E"), | ||
Path::new(NIX_EXPR), | ||
Path::new("-o"), | ||
&nix_deps_dir, | ||
])); | ||
nix_deps_dir | ||
}); | ||
if !nix_build_succeeded { | ||
return; | ||
} | ||
|
||
let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); | ||
let rpath_entries = { | ||
// ORIGIN is a relative default, all binary and dynamic libraries we ship | ||
// appear to have this (even when `../lib` is redundant). | ||
// NOTE: there are only two paths here, delimited by a `:` | ||
let mut entries = OsString::from("$ORIGIN/../lib:"); | ||
entries.push(t!(fs::canonicalize(nix_deps_dir))); | ||
entries.push("/lib"); | ||
entries | ||
}; | ||
patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); | ||
if !fname.ends_with(".so") { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same thing here, this cannot check for an extension (it's equivalent to |
||
// Finally, set the corret .interp for binaries | ||
let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); | ||
// FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ... | ||
let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); | ||
patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); | ||
} | ||
|
||
builder.try_run(patchelf.arg(fname)); | ||
} | ||
|
||
fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &Path) { | ||
// Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. | ||
let tempfile = builder.tempdir().join(dest_path.file_name().unwrap()); | ||
// FIXME: support `do_verify` (only really needed for nightly rustfmt) | ||
// FIXME: support non-utf8 paths? | ||
download_with_retries(builder, tempfile.to_str().unwrap(), &format!("{}/{}", base, url)); | ||
t!(std::fs::rename(&tempfile, dest_path)); | ||
} | ||
|
||
fn download_with_retries(builder: &Builder<'_>, tempfile: &str, url: &str) { | ||
println!("downloading {}", url); | ||
|
||
// FIXME: check if curl is installed instead of skipping straight to powershell | ||
if builder.build.build.contains("windows-msvc") { | ||
for _ in 0..3 { | ||
Mark-Simulacrum marked this conversation as resolved.
Show resolved
Hide resolved
|
||
if builder.try_run(Command::new("PowerShell.exe").args(&[ | ||
"/nologo", | ||
"-Command", | ||
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", | ||
&format!( | ||
"(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", | ||
url, tempfile | ||
), | ||
])) { | ||
return; | ||
} | ||
println!("\nspurious failure, trying again"); | ||
} | ||
} else { | ||
builder.run(Command::new("curl").args(&[ | ||
"-#", | ||
"-y", | ||
"30", | ||
"-Y", | ||
"10", // timeout if speed is < 10 bytes/sec for > 30 seconds | ||
"--connect-timeout", | ||
"30", // timeout if cannot connect within 30 seconds | ||
"--retry", | ||
"3", | ||
"-Sf", | ||
"-o", | ||
tempfile, | ||
url, | ||
])); | ||
} | ||
} | ||
|
||
fn unpack(builder: &Builder<'_>, tarball: &Path, dst: &Path) { | ||
println!("extracting {} to {}", tarball.display(), dst.display()); | ||
if !dst.exists() { | ||
t!(fs::create_dir_all(dst)); | ||
} | ||
|
||
// FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild | ||
const MATCH: &str = "rust-dev"; | ||
|
||
// `tarball` ends with `.tar.xz`; strip that suffix | ||
Mark-Simulacrum marked this conversation as resolved.
Show resolved
Hide resolved
|
||
// example: `rust-dev-nightly-x86_64-unknown-linux-gnu` | ||
let uncompressed_filename = | ||
Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); | ||
let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); | ||
|
||
// decompress the file | ||
let data = t!(File::open(tarball)); | ||
let decompressor = XzDecoder::new(BufReader::new(data)); | ||
|
||
let mut tar = tar::Archive::new(decompressor); | ||
for member in t!(tar.entries()) { | ||
let mut member = t!(member); | ||
let original_path = t!(member.path()).into_owned(); | ||
// skip the top-level directory | ||
if original_path == directory_prefix { | ||
continue; | ||
} | ||
let mut short_path = t!(original_path.strip_prefix(directory_prefix)); | ||
if !short_path.starts_with(MATCH) { | ||
continue; | ||
} | ||
short_path = t!(short_path.strip_prefix(MATCH)); | ||
let dst_path = dst.join(short_path); | ||
builder.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); | ||
if !t!(member.unpack_in(dst)) { | ||
panic!("path traversal attack ??"); | ||
} | ||
let src_path = dst.join(original_path); | ||
if src_path.is_dir() && dst_path.exists() { | ||
continue; | ||
} | ||
t!(fs::rename(src_path, dst_path)); | ||
} | ||
t!(fs::remove_dir_all(dst.join(directory_prefix))); | ||
} | ||
|
||
fn program_out_of_date(stamp: &Path, key: &str) -> bool { | ||
if !stamp.exists() { | ||
return true; | ||
} | ||
t!(fs::read_to_string(stamp)) != key | ||
} | ||
|
||
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] | ||
pub struct Llvm { | ||
pub target: TargetSelection, | ||
|
@@ -153,7 +437,7 @@ impl Step for Llvm { | |
}; | ||
|
||
builder.update_submodule(&Path::new("src").join("llvm-project")); | ||
if builder.config.llvm_link_shared | ||
if builder.llvm_link_shared() | ||
&& (target.contains("windows") || target.contains("apple-darwin")) | ||
{ | ||
panic!("shared linking to LLVM is not currently supported on {}", target.triple); | ||
|
@@ -255,7 +539,7 @@ impl Step for Llvm { | |
// | ||
// If we're not linking rustc to a dynamic LLVM, though, then don't link | ||
// tools to it. | ||
if builder.llvm_link_tools_dynamically(target) && builder.config.llvm_link_shared { | ||
if builder.llvm_link_tools_dynamically(target) && builder.llvm_link_shared() { | ||
cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); | ||
} | ||
|
||
|
Uh oh!
There was an error while loading. Please reload this page.