diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 31158870f394e..8d999302a6d7b 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -2,14 +2,13 @@ use std::any::{type_name, Any};
use std::cell::{Cell, RefCell};
use std::collections::BTreeSet;
use std::env;
-use std::ffi::{OsStr, OsString};
+use std::ffi::OsStr;
use std::fmt::{Debug, Write};
-use std::fs::{self, File};
+use std::fs::{self};
use std::hash::Hash;
-use std::io::{BufRead, BufReader, ErrorKind};
use std::ops::Deref;
use std::path::{Component, Path, PathBuf};
-use std::process::{Command, Stdio};
+use std::process::Command;
use std::time::{Duration, Instant};
use crate::cache::{Cache, Interned, INTERNER};
@@ -24,14 +23,12 @@ use crate::test;
use crate::tool::{self, SourceType};
use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t};
use crate::EXTRA_CHECK_CFGS;
-use crate::{check, Config};
-use crate::{compile, Crate};
+use crate::{check, compile, Crate};
use crate::{Build, CLang, DocTests, GitRepo, Mode};
pub use crate::Compiler;
// FIXME: replace with std::lazy after it gets stabilized and reaches beta
-use once_cell::sync::{Lazy, OnceCell};
-use xz2::bufread::XzDecoder;
+use once_cell::sync::Lazy;
pub struct Builder<'a> {
pub build: &'a Build,
@@ -853,241 +850,6 @@ impl<'a> Builder<'a> {
StepDescription::run(v, self, paths);
}
- /// Modifies the interpreter section of 'fname' to fix the dynamic linker,
- /// or the RPATH section, to fix the dynamic library search path
- ///
- /// This is only required on NixOS and uses the PatchELF utility to
- /// change the interpreter/RPATH of ELF executables.
- ///
- /// Please see https://nixos.org/patchelf.html for more information
- pub(crate) fn fix_bin_or_dylib(&self, fname: &Path) {
- // FIXME: cache NixOS detection?
- match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
- Err(_) => return,
- Ok(output) if !output.status.success() => return,
- Ok(output) => {
- let mut s = output.stdout;
- if s.last() == Some(&b'\n') {
- s.pop();
- }
- if s != b"Linux" {
- return;
- }
- }
- }
-
- // If the user has asked binaries to be patched for Nix, then
- // don't check for NixOS or `/lib`, just continue to the patching.
- // NOTE: this intentionally comes after the Linux check:
- // - patchelf only works with ELF files, so no need to run it on Mac or Windows
- // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
- if !self.config.patch_binaries_for_nix {
- // Use `/etc/os-release` instead of `/etc/NIXOS`.
- // The latter one does not exist on NixOS when using tmpfs as root.
- const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
- let os_release = match File::open("/etc/os-release") {
- Err(e) if e.kind() == ErrorKind::NotFound => return,
- Err(e) => panic!("failed to access /etc/os-release: {}", e),
- Ok(f) => f,
- };
- if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
- return;
- }
- if Path::new("/lib").exists() {
- return;
- }
- }
-
- // At this point we're pretty sure the user is running NixOS or using Nix
- println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
-
- // Only build `.nix-deps` once.
- static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
- let mut nix_build_succeeded = true;
- let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
- // Run `nix-build` to "build" each dependency (which will likely reuse
- // the existing `/nix/store` copy, or at most download a pre-built copy).
- //
- // Importantly, we create a gc-root called `.nix-deps` in the `build/`
- // directory, but still reference the actual `/nix/store` path in the rpath
- // as it makes it significantly more robust against changes to the location of
- // the `.nix-deps` location.
- //
- // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
- // zlib: Needed as a system dependency of `libLLVM-*.so`.
- // patchelf: Needed for patching ELF binaries (see doc comment above).
- let nix_deps_dir = self.out.join(".nix-deps");
- const NIX_EXPR: &str = "
- with (import <nixpkgs> {});
- symlinkJoin {
- name = \"rust-stage0-dependencies\";
- paths = [
- zlib
- patchelf
- stdenv.cc.bintools
- ];
- }
- ";
- nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[
- Path::new("-E"),
- Path::new(NIX_EXPR),
- Path::new("-o"),
- &nix_deps_dir,
- ]));
- nix_deps_dir
- });
- if !nix_build_succeeded {
- return;
- }
-
- let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
- let rpath_entries = {
- // ORIGIN is a relative default, all binary and dynamic libraries we ship
- // appear to have this (even when `../lib` is redundant).
- // NOTE: there are only two paths here, delimited by a `:`
- let mut entries = OsString::from("$ORIGIN/../lib:");
- entries.push(t!(fs::canonicalize(nix_deps_dir)));
- entries.push("/lib");
- entries
- };
- patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
- if !fname.extension().map_or(false, |ext| ext == "so") {
- // Finally, set the correct .interp for binaries
- let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
- // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
- let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
- patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
- }
-
- self.try_run(patchelf.arg(fname));
- }
-
- pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) {
- self.verbose(&format!("download {url}"));
- // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
- let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
- // While bootstrap itself only supports http and https downloads, downstream forks might
- // need to download components from other protocols. The match allows them adding more
- // protocols without worrying about merge conflicts if we change the HTTP implementation.
- match url.split_once("://").map(|(proto, _)| proto) {
- Some("http") | Some("https") => {
- self.download_http_with_retries(&tempfile, url, help_on_error)
- }
- Some(other) => panic!("unsupported protocol {other} in {url}"),
- None => panic!("no protocol in {url}"),
- }
- t!(std::fs::rename(&tempfile, dest_path));
- }
-
- fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
- println!("downloading {}", url);
- // Try curl. If that fails and we are on windows, fallback to PowerShell.
- let mut curl = Command::new("curl");
- curl.args(&[
- "-#",
- "-y",
- "30",
- "-Y",
- "10", // timeout if speed is < 10 bytes/sec for > 30 seconds
- "--connect-timeout",
- "30", // timeout if cannot connect within 30 seconds
- "--retry",
- "3",
- "-Sf",
- "-o",
- ]);
- curl.arg(tempfile);
- curl.arg(url);
- if !self.check_run(&mut curl) {
- if self.build.build.contains("windows-msvc") {
- println!("Fallback to PowerShell");
- for _ in 0..3 {
- if self.try_run(Command::new("PowerShell.exe").args(&[
- "/nologo",
- "-Command",
- "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
- &format!(
- "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
- url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"),
- ),
- ])) {
- return;
- }
- println!("\nspurious failure, trying again");
- }
- }
- if !help_on_error.is_empty() {
- eprintln!("{}", help_on_error);
- }
- crate::detail_exit(1);
- }
- }
-
- pub(crate) fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) {
- println!("extracting {} to {}", tarball.display(), dst.display());
- if !dst.exists() {
- t!(fs::create_dir_all(dst));
- }
-
- // `tarball` ends with `.tar.xz`; strip that suffix
- // example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
- let uncompressed_filename =
- Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
- let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
-
- // decompress the file
- let data = t!(File::open(tarball));
- let decompressor = XzDecoder::new(BufReader::new(data));
-
- let mut tar = tar::Archive::new(decompressor);
- for member in t!(tar.entries()) {
- let mut member = t!(member);
- let original_path = t!(member.path()).into_owned();
- // skip the top-level directory
- if original_path == directory_prefix {
- continue;
- }
- let mut short_path = t!(original_path.strip_prefix(directory_prefix));
- if !short_path.starts_with(pattern) {
- continue;
- }
- short_path = t!(short_path.strip_prefix(pattern));
- let dst_path = dst.join(short_path);
- self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
- if !t!(member.unpack_in(dst)) {
- panic!("path traversal attack ??");
- }
- let src_path = dst.join(original_path);
- if src_path.is_dir() && dst_path.exists() {
- continue;
- }
- t!(fs::rename(src_path, dst_path));
- }
- t!(fs::remove_dir_all(dst.join(directory_prefix)));
- }
-
- /// Returns whether the SHA256 checksum of `path` matches `expected`.
- pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool {
- use sha2::Digest;
-
- self.verbose(&format!("verifying {}", path.display()));
- let mut hasher = sha2::Sha256::new();
- // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components.
- // Consider using streaming IO instead?
- let contents = if self.config.dry_run() { vec![] } else { t!(fs::read(path)) };
- hasher.update(&contents);
- let found = hex::encode(hasher.finalize().as_slice());
- let verified = found == expected;
- if !verified && !self.config.dry_run() {
- println!(
- "invalid checksum: \n\
- found: {found}\n\
- expected: {expected}",
- );
- }
- return verified;
- }
-
/// Obtain a compiler at a given stage and for a given host. Explicitly does
/// not take `Compiler` since all `Compiler` instances are meant to be
/// obtained through this function, since it ensures that they are valid
@@ -1301,19 +1063,6 @@ impl<'a> Builder<'a> {
None
}
- /// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`.
- pub(crate) fn llvm_link_shared(&self) -> bool {
- Config::llvm_link_shared(self)
- }
-
- pub(crate) fn download_rustc(&self) -> bool {
- Config::download_rustc(self)
- }
-
- pub(crate) fn initial_rustfmt(&self) -> Option<PathBuf> {
- Config::initial_rustfmt(self)
- }
-
/// Prepares an invocation of `cargo` to be run.
///
/// This will create a `Command` that represents a pending execution of
diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs
index 258352a21a4ad..eae81b9fc69c8 100644
--- a/src/bootstrap/channel.rs
+++ b/src/bootstrap/channel.rs
@@ -13,8 +13,10 @@ use crate::util::output;
use crate::util::t;
use crate::Build;
+#[derive(Clone, Default)]
pub enum GitInfo {
/// This is not a git repository.
+ #[default]
Absent,
/// This is a git repository.
/// If the info should be used (`ignore_git` is false), this will be
@@ -25,6 +27,7 @@ pub enum GitInfo {
RecordedForTarball(Info),
}
+#[derive(Clone)]
pub struct Info {
pub commit_date: String,
pub sha: String,
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index 67947a263dc87..54906a4918bc1 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -763,10 +763,10 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS
cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative);
- if let Some(ref ver_date) = builder.rust_info.commit_date() {
+ if let Some(ref ver_date) = builder.rust_info().commit_date() {
cargo.env("CFG_VER_DATE", ver_date);
}
- if let Some(ref ver_hash) = builder.rust_info.sha() {
+ if let Some(ref ver_hash) = builder.rust_info().sha() {
cargo.env("CFG_VER_HASH", ver_hash);
}
if !builder.unstable_features() {
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index e843bd411c172..af004aa509854 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -7,19 +7,18 @@ use std::cell::{Cell, RefCell};
use std::cmp;
use std::collections::{HashMap, HashSet};
use std::env;
-use std::ffi::OsStr;
use std::fmt;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::str::FromStr;
-use crate::builder::{Builder, TaskPath};
+use crate::builder::TaskPath;
use crate::cache::{Interned, INTERNER};
-use crate::channel::GitInfo;
+use crate::channel::{self, GitInfo};
pub use crate::flags::Subcommand;
use crate::flags::{Color, Flags};
-use crate::util::{exe, output, program_out_of_date, t};
+use crate::util::{exe, output, t};
use once_cell::sync::OnceCell;
use serde::{Deserialize, Deserializer};
@@ -224,6 +223,7 @@ pub struct Config {
#[cfg(test)]
pub initial_rustfmt: RefCell<RustfmtState>,
pub out: PathBuf,
+ pub rust_info: channel::GitInfo,
}
#[derive(Default, Deserialize)]
@@ -1204,7 +1204,7 @@ impl Config {
config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
- config.download_rustc_commit = download_ci_rustc_commit(&config, rust.download_rustc);
+ config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc);
config.rust_lto = rust
.lto
@@ -1326,6 +1326,7 @@ impl Config {
let default = config.channel == "dev";
config.ignore_git = ignore_git.unwrap_or(default);
+ config.rust_info = GitInfo::new(config.ignore_git, &config.src);
let download_rustc = config.download_rustc_commit.is_some();
// See https://github.com/rust-lang/compiler-team/issues/326
@@ -1401,8 +1402,8 @@ impl Config {
/// Bootstrap embeds a version number into the name of shared libraries it uploads in CI.
/// Return the version it would have used for the given commit.
- pub(crate) fn artifact_version_part(&self, builder: &Builder<'_>, commit: &str) -> String {
- let (channel, version) = if builder.rust_info.is_managed_git_subrepository() {
+ pub(crate) fn artifact_version_part(&self, commit: &str) -> String {
+ let (channel, version) = if self.rust_info.is_managed_git_subrepository() {
let mut channel = self.git();
channel.arg("show").arg(format!("{}:src/ci/channel", commit));
let channel = output(&mut channel);
@@ -1411,14 +1412,14 @@ impl Config {
let version = output(&mut version);
(channel.trim().to_owned(), version.trim().to_owned())
} else {
- let channel = fs::read_to_string(builder.src.join("src/ci/channel"));
- let version = fs::read_to_string(builder.src.join("src/version"));
+ let channel = fs::read_to_string(self.src.join("src/ci/channel"));
+ let version = fs::read_to_string(self.src.join("src/version"));
match (channel, version) {
(Ok(channel), Ok(version)) => {
(channel.trim().to_owned(), version.trim().to_owned())
}
(channel, version) => {
- let src = builder.src.display();
+ let src = self.src.display();
eprintln!("error: failed to determine artifact channel and/or version");
eprintln!(
"help: consider using a git checkout or ensure these files are readable"
@@ -1477,17 +1478,17 @@ impl Config {
///
/// If `false`, llvm should be linked statically.
/// This is computed on demand since LLVM might have to first be downloaded from CI.
- pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool {
- let mut opt = builder.config.llvm_link_shared.get();
- if opt.is_none() && builder.config.dry_run() {
+ pub(crate) fn llvm_link_shared(&self) -> bool {
+ let mut opt = self.llvm_link_shared.get();
+ if opt.is_none() && self.dry_run() {
// just assume static for now - dynamic linking isn't supported on all platforms
return false;
}
let llvm_link_shared = *opt.get_or_insert_with(|| {
- if builder.config.llvm_from_ci {
- crate::native::maybe_download_ci_llvm(builder);
- let ci_llvm = builder.config.ci_llvm_root();
+ if self.llvm_from_ci {
+ self.maybe_download_ci_llvm();
+ let ci_llvm = self.ci_llvm_root();
let link_type = t!(
std::fs::read_to_string(ci_llvm.join("link-type.txt")),
format!("CI llvm missing: {}", ci_llvm.display())
@@ -1499,36 +1500,36 @@ impl Config {
false
}
});
- builder.config.llvm_link_shared.set(opt);
+ self.llvm_link_shared.set(opt);
llvm_link_shared
}
/// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source.
- pub(crate) fn download_rustc(builder: &Builder<'_>) -> bool {
+ pub(crate) fn download_rustc(&self) -> bool {
static DOWNLOAD_RUSTC: OnceCell<bool> = OnceCell::new();
- if builder.config.dry_run() && DOWNLOAD_RUSTC.get().is_none() {
+ if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() {
// avoid trying to actually download the commit
return false;
}
- *DOWNLOAD_RUSTC.get_or_init(|| match &builder.config.download_rustc_commit {
+ *DOWNLOAD_RUSTC.get_or_init(|| match &self.download_rustc_commit {
None => false,
Some(commit) => {
- download_ci_rustc(builder, commit);
+ self.download_ci_rustc(commit);
true
}
})
}
- pub(crate) fn initial_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
- match &mut *builder.config.initial_rustfmt.borrow_mut() {
+ pub(crate) fn initial_rustfmt(&self) -> Option<PathBuf> {
+ match &mut *self.initial_rustfmt.borrow_mut() {
RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()),
RustfmtState::Unavailable => None,
r @ RustfmtState::LazyEvaluated => {
- if builder.config.dry_run() {
+ if self.dry_run() {
return Some(PathBuf::new());
}
- let path = maybe_download_rustfmt(builder);
+ let path = self.maybe_download_rustfmt();
*r = if let Some(p) = &path {
RustfmtState::Downloaded(p.clone())
} else {
@@ -1539,8 +1540,10 @@ impl Config {
}
}
- pub fn verbose(&self) -> bool {
- self.verbose > 0
+ pub fn verbose(&self, msg: &str) {
+ if self.verbose > 0 {
+ println!("{}", msg);
+ }
}
pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool {
@@ -1578,218 +1581,77 @@ impl Config {
pub fn submodules(&self, rust_info: &GitInfo) -> bool {
self.submodules.unwrap_or(rust_info.is_managed_git_subrepository())
}
-}
-
-fn set<T>(field: &mut T, val: Option<T>) {
- if let Some(v) = val {
- *field = v;
- }
-}
-fn threads_from_config(v: u32) -> u32 {
- match v {
- 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32,
- n => n,
- }
-}
+ /// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
+ fn download_ci_rustc_commit(&self, download_rustc: Option<StringOrBool>) -> Option<String> {
+ // If `download-rustc` is not set, default to rebuilding.
+ let if_unchanged = match download_rustc {
+ None | Some(StringOrBool::Bool(false)) => return None,
+ Some(StringOrBool::Bool(true)) => false,
+ Some(StringOrBool::String(s)) if s == "if-unchanged" => true,
+ Some(StringOrBool::String(other)) => {
+ panic!("unrecognized option for download-rustc: {}", other)
+ }
+ };
-/// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
-fn download_ci_rustc_commit(
- config: &Config,
- download_rustc: Option<StringOrBool>,
-) -> Option<String> {
- // If `download-rustc` is not set, default to rebuilding.
- let if_unchanged = match download_rustc {
- None | Some(StringOrBool::Bool(false)) => return None,
- Some(StringOrBool::Bool(true)) => false,
- Some(StringOrBool::String(s)) if s == "if-unchanged" => true,
- Some(StringOrBool::String(other)) => {
- panic!("unrecognized option for download-rustc: {}", other)
+ // Handle running from a directory other than the top level
+ let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"]));
+ let top_level = top_level.trim_end();
+ let compiler = format!("{top_level}/compiler/");
+ let library = format!("{top_level}/library/");
+
+ // Look for a version to compare to based on the current commit.
+ // Only commits merged by bors will have CI artifacts.
+ let merge_base = output(
+ self.git()
+ .arg("rev-list")
+ .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email))
+ .args(&["-n1", "--first-parent", "HEAD"]),
+ );
+ let commit = merge_base.trim_end();
+ if commit.is_empty() {
+ println!("error: could not find commit hash for downloading rustc");
+ println!("help: maybe your repository history is too shallow?");
+ println!("help: consider disabling `download-rustc`");
+ println!("help: or fetch enough history to include one upstream commit");
+ crate::detail_exit(1);
}
- };
-
- // Handle running from a directory other than the top level
- let top_level = output(config.git().args(&["rev-parse", "--show-toplevel"]));
- let top_level = top_level.trim_end();
- let compiler = format!("{top_level}/compiler/");
- let library = format!("{top_level}/library/");
- // Look for a version to compare to based on the current commit.
- // Only commits merged by bors will have CI artifacts.
- let merge_base = output(
- config
+ // Warn if there were changes to the compiler or standard library since the ancestor commit.
+ let has_changes = !t!(self
.git()
- .arg("rev-list")
- .arg(format!("--author={}", config.stage0_metadata.config.git_merge_commit_email))
- .args(&["-n1", "--first-parent", "HEAD"]),
- );
- let commit = merge_base.trim_end();
- if commit.is_empty() {
- println!("error: could not find commit hash for downloading rustc");
- println!("help: maybe your repository history is too shallow?");
- println!("help: consider disabling `download-rustc`");
- println!("help: or fetch enough history to include one upstream commit");
- crate::detail_exit(1);
- }
-
- // Warn if there were changes to the compiler or standard library since the ancestor commit.
- let has_changes = !t!(config
- .git()
- .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library])
- .status())
- .success();
- if has_changes {
- if if_unchanged {
- if config.verbose > 0 {
- println!(
- "warning: saw changes to compiler/ or library/ since {commit}; \
- ignoring `download-rustc`"
- );
+ .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library])
+ .status())
+ .success();
+ if has_changes {
+ if if_unchanged {
+ if self.verbose > 0 {
+ println!(
+ "warning: saw changes to compiler/ or library/ since {commit}; \
+ ignoring `download-rustc`"
+ );
+ }
+ return None;
}
- return None;
+ println!(
+ "warning: `download-rustc` is enabled, but there are changes to \
+ compiler/ or library/"
+ );
}
- println!(
- "warning: `download-rustc` is enabled, but there are changes to \
- compiler/ or library/"
- );
- }
-
- Some(commit.to_string())
-}
-
-fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
- let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?;
- let channel = format!("{version}-{date}");
- let host = builder.config.build;
- let rustfmt_path = builder.config.initial_rustc.with_file_name(exe("rustfmt", host));
- let bin_root = builder.config.out.join(host.triple).join("stage0");
- let rustfmt_stamp = bin_root.join(".rustfmt-stamp");
- if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) {
- return Some(rustfmt_path);
+ Some(commit.to_string())
}
-
- let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple);
- download_component(builder, DownloadSource::Dist, filename, "rustfmt-preview", &date, "stage0");
-
- builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
- builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
-
- builder.create(&rustfmt_stamp, &channel);
- Some(rustfmt_path)
}
-fn download_ci_rustc(builder: &Builder<'_>, commit: &str) {
- builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})"));
- let version = builder.config.artifact_version_part(builder, commit);
- let host = builder.config.build.triple;
- let bin_root = builder.out.join(host).join("ci-rustc");
- let rustc_stamp = bin_root.join(".rustc-stamp");
-
- if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) {
- if bin_root.exists() {
- t!(fs::remove_dir_all(&bin_root));
- }
- let filename = format!("rust-std-{version}-{host}.tar.xz");
- let pattern = format!("rust-std-{host}");
- download_ci_component(builder, filename, &pattern, commit);
- let filename = format!("rustc-{version}-{host}.tar.xz");
- download_ci_component(builder, filename, "rustc", commit);
- // download-rustc doesn't need its own cargo, it can just use beta's.
- let filename = format!("rustc-dev-{version}-{host}.tar.xz");
- download_ci_component(builder, filename, "rustc-dev", commit);
-
- builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
- builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
- let lib_dir = bin_root.join("lib");
- for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
- let lib = t!(lib);
- if lib.path().extension() == Some(OsStr::new("so")) {
- builder.fix_bin_or_dylib(&lib.path());
- }
- }
- t!(fs::write(rustc_stamp, commit));
+fn set<T>(field: &mut T, val: Option<T>) {
+ if let Some(v) = val {
+ *field = v;
}
}
-pub(crate) enum DownloadSource {
- CI,
- Dist,
-}
-
-/// Download a single component of a CI-built toolchain (not necessarily a published nightly).
-// NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
-fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) {
- download_component(builder, DownloadSource::CI, filename, prefix, commit, "ci-rustc")
-}
-
-fn download_component(
- builder: &Builder<'_>,
- mode: DownloadSource,
- filename: String,
- prefix: &str,
- key: &str,
- destination: &str,
-) {
- let cache_dst = builder.out.join("cache");
- let cache_dir = cache_dst.join(key);
- if !cache_dir.exists() {
- t!(fs::create_dir_all(&cache_dir));
- }
-
- let bin_root = builder.out.join(builder.config.build.triple).join(destination);
- let tarball = cache_dir.join(&filename);
- let (base_url, url, should_verify) = match mode {
- DownloadSource::CI => (
- builder.config.stage0_metadata.config.artifacts_server.clone(),
- format!("{key}/{filename}"),
- false,
- ),
- DownloadSource::Dist => {
- let dist_server = env::var("RUSTUP_DIST_SERVER")
- .unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string());
- // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
- (dist_server, format!("dist/{key}/{filename}"), true)
- }
- };
-
- // For the beta compiler, put special effort into ensuring the checksums are valid.
- // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update
- // this on each and every nightly ...
- let checksum = if should_verify {
- let error = format!(
- "src/stage0.json doesn't contain a checksum for {url}. \
- Pre-built artifacts might not be available for this \
- target at this time, see https://doc.rust-lang.org/nightly\
- /rustc/platform-support.html for more information."
- );
- let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error);
- if tarball.exists() {
- if builder.verify(&tarball, sha256) {
- builder.unpack(&tarball, &bin_root, prefix);
- return;
- } else {
- builder.verbose(&format!(
- "ignoring cached file {} due to failed verification",
- tarball.display()
- ));
- builder.remove(&tarball);
- }
- }
- Some(sha256)
- } else if tarball.exists() {
- builder.unpack(&tarball, &bin_root, prefix);
- return;
- } else {
- None
- };
-
- builder.download_component(&format!("{base_url}/{url}"), &tarball, "");
- if let Some(sha256) = checksum {
- if !builder.verify(&tarball, sha256) {
- panic!("failed to verify {}", tarball.display());
- }
+fn threads_from_config(v: u32) -> u32 {
+ match v {
+ 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32,
+ n => n,
}
-
- builder.unpack(&tarball, &bin_root, prefix);
}
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 9fbe476534eb7..aacd2c7eab981 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -924,13 +924,13 @@ impl Step for PlainSourceTarball {
// Create the version file
builder.create(&plain_dst_src.join("version"), &builder.rust_version());
- if let Some(info) = builder.rust_info.info() {
+ if let Some(info) = builder.rust_info().info() {
channel::write_commit_hash_file(&plain_dst_src, &info.sha);
channel::write_commit_info_file(&plain_dst_src, info);
}
// If we're building from git sources, we need to vendor a complete distribution.
- if builder.rust_info.is_managed_git_subrepository() {
+ if builder.rust_info().is_managed_git_subrepository() {
// Ensure we have the submodules checked out.
builder.update_submodule(Path::new("src/tools/rust-analyzer"));
diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs
index c7d21bf3cdb3f..3180a12c85be7 100644
--- a/src/bootstrap/doc.rs
+++ b/src/bootstrap/doc.rs
@@ -405,8 +405,8 @@ impl Step for SharedAssets {
if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) {
let info = t!(fs::read_to_string(&version_input))
.replace("VERSION", &builder.rust_release())
- .replace("SHORT_HASH", builder.rust_info.sha_short().unwrap_or(""))
- .replace("STAMP", builder.rust_info.sha().unwrap_or(""));
+ .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or(""))
+ .replace("STAMP", builder.rust_info().sha().unwrap_or(""));
t!(fs::write(&version_info, &info));
}
@@ -965,7 +965,7 @@ impl Step for RustcBook {
cmd.arg("--rustc");
cmd.arg(&rustc);
cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg());
- if builder.config.verbose() {
+ if builder.is_verbose() {
cmd.arg("--verbose");
}
if self.validate {
diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs
new file mode 100644
index 0000000000000..d0f389df97344
--- /dev/null
+++ b/src/bootstrap/download.rs
@@ -0,0 +1,519 @@
+use std::{
+ env,
+ ffi::{OsStr, OsString},
+ fs::{self, File},
+ io::{BufRead, BufReader, ErrorKind},
+ path::{Path, PathBuf},
+ process::{Command, Stdio},
+};
+
+use once_cell::sync::OnceCell;
+use xz2::bufread::XzDecoder;
+
+use crate::{
+ config::RustfmtMetadata,
+ native::detect_llvm_sha,
+ t,
+ util::{check_run, exe, program_out_of_date, try_run},
+ Config,
+};
+
+/// Generic helpers that are useful anywhere in bootstrap.
+impl Config {
+ pub fn is_verbose(&self) -> bool {
+ self.verbose > 0
+ }
+
+ pub(crate) fn create(&self, path: &Path, s: &str) {
+ if self.dry_run() {
+ return;
+ }
+ t!(fs::write(path, s));
+ }
+
+ pub(crate) fn remove(&self, f: &Path) {
+ if self.dry_run() {
+ return;
+ }
+ fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f));
+ }
+
+ /// Create a temporary directory in `out` and return its path.
+ ///
+ /// NOTE: this temporary directory is shared between all steps;
+ /// if you need an empty directory, create a new subdirectory inside it.
+ pub(crate) fn tempdir(&self) -> PathBuf {
+ let tmp = self.out.join("tmp");
+ t!(fs::create_dir_all(&tmp));
+ tmp
+ }
+
+ /// Runs a command, printing out nice contextual information if it fails.
+ /// Exits if the command failed to execute at all, otherwise returns its
+ /// `status.success()`.
+ pub(crate) fn try_run(&self, cmd: &mut Command) -> bool {
+ if self.dry_run() {
+ return true;
+ }
+ self.verbose(&format!("running: {:?}", cmd));
+ try_run(cmd, self.is_verbose())
+ }
+
+ /// Runs a command, printing out nice contextual information if it fails.
+ /// Returns false if do not execute at all, otherwise returns its
+ /// `status.success()`.
+ pub(crate) fn check_run(&self, cmd: &mut Command) -> bool {
+ if self.dry_run() {
+ return true;
+ }
+ self.verbose(&format!("running: {:?}", cmd));
+ check_run(cmd, self.is_verbose())
+ }
+
+ /// Modifies the interpreter section of 'fname' to fix the dynamic linker,
+ /// or the RPATH section, to fix the dynamic library search path
+ ///
+ /// This is only required on NixOS and uses the PatchELF utility to
+ /// change the interpreter/RPATH of ELF executables.
+ ///
+ /// Please see https://nixos.org/patchelf.html for more information
+ fn fix_bin_or_dylib(&self, fname: &Path) {
+ // FIXME: cache NixOS detection?
+ match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
+ Err(_) => return,
+ Ok(output) if !output.status.success() => return,
+ Ok(output) => {
+ let mut s = output.stdout;
+ if s.last() == Some(&b'\n') {
+ s.pop();
+ }
+ if s != b"Linux" {
+ return;
+ }
+ }
+ }
+
+ // If the user has asked binaries to be patched for Nix, then
+ // don't check for NixOS or `/lib`, just continue to the patching.
+ // NOTE: this intentionally comes after the Linux check:
+ // - patchelf only works with ELF files, so no need to run it on Mac or Windows
+ // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
+ if !self.patch_binaries_for_nix {
+ // Use `/etc/os-release` instead of `/etc/NIXOS`.
+ // The latter one does not exist on NixOS when using tmpfs as root.
+ const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
+ let os_release = match File::open("/etc/os-release") {
+ Err(e) if e.kind() == ErrorKind::NotFound => return,
+ Err(e) => panic!("failed to access /etc/os-release: {}", e),
+ Ok(f) => f,
+ };
+ if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
+ return;
+ }
+ if Path::new("/lib").exists() {
+ return;
+ }
+ }
+
+ // At this point we're pretty sure the user is running NixOS or using Nix
+ println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
+
+ // Only build `.nix-deps` once.
+ static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
+ let mut nix_build_succeeded = true;
+ let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
+ // Run `nix-build` to "build" each dependency (which will likely reuse
+ // the existing `/nix/store` copy, or at most download a pre-built copy).
+ //
+ // Importantly, we create a gc-root called `.nix-deps` in the `build/`
+ // directory, but still reference the actual `/nix/store` path in the rpath
+ // as it makes it significantly more robust against changes to the location of
+ // the `.nix-deps` location.
+ //
+ // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
+ // zlib: Needed as a system dependency of `libLLVM-*.so`.
+ // patchelf: Needed for patching ELF binaries (see doc comment above).
+ let nix_deps_dir = self.out.join(".nix-deps");
+ const NIX_EXPR: &str = "
+ with (import <nixpkgs> {});
+ symlinkJoin {
+ name = \"rust-stage0-dependencies\";
+ paths = [
+ zlib
+ patchelf
+ stdenv.cc.bintools
+ ];
+ }
+ ";
+ nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[
+ Path::new("-E"),
+ Path::new(NIX_EXPR),
+ Path::new("-o"),
+ &nix_deps_dir,
+ ]));
+ nix_deps_dir
+ });
+ if !nix_build_succeeded {
+ return;
+ }
+
+ let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
+ let rpath_entries = {
+ // ORIGIN is a relative default, all binary and dynamic libraries we ship
+ // appear to have this (even when `../lib` is redundant).
+ // NOTE: there are only two paths here, delimited by a `:`
+ let mut entries = OsString::from("$ORIGIN/../lib:");
+ entries.push(t!(fs::canonicalize(nix_deps_dir)));
+ entries.push("/lib");
+ entries
+ };
+ patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
+ if !fname.extension().map_or(false, |ext| ext == "so") {
+ // Finally, set the correct .interp for binaries
+ let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
+ // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
+ let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
+ patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
+ }
+
+ self.try_run(patchelf.arg(fname));
+ }
+
+ fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) {
+ self.verbose(&format!("download {url}"));
+ // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
+ let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
+ // While bootstrap itself only supports http and https downloads, downstream forks might
+ // need to download components from other protocols. The match allows them adding more
+ // protocols without worrying about merge conflicts if we change the HTTP implementation.
+ match url.split_once("://").map(|(proto, _)| proto) {
+ Some("http") | Some("https") => {
+ self.download_http_with_retries(&tempfile, url, help_on_error)
+ }
+ Some(other) => panic!("unsupported protocol {other} in {url}"),
+ None => panic!("no protocol in {url}"),
+ }
+ t!(std::fs::rename(&tempfile, dest_path));
+ }
+
+ fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
+ println!("downloading {}", url);
+ // Try curl. If that fails and we are on windows, fallback to PowerShell.
+ let mut curl = Command::new("curl");
+ curl.args(&[
+ "-#",
+ "-y",
+ "30",
+ "-Y",
+ "10", // timeout if speed is < 10 bytes/sec for > 30 seconds
+ "--connect-timeout",
+ "30", // timeout if cannot connect within 30 seconds
+ "--retry",
+ "3",
+ "-Sf",
+ "-o",
+ ]);
+ curl.arg(tempfile);
+ curl.arg(url);
+ if !self.check_run(&mut curl) {
+ if self.build.contains("windows-msvc") {
+ println!("Fallback to PowerShell");
+ for _ in 0..3 {
+ if self.try_run(Command::new("PowerShell.exe").args(&[
+ "/nologo",
+ "-Command",
+ "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
+ &format!(
+ "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
+ url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"),
+ ),
+ ])) {
+ return;
+ }
+ println!("\nspurious failure, trying again");
+ }
+ }
+ if !help_on_error.is_empty() {
+ eprintln!("{}", help_on_error);
+ }
+ crate::detail_exit(1);
+ }
+ }
+
+ fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) {
+ println!("extracting {} to {}", tarball.display(), dst.display());
+ if !dst.exists() {
+ t!(fs::create_dir_all(dst));
+ }
+
+ // `tarball` ends with `.tar.xz`; strip that suffix
+ // example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
+ let uncompressed_filename =
+ Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
+ let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
+
+ // decompress the file
+ let data = t!(File::open(tarball));
+ let decompressor = XzDecoder::new(BufReader::new(data));
+
+ let mut tar = tar::Archive::new(decompressor);
+ for member in t!(tar.entries()) {
+ let mut member = t!(member);
+ let original_path = t!(member.path()).into_owned();
+ // skip the top-level directory
+ if original_path == directory_prefix {
+ continue;
+ }
+ let mut short_path = t!(original_path.strip_prefix(directory_prefix));
+ if !short_path.starts_with(pattern) {
+ continue;
+ }
+ short_path = t!(short_path.strip_prefix(pattern));
+ let dst_path = dst.join(short_path);
+ self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
+ if !t!(member.unpack_in(dst)) {
+ panic!("path traversal attack ??");
+ }
+ let src_path = dst.join(original_path);
+ if src_path.is_dir() && dst_path.exists() {
+ continue;
+ }
+ t!(fs::rename(src_path, dst_path));
+ }
+ t!(fs::remove_dir_all(dst.join(directory_prefix)));
+ }
+
+ /// Returns whether the SHA256 checksum of `path` matches `expected`.
+ fn verify(&self, path: &Path, expected: &str) -> bool {
+ use sha2::Digest;
+
+ self.verbose(&format!("verifying {}", path.display()));
+ let mut hasher = sha2::Sha256::new();
+ // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components.
+ // Consider using streaming IO instead?
+ let contents = if self.dry_run() { vec![] } else { t!(fs::read(path)) };
+ hasher.update(&contents);
+ let found = hex::encode(hasher.finalize().as_slice());
+ let verified = found == expected;
+ if !verified && !self.dry_run() {
+ println!(
+ "invalid checksum: \n\
+ found: {found}\n\
+ expected: {expected}",
+ );
+ }
+ return verified;
+ }
+}
+
+enum DownloadSource {
+ CI,
+ Dist,
+}
+
+/// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions.
+impl Config {
+ pub(crate) fn maybe_download_rustfmt(&self) -> Option<PathBuf> {
+ let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?;
+ let channel = format!("{version}-{date}");
+
+ let host = self.build;
+ let rustfmt_path = self.initial_rustc.with_file_name(exe("rustfmt", host));
+ let bin_root = self.out.join(host.triple).join("stage0");
+ let rustfmt_stamp = bin_root.join(".rustfmt-stamp");
+ if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) {
+ return Some(rustfmt_path);
+ }
+
+ let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple);
+ self.download_component(DownloadSource::Dist, filename, "rustfmt-preview", &date, "stage0");
+
+ self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
+ self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
+
+ self.create(&rustfmt_stamp, &channel);
+ Some(rustfmt_path)
+ }
+
+ pub(crate) fn download_ci_rustc(&self, commit: &str) {
+ self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})"));
+ let version = self.artifact_version_part(commit);
+ let host = self.build.triple;
+ let bin_root = self.out.join(host).join("ci-rustc");
+ let rustc_stamp = bin_root.join(".rustc-stamp");
+
+ if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit)
+ {
+ if bin_root.exists() {
+ t!(fs::remove_dir_all(&bin_root));
+ }
+ let filename = format!("rust-std-{version}-{host}.tar.xz");
+ let pattern = format!("rust-std-{host}");
+ self.download_ci_component(filename, &pattern, commit);
+ let filename = format!("rustc-{version}-{host}.tar.xz");
+ self.download_ci_component(filename, "rustc", commit);
+ // download-rustc doesn't need its own cargo, it can just use beta's.
+ let filename = format!("rustc-dev-{version}-{host}.tar.xz");
+ self.download_ci_component(filename, "rustc-dev", commit);
+ let filename = format!("rust-src-{version}.tar.xz");
+ self.download_ci_component(filename, "rust-src", commit);
+
+ self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
+ self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
+ let lib_dir = bin_root.join("lib");
+ for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
+ let lib = t!(lib);
+ if lib.path().extension() == Some(OsStr::new("so")) {
+ self.fix_bin_or_dylib(&lib.path());
+ }
+ }
+ t!(fs::write(rustc_stamp, commit));
+ }
+ }
+
+ /// Download a single component of a CI-built toolchain (not necessarily a published nightly).
+ // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
+ fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) {
+ Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc")
+ }
+
+ fn download_component(
+ &self,
+ mode: DownloadSource,
+ filename: String,
+ prefix: &str,
+ key: &str,
+ destination: &str,
+ ) {
+ let cache_dst = self.out.join("cache");
+ let cache_dir = cache_dst.join(key);
+ if !cache_dir.exists() {
+ t!(fs::create_dir_all(&cache_dir));
+ }
+
+ let bin_root = self.out.join(self.build.triple).join(destination);
+ let tarball = cache_dir.join(&filename);
+ let (base_url, url, should_verify) = match mode {
+ DownloadSource::CI => (
+ self.stage0_metadata.config.artifacts_server.clone(),
+ format!("{key}/{filename}"),
+ false,
+ ),
+ DownloadSource::Dist => {
+ let dist_server = env::var("RUSTUP_DIST_SERVER")
+ .unwrap_or(self.stage0_metadata.config.dist_server.to_string());
+ // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
+ (dist_server, format!("dist/{key}/{filename}"), true)
+ }
+ };
+
+ // For the beta compiler, put special effort into ensuring the checksums are valid.
+ // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update
+ // this on each and every nightly ...
+ let checksum = if should_verify {
+ let error = format!(
+ "src/stage0.json doesn't contain a checksum for {url}. \
+ Pre-built artifacts might not be available for this \
+ target at this time, see https://doc.rust-lang.org/nightly\
+ /rustc/platform-support.html for more information."
+ );
+ let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error);
+ if tarball.exists() {
+ if self.verify(&tarball, sha256) {
+ self.unpack(&tarball, &bin_root, prefix);
+ return;
+ } else {
+ self.verbose(&format!(
+ "ignoring cached file {} due to failed verification",
+ tarball.display()
+ ));
+ self.remove(&tarball);
+ }
+ }
+ Some(sha256)
+ } else if tarball.exists() {
+ self.unpack(&tarball, &bin_root, prefix);
+ return;
+ } else {
+ None
+ };
+
+ self.download_file(&format!("{base_url}/{url}"), &tarball, "");
+ if let Some(sha256) = checksum {
+ if !self.verify(&tarball, sha256) {
+ panic!("failed to verify {}", tarball.display());
+ }
+ }
+
+ self.unpack(&tarball, &bin_root, prefix);
+ }
+
+ pub(crate) fn maybe_download_ci_llvm(&self) {
+ if !self.llvm_from_ci {
+ return;
+ }
+ let llvm_root = self.ci_llvm_root();
+ let llvm_stamp = llvm_root.join(".llvm-stamp");
+ let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository());
+ let key = format!("{}{}", llvm_sha, self.llvm_assertions);
+ if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() {
+ self.download_ci_llvm(&llvm_sha);
+ for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
+ self.fix_bin_or_dylib(&t!(entry).path());
+ }
+
+ // Update the timestamp of llvm-config to force rustc_llvm to be
+ // rebuilt. This is a hacky workaround for a deficiency in Cargo where
+ // the rerun-if-changed directive doesn't handle changes very well.
+ // https://github.com/rust-lang/cargo/issues/10791
+ // Cargo only compares the timestamp of the file relative to the last
+ // time `rustc_llvm` build script ran. However, the timestamps of the
+ // files in the tarball are in the past, so it doesn't trigger a
+ // rebuild.
+ let now = filetime::FileTime::from_system_time(std::time::SystemTime::now());
+ let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build));
+ t!(filetime::set_file_times(&llvm_config, now, now));
+
+ let llvm_lib = llvm_root.join("lib");
+ for entry in t!(fs::read_dir(&llvm_lib)) {
+ let lib = t!(entry).path();
+ if lib.extension().map_or(false, |ext| ext == "so") {
+ self.fix_bin_or_dylib(&lib);
+ }
+ }
+ t!(fs::write(llvm_stamp, key));
+ }
+ }
+
+ fn download_ci_llvm(&self, llvm_sha: &str) {
+ let llvm_assertions = self.llvm_assertions;
+
+ let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions);
+ let cache_dst = self.out.join("cache");
+ let rustc_cache = cache_dst.join(cache_prefix);
+ if !rustc_cache.exists() {
+ t!(fs::create_dir_all(&rustc_cache));
+ }
+ let base = if llvm_assertions {
+ &self.stage0_metadata.config.artifacts_with_llvm_assertions_server
+ } else {
+ &self.stage0_metadata.config.artifacts_server
+ };
+ let version = self.artifact_version_part(llvm_sha);
+ let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple);
+ let tarball = rustc_cache.join(&filename);
+ if !tarball.exists() {
+ let help_on_error = "error: failed to download llvm from ci
+
+ help: old builds get deleted after a certain time
+ help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml:
+
+ [llvm]
+ download-ci-llvm = false
+ ";
+ self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error);
+ }
+ let llvm_root = self.ci_llvm_root();
+ self.unpack(&tarball, &llvm_root, "rust-dev");
+ }
+}
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index bbb5a18ba07ba..f4fa556b97450 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -112,15 +112,14 @@ use std::path::{Path, PathBuf};
use std::process::Command;
use std::str;
+use channel::GitInfo;
use config::{DryRun, Target};
use filetime::FileTime;
use once_cell::sync::OnceCell;
use crate::builder::Kind;
use crate::config::{LlvmLibunwind, TargetSelection};
-use crate::util::{
- check_run, exe, libdir, mtime, output, run, run_suppressed, try_run, try_run_suppressed, CiEnv,
-};
+use crate::util::{exe, libdir, mtime, output, run, run_suppressed, try_run_suppressed, CiEnv};
mod bolt;
mod builder;
@@ -133,6 +132,7 @@ mod compile;
mod config;
mod dist;
mod doc;
+mod download;
mod flags;
mod format;
mod install;
@@ -281,7 +281,6 @@ pub struct Build {
src: PathBuf,
out: PathBuf,
bootstrap_out: PathBuf,
- rust_info: channel::GitInfo,
cargo_info: channel::GitInfo,
rust_analyzer_info: channel::GitInfo,
clippy_info: channel::GitInfo,
@@ -396,6 +395,28 @@ pub enum CLang {
Cxx,
}
+macro_rules! forward {
+ ( $( $fn:ident( $($param:ident: $ty:ty),* ) $( -> $ret:ty)? ),+ $(,)? ) => {
+ impl Build {
+ $( fn $fn(&self, $($param: $ty),* ) $( -> $ret)? {
+ self.config.$fn( $($param),* )
+ } )+
+ }
+ }
+}
+
+forward! {
+ verbose(msg: &str),
+ is_verbose() -> bool,
+ create(path: &Path, s: &str),
+ remove(f: &Path),
+ tempdir() -> PathBuf,
+ try_run(cmd: &mut Command) -> bool,
+ llvm_link_shared() -> bool,
+ download_rustc() -> bool,
+ initial_rustfmt() -> Option<PathBuf>,
+}
+
impl Build {
/// Creates a new set of build configuration from the `flags` on the command
/// line and the filesystem `config`.
@@ -499,7 +520,6 @@ impl Build {
out,
bootstrap_out,
- rust_info,
cargo_info,
rust_analyzer_info,
clippy_info,
@@ -570,7 +590,7 @@ impl Build {
t!(std::fs::read_dir(dir)).next().is_none()
}
- if !self.config.submodules(&self.rust_info) {
+ if !self.config.submodules(&self.rust_info()) {
return;
}
@@ -636,7 +656,7 @@ impl Build {
/// This avoids contributors checking in a submodule change by accident.
pub fn maybe_update_submodules(&self) {
// Avoid running git when there isn't a git checkout.
- if !self.config.submodules(&self.rust_info) {
+ if !self.config.submodules(&self.rust_info()) {
return;
}
let output = output(
@@ -735,6 +755,10 @@ impl Build {
cleared
}
+ fn rust_info(&self) -> &GitInfo {
+ &self.config.rust_info
+ }
+
/// Gets the space-separated set of activated features for the standard
/// library.
fn std_features(&self, target: TargetSelection) -> String {
@@ -963,17 +987,6 @@ impl Build {
run_suppressed(cmd)
}
- /// Runs a command, printing out nice contextual information if it fails.
- /// Exits if the command failed to execute at all, otherwise returns its
- /// `status.success()`.
- fn try_run(&self, cmd: &mut Command) -> bool {
- if self.config.dry_run() {
- return true;
- }
- self.verbose(&format!("running: {:?}", cmd));
- try_run(cmd, self.is_verbose())
- }
-
/// Runs a command, printing out nice contextual information if it fails.
/// Exits if the command failed to execute at all, otherwise returns its
/// `status.success()`.
@@ -985,28 +998,6 @@ impl Build {
try_run_suppressed(cmd)
}
- /// Runs a command, printing out nice contextual information if it fails.
- /// Returns false if do not execute at all, otherwise returns its
- /// `status.success()`.
- fn check_run(&self, cmd: &mut Command) -> bool {
- if self.config.dry_run() {
- return true;
- }
- self.verbose(&format!("running: {:?}", cmd));
- check_run(cmd, self.is_verbose())
- }
-
- pub fn is_verbose(&self) -> bool {
- self.verbosity > 0
- }
-
- /// Prints a message if this build is configured in verbose mode.
- fn verbose(&self, msg: &str) {
- if self.is_verbose() {
- println!("{}", msg);
- }
- }
-
pub fn is_verbose_than(&self, level: usize) -> bool {
self.verbosity > level
}
@@ -1269,7 +1260,7 @@ impl Build {
match &self.config.channel[..] {
"stable" => num.to_string(),
"beta" => {
- if self.rust_info.is_managed_git_subrepository() && !self.config.ignore_git {
+ if self.rust_info().is_managed_git_subrepository() && !self.config.ignore_git {
format!("{}-beta.{}", num, self.beta_prerelease_version())
} else {
format!("{}-beta", num)
@@ -1329,7 +1320,7 @@ impl Build {
/// Note that this is a descriptive string which includes the commit date,
/// sha, version, etc.
fn rust_version(&self) -> String {
- let mut version = self.rust_info.version(self, &self.version);
+ let mut version = self.rust_info().version(self, &self.version);
if let Some(ref s) = self.config.description {
version.push_str(" (");
version.push_str(s);
@@ -1340,7 +1331,7 @@ impl Build {
/// Returns the full commit hash.
fn rust_sha(&self) -> Option<&str> {
- self.rust_info.sha()
+ self.rust_info().sha()
}
/// Returns the `a.b.c` version that the given package is at.
@@ -1426,16 +1417,6 @@ impl Build {
paths
}
- /// Create a temporary directory in `out` and return its path.
- ///
- /// NOTE: this temporary directory is shared between all steps;
- /// if you need an empty directory, create a new subdirectory inside it.
- fn tempdir(&self) -> PathBuf {
- let tmp = self.out.join("tmp");
- t!(fs::create_dir_all(&tmp));
- tmp
- }
-
/// Copies a file from `src` to `dst`
pub fn copy(&self, src: &Path, dst: &Path) {
self.copy_internal(src, dst, false);
@@ -1545,13 +1526,6 @@ impl Build {
chmod(&dst, perms);
}
- fn create(&self, path: &Path, s: &str) {
- if self.config.dry_run() {
- return;
- }
- t!(fs::write(path, s));
- }
-
fn read(&self, path: &Path) -> String {
if self.config.dry_run() {
return String::new();
@@ -1590,13 +1564,6 @@ impl Build {
if !self.config.dry_run() { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) }
}
- fn remove(&self, f: &Path) {
- if self.config.dry_run() {
- return;
- }
- fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f));
- }
-
/// Returns if config.ninja is enabled, and checks for ninja existence,
/// exiting with a nicer error message if not.
fn ninja(&self) -> bool {
diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index 2407291ceea30..f6c453ebe107b 100644
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -19,9 +19,9 @@ use std::process::Command;
use crate::bolt::{instrument_with_bolt_inplace, optimize_library_with_bolt_inplace};
use crate::builder::{Builder, RunConfig, ShouldRun, Step};
use crate::channel;
-use crate::config::TargetSelection;
+use crate::config::{Config, TargetSelection};
use crate::util::get_clang_cl_resource_dir;
-use crate::util::{self, exe, output, program_out_of_date, t, up_to_date};
+use crate::util::{self, exe, output, t, up_to_date};
use crate::{CLang, GitRepo};
pub struct Meta {
@@ -65,7 +65,7 @@ pub fn prebuilt_llvm_config(
builder: &Builder<'_>,
target: TargetSelection,
) -> Result<PathBuf, Meta> {
- maybe_download_ci_llvm(builder);
+ builder.config.maybe_download_ci_llvm();
// If we're using a custom LLVM bail out here, but we can only use a
// custom LLVM for the build triple.
@@ -117,7 +117,7 @@ pub fn prebuilt_llvm_config(
}
/// This retrieves the LLVM sha we *want* to use, according to git history.
-pub(crate) fn detect_llvm_sha(config: &crate::config::Config, is_git: bool) -> String {
+pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String {
let llvm_sha = if is_git {
let mut rev_list = config.git();
rev_list.args(&[
@@ -155,7 +155,7 @@ pub(crate) fn detect_llvm_sha(config: &crate::config::Config, is_git: bool) -> S
/// This checks both the build triple platform to confirm we're usable at all,
/// and then verifies if the current HEAD matches the detected LLVM SHA head,
/// in which case LLVM is indicated as not available.
-pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool) -> bool {
+pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool {
// This is currently all tier 1 targets and tier 2 targets with host tools
// (since others may not have CI artifacts)
// https://doc.rust-lang.org/rustc/platform-support.html#tier-1
@@ -217,80 +217,6 @@ pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool
true
}
-pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
- let config = &builder.config;
- if !config.llvm_from_ci {
- return;
- }
- let llvm_root = config.ci_llvm_root();
- let llvm_stamp = llvm_root.join(".llvm-stamp");
- let llvm_sha = detect_llvm_sha(&config, builder.rust_info.is_managed_git_subrepository());
- let key = format!("{}{}", llvm_sha, config.llvm_assertions);
- if program_out_of_date(&llvm_stamp, &key) && !config.dry_run() {
- download_ci_llvm(builder, &llvm_sha);
- for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
- builder.fix_bin_or_dylib(&t!(entry).path());
- }
-
- // Update the timestamp of llvm-config to force rustc_llvm to be
- // rebuilt. This is a hacky workaround for a deficiency in Cargo where
- // the rerun-if-changed directive doesn't handle changes very well.
- // https://github.com/rust-lang/cargo/issues/10791
- // Cargo only compares the timestamp of the file relative to the last
- // time `rustc_llvm` build script ran. However, the timestamps of the
- // files in the tarball are in the past, so it doesn't trigger a
- // rebuild.
- let now = filetime::FileTime::from_system_time(std::time::SystemTime::now());
- let llvm_config = llvm_root.join("bin").join(exe("llvm-config", builder.config.build));
- t!(filetime::set_file_times(&llvm_config, now, now));
-
- let llvm_lib = llvm_root.join("lib");
- for entry in t!(fs::read_dir(&llvm_lib)) {
- let lib = t!(entry).path();
- if lib.extension().map_or(false, |ext| ext == "so") {
- builder.fix_bin_or_dylib(&lib);
- }
- }
- t!(fs::write(llvm_stamp, key));
- }
-}
-
-fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
- let llvm_assertions = builder.config.llvm_assertions;
-
- let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions);
- let cache_dst = builder.out.join("cache");
- let rustc_cache = cache_dst.join(cache_prefix);
- if !rustc_cache.exists() {
- t!(fs::create_dir_all(&rustc_cache));
- }
- let base = if llvm_assertions {
- &builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server
- } else {
- &builder.config.stage0_metadata.config.artifacts_server
- };
- let version = builder.config.artifact_version_part(builder, llvm_sha);
- let filename = format!("rust-dev-{}-{}.tar.xz", version, builder.build.build.triple);
- let tarball = rustc_cache.join(&filename);
- if !tarball.exists() {
- let help_on_error = "error: failed to download llvm from ci
-
-help: old builds get deleted after a certain time
-help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml:
-
-[llvm]
-download-ci-llvm = false
-";
- builder.download_component(
- &format!("{base}/{llvm_sha}/{filename}"),
- &tarball,
- help_on_error,
- );
- }
- let llvm_root = builder.config.ci_llvm_root();
- builder.unpack(&tarball, &llvm_root, "rust-dev");
-}
-
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Llvm {
pub target: TargetSelection,
diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs
index 35c66cfd95f2f..631d42acb93fc 100644
--- a/src/bootstrap/sanity.rs
+++ b/src/bootstrap/sanity.rs
@@ -74,7 +74,7 @@ pub fn check(build: &mut Build) {
let mut cmd_finder = Finder::new();
// If we've got a git directory we're gonna need git to update
// submodules and learn about various other aspects.
- if build.rust_info.is_managed_git_subrepository() {
+ if build.rust_info().is_managed_git_subrepository() {
cmd_finder.must_have("git");
}
diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs
index 82b063583c9f9..fc850a22b2f6f 100644
--- a/src/bootstrap/tarball.rs
+++ b/src/bootstrap/tarball.rs
@@ -298,7 +298,7 @@ impl<'a> Tarball<'a> {
fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball {
t!(std::fs::create_dir_all(&self.overlay_dir));
self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder));
- if let Some(info) = self.builder.rust_info.info() {
+ if let Some(info) = self.builder.rust_info().info() {
channel::write_commit_hash_file(&self.overlay_dir, &info.sha);
channel::write_commit_info_file(&self.overlay_dir, info);
}