Skip to content

use precompiled rustc for non-dist builders #122709

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Oct 8, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/bootstrap/src/core/builder.rs
Original file line number Diff line number Diff line change
@@ -414,6 +414,15 @@ impl StepDescription {
.map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind)))
.collect::<Vec<_>>();

if builder.download_rustc() && (builder.kind == Kind::Dist || builder.kind == Kind::Install)
{
eprintln!(
"ERROR: '{}' subcommand is incompatible with `rust.download-rustc`.",
builder.kind.as_str()
);
crate::exit!(1);
}

// sanity checks on rules
for (desc, should_run) in v.iter().zip(&should_runs) {
assert!(
48 changes: 48 additions & 0 deletions src/bootstrap/src/core/builder/tests.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use std::thread;

use build_helper::git::get_closest_merge_commit;

use super::*;
use crate::Flags;
use crate::core::build_steps::doc::DocumentationFormat;
@@ -212,6 +214,52 @@ fn alias_and_path_for_library() {
assert_eq!(first(cache.all::<doc::Std>()), &[doc_std!(A => A, stage = 0)]);
}

#[test]
fn ci_rustc_if_unchanged_logic() {
let config = Config::parse_inner(
Flags::parse(&[
"build".to_owned(),
"--dry-run".to_owned(),
"--set=rust.download-rustc='if-unchanged'".to_owned(),
]),
|&_| Ok(Default::default()),
);

if config.rust_info.is_from_tarball() {
return;
}

let build = Build::new(config.clone());
let builder = Builder::new(&build);

if config.out.exists() {
fs::remove_dir_all(&config.out).unwrap();
}

builder.run_step_descriptions(&Builder::get_step_descriptions(config.cmd.kind()), &[]);

let compiler_path = build.src.join("compiler");
let library_path = build.src.join("compiler");

let commit =
get_closest_merge_commit(Some(&builder.config.src), &builder.config.git_config(), &[
compiler_path.clone(),
library_path.clone(),
])
.unwrap();

let has_changes = !helpers::git(Some(&builder.src))
.args(["diff-index", "--quiet", &commit])
.arg("--")
.args([compiler_path, library_path])
.as_command_mut()
.status()
.unwrap()
.success();

assert!(has_changes == config.download_rustc_commit.is_none());
}

mod defaults {
use pretty_assertions::assert_eq;

51 changes: 40 additions & 11 deletions src/bootstrap/src/core/config/config.rs
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@ use std::str::FromStr;
use std::sync::OnceLock;
use std::{cmp, env, fs};

use build_helper::ci::CiEnv;
use build_helper::exit;
use build_helper::git::{GitConfig, get_closest_merge_commit, output_result};
use serde::{Deserialize, Deserializer};
@@ -22,6 +23,7 @@ use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX;
use crate::core::build_steps::llvm;
pub use crate::core::config::flags::Subcommand;
use crate::core::config::flags::{Color, Flags, Warnings};
use crate::core::download::is_download_ci_available;
use crate::utils::cache::{INTERNER, Interned};
use crate::utils::channel::{self, GitInfo};
use crate::utils::helpers::{self, exe, output, t};
@@ -1627,9 +1629,11 @@ impl Config {
config.mandir = mandir.map(PathBuf::from);
}

config.llvm_assertions =
toml.llvm.as_ref().map_or(false, |llvm| llvm.assertions.unwrap_or(false));

// Store off these values as options because if they're not provided
// we'll infer default values for them later
let mut llvm_assertions = None;
let mut llvm_tests = None;
let mut llvm_enzyme = None;
let mut llvm_plugins = None;
@@ -1712,7 +1716,8 @@ impl Config {
is_user_configured_rust_channel = channel.is_some();
set(&mut config.channel, channel.clone());

config.download_rustc_commit = config.download_ci_rustc_commit(download_rustc);
config.download_rustc_commit =
config.download_ci_rustc_commit(download_rustc, config.llvm_assertions);

debug = debug_toml;
debug_assertions = debug_assertions_toml;
@@ -1848,7 +1853,7 @@ impl Config {
optimize: optimize_toml,
thin_lto,
release_debuginfo,
assertions,
assertions: _,
tests,
enzyme,
plugins,
@@ -1882,7 +1887,6 @@ impl Config {
Some(StringOrBool::Bool(false)) | None => {}
}
set(&mut config.ninja_in_file, ninja);
llvm_assertions = assertions;
llvm_tests = tests;
llvm_enzyme = enzyme;
llvm_plugins = plugins;
@@ -1911,8 +1915,8 @@ impl Config {
config.llvm_enable_warnings = enable_warnings.unwrap_or(false);
config.llvm_build_config = build_config.clone().unwrap_or(Default::default());

let asserts = llvm_assertions.unwrap_or(false);
config.llvm_from_ci = config.parse_download_ci_llvm(download_ci_llvm, asserts);
config.llvm_from_ci =
config.parse_download_ci_llvm(download_ci_llvm, config.llvm_assertions);

if config.llvm_from_ci {
let warn = |option: &str| {
@@ -2080,7 +2084,6 @@ impl Config {
// Now that we've reached the end of our configuration, infer the
// default values for all options that we haven't otherwise stored yet.

config.llvm_assertions = llvm_assertions.unwrap_or(false);
config.llvm_tests = llvm_tests.unwrap_or(false);
config.llvm_enzyme = llvm_enzyme.unwrap_or(false);
config.llvm_plugins = llvm_plugins.unwrap_or(false);
@@ -2419,8 +2422,9 @@ impl Config {
ci_config_toml,
);

let disable_ci_rustc_if_incompatible =
env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE")
// Primarily used by CI runners to avoid handling download-rustc incompatible
// options one by one on shell scripts.
let disable_ci_rustc_if_incompatible = env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE")
.is_some_and(|s| s == "1" || s == "true");

if disable_ci_rustc_if_incompatible && res.is_err() {
@@ -2711,7 +2715,15 @@ impl Config {
}

/// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
fn download_ci_rustc_commit(&self, download_rustc: Option<StringOrBool>) -> Option<String> {
fn download_ci_rustc_commit(
&self,
download_rustc: Option<StringOrBool>,
llvm_assertions: bool,
) -> Option<String> {
if !is_download_ci_available(&self.build.triple, llvm_assertions) {
return None;
}

// If `download-rustc` is not set, default to rebuilding.
let if_unchanged = match download_rustc {
None | Some(StringOrBool::Bool(false)) => return None,
@@ -2724,7 +2736,11 @@ impl Config {

// Look for a version to compare to based on the current commit.
// Only commits merged by bors will have CI artifacts.
let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[]).unwrap();
let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[
self.src.join("compiler"),
self.src.join("library"),
])
.unwrap();
if commit.is_empty() {
println!("ERROR: could not find commit hash for downloading rustc");
println!("HELP: maybe your repository history is too shallow?");
@@ -2733,6 +2749,19 @@ impl Config {
crate::exit!(1);
}

if CiEnv::is_ci() && {
let head_sha =
output(helpers::git(Some(&self.src)).arg("rev-parse").arg("HEAD").as_command_mut());
let head_sha = head_sha.trim();
commit == head_sha
} {
eprintln!("CI rustc commit matches with HEAD and we are in CI.");
eprintln!(
"`rustc.download-ci` functionality will be skipped as artifacts are not available."
);
return None;
}

// Warn if there were changes to the compiler or standard library since the ancestor commit.
let has_changes = !t!(helpers::git(Some(&self.src))
.args(["diff-index", "--quiet", &commit])
2 changes: 1 addition & 1 deletion src/bootstrap/src/core/config/tests.rs
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ use super::{ChangeIdWrapper, Config};
use crate::core::build_steps::clippy::get_clippy_rules_in_order;
use crate::core::config::{LldMode, Target, TargetSelection, TomlConfig};

fn parse(config: &str) -> Config {
pub(crate) fn parse(config: &str) -> Config {
Config::parse_inner(
Flags::parse(&["check".to_string(), "--config=/does/not/exist".to_string()]),
|&_| toml::from_str(&config),
40 changes: 40 additions & 0 deletions src/bootstrap/src/core/download.rs
Original file line number Diff line number Diff line change
@@ -832,3 +832,43 @@ fn path_is_dylib(path: &Path) -> bool {
// The .so is not necessarily the extension, it might be libLLVM.so.18.1
path.to_str().map_or(false, |path| path.contains(".so"))
}

/// Checks whether the CI rustc is available for the given target triple.
pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool {
// All tier 1 targets and tier 2 targets with host tools.
const SUPPORTED_PLATFORMS: &[&str] = &[
"aarch64-apple-darwin",
"aarch64-pc-windows-msvc",
"aarch64-unknown-linux-gnu",
"aarch64-unknown-linux-musl",
"arm-unknown-linux-gnueabi",
"arm-unknown-linux-gnueabihf",
"armv7-unknown-linux-gnueabihf",
"i686-pc-windows-gnu",
"i686-pc-windows-msvc",
"i686-unknown-linux-gnu",
"loongarch64-unknown-linux-gnu",
"powerpc-unknown-linux-gnu",
"powerpc64-unknown-linux-gnu",
"powerpc64le-unknown-linux-gnu",
"riscv64gc-unknown-linux-gnu",
"s390x-unknown-linux-gnu",
"x86_64-apple-darwin",
"x86_64-pc-windows-gnu",
"x86_64-pc-windows-msvc",
"x86_64-unknown-freebsd",
"x86_64-unknown-illumos",
"x86_64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",
"x86_64-unknown-netbsd",
];

const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] =
&["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"];

if llvm_assertions {
SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple)
} else {
SUPPORTED_PLATFORMS.contains(&target_triple)
}
}
15 changes: 14 additions & 1 deletion src/ci/docker/host-x86_64/mingw-check/Dockerfile
Original file line number Diff line number Diff line change
@@ -46,7 +46,20 @@ ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
# Check library crates on all tier 1 targets.
# We disable optimized compiler built-ins because that requires a C toolchain for the target.
# We also skip the x86_64-unknown-linux-gnu target as it is well-tested by other jobs.
ENV SCRIPT python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
ENV SCRIPT \
# `core::builder::tests::ci_rustc_if_unchanged_logic` bootstrap test covers the `rust.download-rustc=if-unchanged` logic.
# Here we are adding a dummy commit on compiler and running that test to ensure when there is a change on the compiler,
# we never download ci rustc with `rust.download-rustc=if-unchanged` option.
echo \"\" >> ../compiler/rustc/src/main.rs && \
git config --global user.email \"[email protected]\" && \
git config --global user.name \"dummy\" && \
git add ../compiler/rustc/src/main.rs && \
git commit -m \"test commit for rust.download-rustc=if-unchanged logic\" && \
DISABLE_CI_RUSTC_IF_INCOMPATIBLE=0 python3 ../x.py test bootstrap -- core::builder::tests::ci_rustc_if_unchanged_logic && \
# Revert the dummy commit
git reset --hard HEAD~1 && \

python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
/scripts/check-default-config-profiles.sh && \
python3 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
python3 ../x.py clippy bootstrap -Dwarnings && \
4 changes: 4 additions & 0 deletions src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile
Original file line number Diff line number Diff line change
@@ -58,6 +58,9 @@ RUN mkdir -p $RUST_INSTALL_DIR/etc
# Fuchsia only supports LLVM.
ENV CODEGEN_BACKENDS llvm

# download-rustc is not allowed for `x install`
ENV NO_DOWNLOAD_CI_RUSTC 1

ENV RUST_CONFIGURE_ARGS \
--prefix=$RUST_INSTALL_DIR \
--sysconfdir=etc \
@@ -70,6 +73,7 @@ ENV RUST_CONFIGURE_ARGS \
--set target.x86_64-unknown-fuchsia.ar=/usr/local/bin/llvm-ar \
--set target.x86_64-unknown-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \
--set target.x86_64-unknown-fuchsia.linker=/usr/local/bin/ld.lld

ENV SCRIPT \
python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \
bash ../src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh
1 change: 1 addition & 0 deletions src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
Original file line number Diff line number Diff line change
@@ -84,6 +84,7 @@ ENV RUST_CONFIGURE_ARGS \
--enable-new-symbol-mangling

ENV HOST_TARGET x86_64-unknown-linux-gnu
ENV FORCE_CI_RUSTC 1

COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/
COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/
1 change: 1 addition & 0 deletions src/ci/docker/run.sh
Original file line number Diff line number Diff line change
@@ -343,6 +343,7 @@ docker \
--env PR_CI_JOB \
--env OBJDIR_ON_HOST="$objdir" \
--env CODEGEN_BACKENDS \
--env DISABLE_CI_RUSTC_IF_INCOMPATIBLE="$DISABLE_CI_RUSTC_IF_INCOMPATIBLE" \
--init \
--rm \
rust-ci \
5 changes: 5 additions & 0 deletions src/ci/github-actions/jobs.yml
Original file line number Diff line number Diff line change
@@ -85,6 +85,9 @@ envs:
# it in each job definition.
pr:
- image: mingw-check
env:
# We are adding (temporarily) a dummy commit on the compiler
READ_ONLY_SRC: "0"
<<: *job-linux-4c
- image: mingw-check-tidy
continue_on_error: true
@@ -207,6 +210,8 @@ auto:
<<: *job-linux-8c

- image: mingw-check
env:
READ_ONLY_SRC: 0
<<: *job-linux-4c

- image: test-various
13 changes: 13 additions & 0 deletions src/ci/run.sh
Original file line number Diff line number Diff line change
@@ -52,6 +52,13 @@ if [ "$CI" != "" ]; then
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set change-id=99999999"
fi

# If runner uses an incompatible option and `FORCE_CI_RUSTC` is not defined,
# switch to in-tree rustc.
if [ "$FORCE_CI_RUSTC" == "" ]; then
echo "debug: `DISABLE_CI_RUSTC_IF_INCOMPATIBLE` configured."
DISABLE_CI_RUSTC_IF_INCOMPATIBLE=1
fi

if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf || \
isCiBranch automation/bors/try; then
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
@@ -169,10 +176,16 @@ else
if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-unchanged"
else
# CI rustc requires CI LLVM to be enabled (see https://github.com/rust-lang/rust/issues/123586).
NO_DOWNLOAD_CI_RUSTC=1
# When building for CI we want to use the static C++ Standard library
# included with LLVM, since a dynamic libstdcpp may not be available.
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.static-libstdcpp"
fi

if [ "$NO_DOWNLOAD_CI_RUSTC" = "" ]; then
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged"
fi
fi

if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then