Skip to content

Refactor tools/build-mainfest #58995

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 22, 2019
+197 −204
Merged

Refactor tools/build-mainfest #58995

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
401 changes: 197 additions & 204 deletions src/tools/build-manifest/src/main.rs
View file
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@ use std::io::{self, Read, Write};
use std::path::{PathBuf, Path};
use std::process::{Command, Stdio};

static HOSTS: &'static [&'static str] = &[
static HOSTS: &[&str] = &[
"aarch64-unknown-linux-gnu",
"arm-unknown-linux-gnueabi",
"arm-unknown-linux-gnueabihf",
@@ -35,7 +35,7 @@ static HOSTS: &'static [&'static str] = &[
"x86_64-unknown-netbsd",
];

static TARGETS: &'static [&'static str] = &[
static TARGETS: &[&str] = &[
"aarch64-apple-ios",
"aarch64-fuchsia",
"aarch64-linux-android",
@@ -115,7 +115,7 @@ static TARGETS: &'static [&'static str] = &[
"x86_64-unknown-redox",
];

static DOCS_TARGETS: &'static [&'static str] = &[
static DOCS_TARGETS: &[&str] = &[
"i686-apple-darwin",
"i686-pc-windows-gnu",
"i686-pc-windows-msvc",
@@ -126,7 +126,7 @@ static DOCS_TARGETS: &'static [&'static str] = &[
"x86_64-unknown-linux-gnu",
];

static MINGW: &'static [&'static str] = &[
static MINGW: &[&str] = &[
"i686-pc-windows-gnu",
"x86_64-pc-windows-gnu",
];
@@ -153,7 +153,7 @@ struct Rename {
to: String,
}

#[derive(Serialize)]
#[derive(Serialize, Default)]
struct Target {
available: bool,
url: Option<String>,
@@ -165,17 +165,7 @@ struct Target {
}

impl Target {
fn unavailable() -> Target {
Target {
available: false,
url: None,
hash: None,
xz_url: None,
xz_hash: None,
components: None,
extensions: None,
}
}
fn unavailable() -> Self { Self::default() }
}

#[derive(Serialize)]
@@ -184,6 +174,12 @@ struct Component {
target: String,
}

impl Component {
fn from_str(pkg: &str, target: &str) -> Self {
Self { pkg: pkg.to_string(), target: target.to_string() }
}
}

macro_rules! t {
($e:expr) => (match $e {
Ok(e) => e,
@@ -301,6 +297,25 @@ fn main() {
}.build();
}

enum PkgType { RustSrc, Cargo, Rls, Clippy, Rustfmt, LlvmTools, Lldb, Miri, Other }

impl PkgType {
fn from_component(component: &str) -> Self {
use PkgType::*;
match component {
"rust-src" => RustSrc,
"cargo" => Cargo,
"rls" | "rls-preview" => Rls,
"clippy" | "clippy-preview" => Clippy,
"rustfmt" | "rustfmt-preview" => Rustfmt,
"llvm-tools" | "llvm-tools-preview" => LlvmTools,
"lldb" | "lldb-preview" => Lldb,
"miri" | "miri-preview" => Miri,
_ => Other,
}
}
}

impl Builder {
fn build(&mut self) {
self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
@@ -349,39 +364,56 @@ impl Builder {
renames: BTreeMap::new(),
profiles: BTreeMap::new(),
};
self.add_packages_to(&mut manifest);
self.add_profiles_to(&mut manifest);
self.add_renames_to(&mut manifest);
manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
manifest
}

self.package("rustc", &mut manifest.pkg, HOSTS);
self.package("cargo", &mut manifest.pkg, HOSTS);
self.package("rust-mingw", &mut manifest.pkg, MINGW);
self.package("rust-std", &mut manifest.pkg, TARGETS);
self.package("rust-docs", &mut manifest.pkg, DOCS_TARGETS);
self.package("rust-src", &mut manifest.pkg, &["*"]);
self.package("rls-preview", &mut manifest.pkg, HOSTS);
self.package("clippy-preview", &mut manifest.pkg, HOSTS);
self.package("miri", &mut manifest.pkg, HOSTS);
self.package("rustfmt-preview", &mut manifest.pkg, HOSTS);
self.package("rust-analysis", &mut manifest.pkg, TARGETS);
self.package("llvm-tools-preview", &mut manifest.pkg, TARGETS);
self.package("lldb-preview", &mut manifest.pkg, TARGETS);

self.profile("minimal",
&mut manifest.profiles,
&["rustc", "cargo", "rust-std", "rust-mingw"]);
self.profile("default",
&mut manifest.profiles,
&["rustc", "cargo", "rust-std", "rust-mingw",
"rust-docs", "rustfmt-preview", "clippy-preview"]);
self.profile("complete",
&mut manifest.profiles,
&["rustc", "cargo", "rust-std", "rust-mingw",
"rust-docs", "rustfmt-preview", "clippy-preview",
"rls-preview", "rust-src", "llvm-tools-preview",
"lldb-preview", "rust-analysis", "miri"]);

manifest.renames.insert("rls".to_owned(), Rename { to: "rls-preview".to_owned() });
manifest.renames.insert("rustfmt".to_owned(), Rename { to: "rustfmt-preview".to_owned() });
manifest.renames.insert("clippy".to_owned(), Rename { to: "clippy-preview".to_owned() });
fn add_packages_to(&mut self, manifest: &mut Manifest) {
let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
package("rustc", HOSTS);
package("cargo", HOSTS);
package("rust-mingw", MINGW);
package("rust-std", TARGETS);
package("rust-docs", DOCS_TARGETS);
package("rust-src", &["*"]);
package("rls-preview", HOSTS);
package("clippy-preview", HOSTS);
package("miri", HOSTS);
package("rustfmt-preview", HOSTS);
package("rust-analysis", TARGETS);
package("llvm-tools-preview", TARGETS);
package("lldb-preview", TARGETS);
}

fn add_profiles_to(&mut self, manifest: &mut Manifest) {
let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
profile("default", &[
"rustc", "cargo", "rust-std", "rust-mingw",
"rust-docs", "rustfmt-preview", "clippy-preview"
]);
profile("complete", &[
"rustc", "cargo", "rust-std", "rust-mingw",
"rust-docs", "rustfmt-preview", "clippy-preview",
"rls-preview", "rust-src", "llvm-tools-preview",
"lldb-preview", "rust-analysis", "miri"
]);
}

fn add_renames_to(&self, manifest: &mut Manifest) {
let mut rename = |from: &str, to: &str| manifest.renames.insert(
from.to_owned(),
Rename { to: to.to_owned() }
);
rename("rls", "rls-preview");
rename("rustfmt", "rustfmt-preview");
rename("clippy", "clippy-preview");
}

fn rust_package(&mut self, manifest: &Manifest) -> Package {
let mut pkg = Package {
version: self.cached_version("rust")
.as_ref()
@@ -391,90 +423,82 @@ impl Builder {
target: BTreeMap::new(),
};
for host in HOSTS {
let filename = self.filename("rust", host);
let digest = match self.digests.remove(&filename) {
Some(digest) => digest,
None => {
pkg.target.insert(host.to_string(), Target::unavailable());
continue
}
};
let xz_filename = filename.replace(".tar.gz", ".tar.xz");
let xz_digest = self.digests.remove(&xz_filename);
let mut components = Vec::new();
let mut extensions = Vec::new();

// rustc/rust-std/cargo/docs are all required, and so is rust-mingw
// if it's available for the target.
components.extend(vec![
Component { pkg: "rustc".to_string(), target: host.to_string() },
Component { pkg: "rust-std".to_string(), target: host.to_string() },
Component { pkg: "cargo".to_string(), target: host.to_string() },
Component { pkg: "rust-docs".to_string(), target: host.to_string() },
]);
if host.contains("pc-windows-gnu") {
components.push(Component {
pkg: "rust-mingw".to_string(),
target: host.to_string(),
});
}

// Tools are always present in the manifest, but might be marked as unavailable if they
// weren't built
extensions.extend(vec![
Component { pkg: "clippy-preview".to_string(), target: host.to_string() },
Component { pkg: "miri".to_string(), target: host.to_string() },
Component { pkg: "rls-preview".to_string(), target: host.to_string() },
Component { pkg: "rustfmt-preview".to_string(), target: host.to_string() },
Component { pkg: "llvm-tools-preview".to_string(), target: host.to_string() },
Component { pkg: "lldb-preview".to_string(), target: host.to_string() },
Component { pkg: "rust-analysis".to_string(), target: host.to_string() },
]);

for target in TARGETS {
if target != host {
extensions.push(Component {
pkg: "rust-std".to_string(),
target: target.to_string(),
});
}
}
extensions.push(Component {
pkg: "rust-src".to_string(),
target: "*".to_string(),
});

// If the components/extensions don't actually exist for this
// particular host/target combination then nix it entirely from our
// lists.
{
let has_component = |c: &Component| {
if c.target == "*" {
return true
}
let pkg = match manifest.pkg.get(&c.pkg) {
Some(p) => p,
None => return false,
};
pkg.target.get(&c.target).is_some()
};
extensions.retain(&has_component);
components.retain(&has_component);
if let Some(target) = self.target_host_combination(host, &manifest) {
pkg.target.insert(host.to_string(), target);
} else {
pkg.target.insert(host.to_string(), Target::unavailable());
continue
}
}
pkg
}

pkg.target.insert(host.to_string(), Target {
available: true,
url: Some(self.url(&filename)),
hash: Some(digest),
xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
xz_hash: xz_digest,
components: Some(components),
extensions: Some(extensions),
});
fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
let filename = self.filename("rust", host);
let digest = self.digests.remove(&filename)?;
let xz_filename = filename.replace(".tar.gz", ".tar.xz");
let xz_digest = self.digests.remove(&xz_filename);
let mut components = Vec::new();
let mut extensions = Vec::new();

let host_component = |pkg| Component::from_str(pkg, host);

// rustc/rust-std/cargo/docs are all required,
// and so is rust-mingw if it's available for the target.
components.extend(vec![
host_component("rustc"),
host_component("rust-std"),
host_component("cargo"),
host_component("rust-docs"),
]);
if host.contains("pc-windows-gnu") {
components.push(host_component("rust-mingw"));
}
manifest.pkg.insert("rust".to_string(), pkg);

manifest
// Tools are always present in the manifest,
// but might be marked as unavailable if they weren't built.
extensions.extend(vec![
host_component("clippy-preview"),
host_component("miri"),
host_component("rls-preview"),
host_component("rustfmt-preview"),
host_component("llvm-tools-preview"),
host_component("lldb-preview"),
host_component("rust-analysis"),
]);

extensions.extend(
TARGETS.iter()
.filter(|&&target| target != host)
.map(|target| Component::from_str("rust-std", target))
);
extensions.push(Component::from_str("rust-src", "*"));

// If the components/extensions don't actually exist for this
// particular host/target combination then nix it entirely from our
// lists.
let has_component = |c: &Component| {
if c.target == "*" {
return true
}
let pkg = match manifest.pkg.get(&c.pkg) {
Some(p) => p,
None => return false,
};
pkg.target.get(&c.target).is_some()
};
extensions.retain(&has_component);
components.retain(&has_component);

Some(Target {
available: true,
url: Some(self.url(&filename)),
hash: Some(digest),
xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
xz_hash: xz_digest,
components: Some(components),
extensions: Some(extensions),
})
}

fn profile(&mut self,
@@ -488,10 +512,11 @@ impl Builder {
pkgname: &str,
dst: &mut BTreeMap<String, Package>,
targets: &[&str]) {
let (version, is_present) = match *self.cached_version(pkgname) {
Some(ref version) => (version.clone(), true),
None => (String::new(), false),
};
let (version, is_present) = self.cached_version(pkgname)
.as_ref()
.cloned()
.map(|version| (version, true))
.unwrap_or_default();

let targets = targets.iter().map(|name| {
if is_present {
@@ -515,15 +540,7 @@ impl Builder {
} else {
// If the component is not present for this build add it anyway but mark it as
// unavailable -- this way rustup won't allow upgrades without --force
(name.to_string(), Target {
available: false,
url: None,
hash: None,
xz_url: None,
xz_hash: None,
components: None,
extensions: None,
})
(name.to_string(), Target::unavailable())
}
}).collect();

@@ -542,89 +559,65 @@ impl Builder {
}

fn filename(&self, component: &str, target: &str) -> String {
if component == "rust-src" {
format!("rust-src-{}.tar.gz", self.rust_release)
} else if component == "cargo" {
format!("cargo-{}-{}.tar.gz", self.cargo_release, target)
} else if component == "rls" || component == "rls-preview" {
format!("rls-{}-{}.tar.gz", self.rls_release, target)
} else if component == "clippy" || component == "clippy-preview" {
format!("clippy-{}-{}.tar.gz", self.clippy_release, target)
} else if component == "rustfmt" || component == "rustfmt-preview" {
format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target)
} else if component == "llvm-tools" || component == "llvm-tools-preview" {
format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target)
} else if component == "lldb" || component == "lldb-preview" {
format!("lldb-{}-{}.tar.gz", self.lldb_release, target)
} else if component == "miri" || component == "miri-preview" {
format!("miri-{}-{}.tar.gz", self.miri_release, target)
} else {
format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
use PkgType::*;
match PkgType::from_component(component) {
RustSrc => format!("rust-src-{}.tar.gz", self.rust_release),
Cargo => format!("cargo-{}-{}.tar.gz", self.cargo_release, target),
Rls => format!("rls-{}-{}.tar.gz", self.rls_release, target),
Clippy => format!("clippy-{}-{}.tar.gz", self.clippy_release, target),
Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target),
LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target),
Lldb => format!("lldb-{}-{}.tar.gz", self.lldb_release, target),
Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target),
Other => format!("{}-{}-{}.tar.gz", component, self.rust_release, target),
}
}

fn cached_version(&self, component: &str) -> &Option<String> {
if component == "cargo" {
&self.cargo_version
} else if component == "rls" || component == "rls-preview" {
&self.rls_version
} else if component == "clippy" || component == "clippy-preview" {
&self.clippy_version
} else if component == "rustfmt" || component == "rustfmt-preview" {
&self.rustfmt_version
} else if component == "llvm-tools" || component == "llvm-tools-preview" {
&self.llvm_tools_version
} else if component == "lldb" || component == "lldb-preview" {
&self.lldb_version
} else if component == "miri" || component == "miri-preview" {
&self.miri_version
} else {
&self.rust_version
use PkgType::*;
match PkgType::from_component(component) {
Cargo => &self.cargo_version,
Rls => &self.rls_version,
Clippy => &self.clippy_version,
Rustfmt => &self.rustfmt_version,
LlvmTools => &self.llvm_tools_version,
Lldb => &self.lldb_version,
Miri => &self.miri_version,
_ => &self.rust_version,
}
}

fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
if component == "cargo" {
&self.cargo_git_commit_hash
} else if component == "rls" || component == "rls-preview" {
&self.rls_git_commit_hash
} else if component == "clippy" || component == "clippy-preview" {
&self.clippy_git_commit_hash
} else if component == "rustfmt" || component == "rustfmt-preview" {
&self.rustfmt_git_commit_hash
} else if component == "llvm-tools" || component == "llvm-tools-preview" {
&self.llvm_tools_git_commit_hash
} else if component == "lldb" || component == "lldb-preview" {
&self.lldb_git_commit_hash
} else if component == "miri" || component == "miri-preview" {
&self.miri_git_commit_hash
} else {
&self.rust_git_commit_hash
use PkgType::*;
match PkgType::from_component(component) {
Cargo => &self.cargo_git_commit_hash,
Rls => &self.rls_git_commit_hash,
Clippy => &self.clippy_git_commit_hash,
Rustfmt => &self.rustfmt_git_commit_hash,
LlvmTools => &self.llvm_tools_git_commit_hash,
Lldb => &self.lldb_git_commit_hash,
Miri => &self.miri_git_commit_hash,
_ => &self.rust_git_commit_hash,
}
}

fn version(&self, component: &str, target: &str) -> Option<String> {
let mut cmd = Command::new("tar");
let filename = self.filename(component, target);
cmd.arg("xf")
.arg(self.input.join(&filename))
.arg(format!("{}/version", filename.replace(".tar.gz", "")))
.arg("-O");
let output = t!(cmd.output());
if output.status.success() {
Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
} else {
// Perhaps we didn't build this package.
None
}
self.untar(component, target, |filename| format!("{}/version", filename))
}

fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
self.untar(component, target, |filename| format!("{}/git-commit-hash", filename))
}

fn untar<F>(&self, component: &str, target: &str, dir: F) -> Option<String>
where
F: FnOnce(String) -> String
{
let mut cmd = Command::new("tar");
let filename = self.filename(component, target);
cmd.arg("xf")
.arg(self.input.join(&filename))
.arg(format!("{}/git-commit-hash", filename.replace(".tar.gz", "")))
.arg(dir(filename.replace(".tar.gz", "")))
.arg("-O");
let output = t!(cmd.output());
if output.status.success() {