mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

load configuration for downloading artifacts from stage0.json

Browse Source
This commit is contained in:
Pietro Albini 2022-06-07 13:15:41 +02:00
parent c1a0f49e9f
commit 97f3ecda01
No known key found for this signature in database
GPG Key ID: CD76B35F7734769E
6 changed files with 62 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/bootstrap/builder.rs
View File

@ -870,16 +870,10 @@ impl<'a> Builder<'a> {
self.try_run(patchelf.arg(fname));
}
pub(crate) fn download_component(
&self,
base: &str,
url: &str,
dest_path: &Path,
help_on_error: &str,
) {
pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) {
// Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error);
self.download_with_retries(&tempfile, url, help_on_error);
t!(std::fs::rename(&tempfile, dest_path));
}

60
src/bootstrap/config.rs
View File

@ -20,7 +20,6 @@ use crate::channel::GitInfo;
pub use crate::flags::Subcommand;
use crate::flags::{Color, Flags};
use crate::util::{exe, output, program_out_of_date, t};
use crate::RustfmtMetadata;
use once_cell::sync::OnceCell;
use serde::{Deserialize, Deserializer};
@ -73,6 +72,7 @@ pub struct Config {
pub test_compare_mode: bool,
pub color: Color,
pub patch_binaries_for_nix: bool,
pub stage0_metadata: Stage0Metadata,
pub on_fail: Option<String>,
pub stage: u32,
@ -720,6 +720,25 @@ define_config! {
}
}
#[derive(Default, Deserialize)]
pub struct Stage0Metadata {
pub config: Stage0Config,
pub checksums_sha256: HashMap<String, String>,
pub rustfmt: Option<RustfmtMetadata>,
}
#[derive(Default, Deserialize)]
pub struct Stage0Config {
pub dist_server: String,
pub artifacts_server: String,
pub artifacts_with_llvm_assertions_server: String,
pub git_merge_commit_email: String,
}
#[derive(Default, Deserialize)]
pub struct RustfmtMetadata {
pub date: String,
pub version: String,
}
impl Config {
pub fn default_opts() -> Config {
let mut config = Config::default();
@ -776,6 +795,9 @@ impl Config {
config.llvm_profile_use = flags.llvm_profile_use;
config.llvm_profile_generate = flags.llvm_profile_generate;
let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json")));
config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
#[cfg(test)]
let get_toml = |_| TomlConfig::default();
#[cfg(not(test))]
@ -1103,8 +1125,11 @@ impl Config {
config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
config.download_rustc_commit =
download_ci_rustc_commit(rust.download_rustc, config.verbose > 0);
config.download_rustc_commit = download_ci_rustc_commit(
&config.stage0_metadata,
rust.download_rustc,
config.verbose > 0,
);
} else {
config.rust_profile_use = flags.rust_profile_use;
config.rust_profile_generate = flags.rust_profile_generate;
@ -1424,7 +1449,11 @@ fn threads_from_config(v: u32) -> u32 {
}
/// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) -> Option<String> {
fn download_ci_rustc_commit(
stage0_metadata: &Stage0Metadata,
download_rustc: Option<StringOrBool>,
verbose: bool,
) -> Option<String> {
// If `download-rustc` is not set, default to rebuilding.
let if_unchanged = match download_rustc {
None | Some(StringOrBool::Bool(false)) => return None,
@ -1443,13 +1472,12 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
// Look for a version to compare to based on the current commit.
// Only commits merged by bors will have CI artifacts.
let merge_base = output(Command::new("git").args(&[
"rev-list",
"--author=bors@rust-lang.org",
"-n1",
"--first-parent",
"HEAD",
]));
let merge_base = output(
Command::new("git")
.arg("rev-list")
.arg(format!("--author={}", stage0_metadata.config.git_merge_commit_email))
.args(&["-n1", "--first-parent", "HEAD"]),
);
let commit = merge_base.trim_end();
if commit.is_empty() {
println!("error: could not find commit hash for downloading rustc");
@ -1484,7 +1512,7 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
}
fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
let RustfmtMetadata { date, version } = builder.stage0_metadata.rustfmt.as_ref()?;
let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?;
let channel = format!("{version}-{date}");
let host = builder.config.build;
@ -1568,13 +1596,13 @@ fn download_component(
let tarball = cache_dir.join(&filename);
let (base_url, url, should_verify) = match mode {
DownloadSource::CI => (
"https://ci-artifacts.rust-lang.org/rustc-builds".to_string(),
builder.config.stage0_metadata.config.artifacts_server.clone(),
format!("{key}/{filename}"),
false,
),
DownloadSource::Dist => {
let dist_server = env::var("RUSTUP_DIST_SERVER")
.unwrap_or(builder.stage0_metadata.dist_server.to_string());
.unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string());
// NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
(dist_server, format!("dist/{key}/{filename}"), true)
}
@ -1590,7 +1618,7 @@ fn download_component(
target at this time, see https://doc.rust-lang.org/nightly\
/rustc/platform-support.html for more information."
);
let sha256 = builder.stage0_metadata.checksums_sha256.get(&url).expect(&error);
let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error);
if tarball.exists() {
if builder.verify(&tarball, sha256) {
builder.unpack(&tarball, &bin_root, prefix);
@ -1610,7 +1638,7 @@ fn download_component(
None
};
builder.download_component(&base_url, &url, &tarball, "");
builder.download_component(&format!("{base_url}/{url}"), &tarball, "");
if let Some(sha256) = checksum {
if !builder.verify(&tarball, sha256) {
panic!("failed to verify {}", tarball.display());

19
src/bootstrap/lib.rs
View File

@ -118,7 +118,6 @@ use std::os::windows::fs::symlink_file;
use filetime::FileTime;
use once_cell::sync::OnceCell;
use serde::Deserialize;
use crate::builder::Kind;
use crate::config::{LlvmLibunwind, TargetSelection};
@ -294,8 +293,6 @@ pub struct Build {
hosts: Vec<TargetSelection>,
targets: Vec<TargetSelection>,
// Stage 0 (downloaded) compiler, lld and cargo or their local rust equivalents
stage0_metadata: Stage0Metadata,
initial_rustc: PathBuf,
initial_cargo: PathBuf,
initial_lld: PathBuf,
@ -322,18 +319,6 @@ pub struct Build {
metrics: metrics::BuildMetrics,
}
#[derive(Deserialize)]
struct Stage0Metadata {
dist_server: String,
checksums_sha256: HashMap<String, String>,
rustfmt: Option<RustfmtMetadata>,
}
#[derive(Deserialize)]
struct RustfmtMetadata {
date: String,
version: String,
}
#[derive(Debug)]
struct Crate {
name: Interned<String>,
@ -482,11 +467,7 @@ impl Build {
bootstrap_out
};
let stage0_json = t!(std::fs::read_to_string(&src.join("src").join("stage0.json")));
let stage0_metadata = t!(serde_json::from_str::<Stage0Metadata>(&stage0_json));
let mut build = Build {
stage0_metadata,
initial_rustc: config.initial_rustc.clone(),
initial_cargo: config.initial_cargo.clone(),
initial_lld,

15
src/bootstrap/native.rs
View File

@ -121,7 +121,7 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
let mut rev_list = Command::new("git");
rev_list.args(&[
PathBuf::from("rev-list"),
"--author=bors@rust-lang.org".into(),
format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(),
"-n1".into(),
"--first-parent".into(),
"HEAD".into(),
@ -170,11 +170,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
if !rustc_cache.exists() {
t!(fs::create_dir_all(&rustc_cache));
}
let base = "https://ci-artifacts.rust-lang.org";
let url = if llvm_assertions {
format!("rustc-builds-alt/{}", llvm_sha)
let base = if llvm_assertions {
&builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server
} else {
format!("rustc-builds/{}", llvm_sha)
&builder.config.stage0_metadata.config.artifacts_server
};
let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
let tarball = rustc_cache.join(&filename);
@ -187,7 +186,11 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
\ndownload-ci-llvm = false
\n
";
builder.download_component(base, &format!("{}/{}", url, filename), &tarball, help_on_error);
builder.download_component(
&format!("{base}/{llvm_sha}/{filename}"),
&tarball,
help_on_error,
);
}
let llvm_root = builder.config.ci_llvm_root();
builder.unpack(&tarball, &llvm_root, "rust-dev");

5
src/stage0.json
View File

@ -1,7 +1,10 @@
{
"__comment": "Generated by `./x.py run src/tools/bump-stage0`. Run that command again to update the bootstrap compiler.",
"config": {
"dist_server": "https://static.rust-lang.org"
"dist_server": "https://static.rust-lang.org",
"artifacts_server": "https://ci-artifacts.rust-lang.org/rustc-builds",
"artifacts_with_llvm_assertions_server": "https://ci-artifacts.rust-lang.org/rustc-builds-alt",
"git_merge_commit_email": "bors@rust-lang.org"
},
"compiler": {
"date": "2022-05-20",

3
src/tools/bump-stage0/src/main.rs
View File

@ -183,6 +183,9 @@ struct Stage0 {
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct Config {
dist_server: String,
artifacts_server: String,
artifacts_with_llvm_assertions_server: String,
git_merge_commit_email: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]