mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(opt-dist): use rustc-perf from rustc checkout

Browse Source 
This replaces the hardcoded rustc-perf commit and ad-hoc downloading and
unpacking of its zipped source with defaulting to use the new rustc-perf
submodule.

While it would be nice to make `opt-dist` able to initialize the
submodule automatically when pointing to a Rust checkout _other_ than
the one opt-dist was built in, that would require a bigger refactor that
moved `update_submodule`, from bootstrap, into build_helper.

Regardless, I imagine it must be quite rare to use `opt-dist` with a
checkout that is neither from a rust-src tarball (which will contain the
submodule), nor the checkout opt-dist itself was built (bootstrap will
update the submodule when opt-dist is built).
This commit is contained in:
Bernardo Meurer Costa 2024-05-15 18:48:26 +00:00
parent 4c075c622c
commit 95c05c3e37
4 changed files with 14 additions and 435 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

390
Cargo.lock
View File

@ -825,16 +825,6 @@ dependencies = [
"rand_xorshift",
]
[[package]]
name = "core-foundation"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.6"
@ -1240,15 +1230,6 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "encoding_rs"
version = "0.8.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
dependencies = [
"cfg-if",
]
[[package]]
name = "env_filter"
version = "0.1.0"
@ -1399,21 +1380,6 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "form_urlencoded"
version = "1.2.1"
@ -1637,25 +1603,6 @@ dependencies = [
"serde",
]
[[package]]
name = "h2"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8"
dependencies = [
"bytes",
"fnv",
"futures-core",
"futures-sink",
"futures-util",
"http",
"indexmap",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "handlebars"
version = "5.1.2"
@ -1766,40 +1713,6 @@ dependencies = [
"syn 2.0.64",
]
[[package]]
name = "http"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]]
name = "http-body"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
dependencies = [
"bytes",
"http",
"pin-project-lite",
]
[[package]]
name = "httparse"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
[[package]]
name = "httpdate"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "humansize"
version = "2.1.3"
@ -1815,43 +1728,6 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "0.14.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80"
dependencies = [
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"socket2",
"tokio",
"tower-service",
"tracing",
"want",
]
[[package]]
name = "hyper-tls"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
"bytes",
"hyper",
"native-tls",
"tokio",
"tokio-native-tls",
]
[[package]]
name = "iana-time-zone"
version = "0.1.60"
@ -2095,12 +1971,6 @@ dependencies = [
"unic-langid",
]
[[package]]
name = "ipnet"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
[[package]]
name = "is_terminal_polyfill"
version = "1.70.0"
@ -2529,17 +2399,6 @@ dependencies = [
"rustc-std-workspace-core",
]
[[package]]
name = "mio"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
dependencies = [
"libc",
"wasi",
"windows-sys 0.48.0",
]
[[package]]
name = "miow"
version = "0.6.0"
@ -2577,24 +2436,6 @@ dependencies = [
name = "miropt-test-tools"
version = "0.1.0"
[[package]]
name = "native-tls"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
dependencies = [
"lazy_static",
"libc",
"log",
"openssl",
"openssl-probe",
"openssl-sys",
"schannel",
"security-framework",
"security-framework-sys",
"tempfile",
]
[[package]]
name = "new_debug_unreachable"
version = "1.0.6"
@ -2758,32 +2599,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "openssl"
version = "0.10.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
dependencies = [
"bitflags 2.5.0",
"cfg-if",
"foreign-types",
"libc",
"once_cell",
"openssl-macros",
"openssl-sys",
]
[[package]]
name = "openssl-macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.64",
]
[[package]]
name = "openssl-probe"
version = "0.1.5"
@ -2817,7 +2632,6 @@ dependencies = [
"humansize",
"humantime",
"log",
"reqwest",
"serde",
"serde_json",
"sysinfo",
@ -2825,7 +2639,6 @@ dependencies = [
"tar",
"tempfile",
"xz2",
"zip",
]
[[package]]
@ -3410,46 +3223,6 @@ dependencies = [
"walkdir",
]
[[package]]
name = "reqwest"
version = "0.11.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
dependencies = [
"base64",
"bytes",
"encoding_rs",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"hyper",
"hyper-tls",
"ipnet",
"js-sys",
"log",
"mime",
"native-tls",
"once_cell",
"percent-encoding",
"pin-project-lite",
"rustls-pemfile",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"system-configuration",
"tokio",
"tokio-native-tls",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"winreg",
]
[[package]]
name = "rls"
version = "2.0.0"
@ -5064,15 +4837,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rustls-pemfile"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [
"base64",
]
[[package]]
name = "rustversion"
version = "1.0.17"
@ -5137,29 +4901,6 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "security-framework"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0"
dependencies = [
"bitflags 2.5.0",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "self_cell"
version = "0.10.3"
@ -5225,18 +4966,6 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
dependencies = [
"form_urlencoded",
"itoa",
"ryu",
"serde",
]
[[package]]
name = "sha1"
version = "0.10.6"
@ -5516,12 +5245,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "synstructure"
version = "0.13.1"
@ -5556,27 +5279,6 @@ dependencies = [
"test",
]
[[package]]
name = "system-configuration"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"system-configuration-sys",
]
[[package]]
name = "system-configuration-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "tabled"
version = "0.15.0"
@ -5812,34 +5514,7 @@ checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787"
dependencies = [
"backtrace",
"bytes",
"libc",
"mio",
"pin-project-lite",
"socket2",
"windows-sys 0.48.0",
]
[[package]]
name = "tokio-native-tls"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
dependencies = [
"native-tls",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.7.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1"
dependencies = [
"bytes",
"futures-core",
"futures-sink",
"pin-project-lite",
"tokio",
]
[[package]]
@ -5891,12 +5566,6 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d"
[[package]]
name = "tower-service"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
[[package]]
name = "tracing"
version = "0.1.37"
@ -5982,12 +5651,6 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "try-lock"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "twox-hash"
version = "1.6.3"
@ -6314,15 +5977,6 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "want"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
dependencies = [
"try-lock",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
@ -6359,18 +6013,6 @@ dependencies = [
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
dependencies = [
"cfg-if",
"js-sys",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.92"
@ -6419,16 +6061,6 @@ dependencies = [
"semver",
]
[[package]]
name = "web-sys"
version = "0.3.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "winapi"
version = "0.3.9"
@ -6647,16 +6279,6 @@ dependencies = [
"memchr",
]
[[package]]
name = "winreg"
version = "0.50.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
dependencies = [
"cfg-if",
"windows-sys 0.48.0",
]
[[package]]
name = "writeable"
version = "0.5.4"
@ -6778,15 +6400,3 @@ dependencies = [
"quote",
"syn 2.0.64",
]
[[package]]
name = "zip"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261"
dependencies = [
"byteorder",
"crc32fast",
"crossbeam-utils",
"flate2",
]

2
src/tools/opt-dist/Cargo.toml
View File

@ -13,8 +13,6 @@ humansize = "2"
sysinfo = { version = "0.30", default-features = false }
fs_extra = "1"
camino = "1"
reqwest = { version = "0.11", features = ["blocking"] }
zip = { version = "0.6", default-features = false, features = ["deflate"] }
tar = "0.4"
xz = { version = "0.1", package = "xz2" }
serde = { version = "1", features = ["derive"] }

55
src/tools/opt-dist/src/main.rs
View File

@ -3,10 +3,7 @@ use anyhow::Context;
use camino::{Utf8Path, Utf8PathBuf};
use clap::Parser;
use log::LevelFilter;
use std::io::Cursor;
use std::time::Duration;
use utils::io;
use zip::ZipArchive;
use crate::environment::{Environment, EnvironmentBuilder};
use crate::exec::{cmd, Bootstrap};
@ -17,9 +14,9 @@ use crate::training::{
rustc_benchmarks,
};
use crate::utils::artifact_size::print_binary_sizes;
use crate::utils::io::{copy_directory, move_directory, reset_directory};
use crate::utils::io::{copy_directory, reset_directory};
use crate::utils::{
clear_llvm_files, format_env_variables, print_free_disk_space, retry_action, with_log_group,
clear_llvm_files, format_env_variables, print_free_disk_space, with_log_group,
write_timer_to_summary,
};
@ -69,7 +66,12 @@ enum EnvironmentCmd {
#[arg(long, default_value = "opt-artifacts")]
artifact_dir: Utf8PathBuf,
/// Checkout directory of `rustc-perf`, it will be fetched automatically if unspecified.
/// Checkout directory of `rustc-perf`.
///
/// If unspecified, defaults to the rustc-perf submodule in the rustc checkout dir
/// (`src/tools/rustc-perf`), which should have been initialized when building this tool.
// FIXME: Move update_submodule into build_helper, that way we can also ensure the submodule
// is updated when _running_ opt-dist, rather than building.
#[arg(long)]
rustc_perf_checkout_dir: Option<Utf8PathBuf>,
@ -146,8 +148,6 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)>
.host_llvm_dir(Utf8PathBuf::from("/rustroot"))
.artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts"))
.build_dir(checkout_dir.join("obj"))
// /tmp/rustc-perf comes from the x64 dist Dockerfile
.prebuilt_rustc_perf(Some(Utf8PathBuf::from("/tmp/rustc-perf")))
.shared_llvm(true)
.use_bolt(true)
.skipped_tests(vec![
@ -191,9 +191,12 @@ fn execute_pipeline(
) -> anyhow::Result<()> {
reset_directory(&env.artifact_dir())?;
with_log_group("Building rustc-perf", || match env.prebuilt_rustc_perf() {
Some(dir) => copy_rustc_perf(env, &dir),
None => download_rustc_perf(env),
with_log_group("Building rustc-perf", || {
let rustc_perf_checkout_dir = match env.prebuilt_rustc_perf() {
Some(dir) => dir,
None => env.checkout_path().join("src").join("tools").join("rustc-perf"),
};
copy_rustc_perf(env, &rustc_perf_checkout_dir)
})?;
// Stage 1: Build PGO instrumented rustc
@ -409,36 +412,6 @@ fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> {
build_rustc_perf(env)
}
// Download and build rustc-perf into the given environment.
fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> {
reset_directory(&env.rustc_perf_dir())?;
// FIXME: add some mechanism for synchronization of this commit SHA with
// Linux (which builds rustc-perf in a Dockerfile)
// rustc-perf version from 2023-10-22
const PERF_COMMIT: &str = "4f313add609f43e928e98132358e8426ed3969ae";
let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(60 * 2))
.connect_timeout(Duration::from_secs(60 * 2))
.build()?;
let response = retry_action(
|| Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
"Download rustc-perf archive",
5,
)?;
let mut archive = ZipArchive::new(Cursor::new(response))?;
archive.extract(env.rustc_perf_dir())?;
move_directory(
&env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
&env.rustc_perf_dir(),
)?;
build_rustc_perf(env)
}
fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> {
cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"])
.workdir(&env.rustc_perf_dir())

2
src/tools/tidy/src/deps.rs
View File

@ -81,12 +81,10 @@ const EXCEPTIONS: ExceptionList = &[
("ar_archive_writer", "Apache-2.0 WITH LLVM-exception"), // rustc
("colored", "MPL-2.0"), // rustfmt
("dissimilar", "Apache-2.0"), // rustdoc, rustc_lexer (few tests) via expect-test, (dev deps)
("encoding_rs", "(Apache-2.0 OR MIT) AND BSD-3-Clause"), // opt-dist
("fluent-langneg", "Apache-2.0"), // rustc (fluent translations)
("fortanix-sgx-abi", "MPL-2.0"), // libstd but only for `sgx` target. FIXME: this dependency violates the documentation comment above.
("instant", "BSD-3-Clause"), // rustc_driver/tracing-subscriber/parking_lot
("mdbook", "MPL-2.0"), // mdbook
("openssl", "Apache-2.0"), // opt-dist
("option-ext", "MPL-2.0"), // cargo-miri (via `directories`)
("rustc_apfloat", "Apache-2.0 WITH LLVM-exception"), // rustc (license is the same as LLVM uses)
("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0 // cargo/... (because of serde)