mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1374 from bjorn3/non_rustup_build3

Browse Source 
Allow building and testing without rustup
This commit is contained in:
bjorn3 2023-05-22 20:58:07 +02:00 committed by GitHub
commit a684753a68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 146 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build_system/abi_cafe.rs
View File

@ -16,10 +16,10 @@ pub(crate) fn run(
sysroot_kind: SysrootKind,
dirs: &Dirs,
cg_clif_dylib: &Path,
rustup_toolchain_name: Option<&str>,
bootstrap_host_compiler: &Compiler,
) {
ABI_CAFE_REPO.fetch(dirs);
spawn_and_wait(ABI_CAFE.fetch("cargo", &bootstrap_host_compiler.rustc, dirs));
eprintln!("Building sysroot for abi-cafe");
build_sysroot::build_sysroot(
@ -28,6 +28,7 @@ pub(crate) fn run(
sysroot_kind,
cg_clif_dylib,
bootstrap_host_compiler,
rustup_toolchain_name,
bootstrap_host_compiler.triple.clone(),
);

28
build_system/bench.rs
View File

@ -4,7 +4,7 @@
use super::path::{Dirs, RelPath};
use super::prepare::GitRepo;
use super::rustc_info::get_file_name;
use super::utils::{hyperfine_command, spawn_and_wait};
use super::utils::{hyperfine_command, spawn_and_wait, Compiler};
static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
"ebobby",
@ -13,11 +13,11 @@
"<none>",
);
pub(crate) fn benchmark(dirs: &Dirs) {
benchmark_simple_raytracer(dirs);
pub(crate) fn benchmark(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
benchmark_simple_raytracer(dirs, bootstrap_host_compiler);
}
fn benchmark_simple_raytracer(dirs: &Dirs) {
fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
if std::process::Command::new("hyperfine").output().is_err() {
eprintln!("Hyperfine not installed");
eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine");
@ -31,8 +31,9 @@ fn benchmark_simple_raytracer(dirs: &Dirs) {
let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap();
eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
let cargo_clif =
RelPath::DIST.to_path(dirs).join(get_file_name("cargo_clif", "bin").replace('_', "-"));
let cargo_clif = RelPath::DIST
.to_path(dirs)
.join(get_file_name(&bootstrap_host_compiler.rustc, "cargo_clif", "bin").replace('_', "-"));
let manifest_path = SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).join("Cargo.toml");
let target_dir = RelPath::BUILD.join("simple_raytracer").to_path(dirs);
@ -75,9 +76,18 @@ fn benchmark_simple_raytracer(dirs: &Dirs) {
bench_runs,
None,
&[
Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
Path::new(".").join(get_file_name("raytracer_cg_clif_opt", "bin")).to_str().unwrap(),
Path::new(".")
.join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_llvm", "bin"))
.to_str()
.unwrap(),
Path::new(".")
.join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif", "bin"))
.to_str()
.unwrap(),
Path::new(".")
.join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif_opt", "bin"))
.to_str()
.unwrap(),
],
);
bench_run.current_dir(RelPath::BUILD.to_path(dirs));

2
build_system/build_backend.rs
View File

@ -53,5 +53,5 @@ pub(crate) fn build_backend(
.target_dir(dirs)
.join(&bootstrap_host_compiler.triple)
.join(channel)
.join(get_file_name("rustc_codegen_cranelift", "dylib"))
.join(get_file_name(&bootstrap_host_compiler.rustc, "rustc_codegen_cranelift", "dylib"))
}

20
build_system/build_sysroot.rs
View File

@ -3,7 +3,7 @@
use std::process::{self, Command};
use super::path::{Dirs, RelPath};
use super::rustc_info::{get_file_name, get_rustc_version, get_toolchain_name};
use super::rustc_info::{get_file_name, get_rustc_version};
use super::utils::{remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler};
use super::SysrootKind;
@ -17,6 +17,7 @@ pub(crate) fn build_sysroot(
sysroot_kind: SysrootKind,
cg_clif_dylib_src: &Path,
bootstrap_host_compiler: &Compiler,
rustup_toolchain_name: Option<&str>,
target_triple: String,
) -> Compiler {
eprintln!("[BUILD] sysroot {:?}", sysroot_kind);
@ -40,19 +41,30 @@ pub(crate) fn build_sysroot(
try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path);
// Build and copy rustc and cargo wrappers
let wrapper_base_name = get_file_name("____", "bin");
let toolchain_name = get_toolchain_name();
let wrapper_base_name = get_file_name(&bootstrap_host_compiler.rustc, "____", "bin");
for wrapper in ["rustc-clif", "rustdoc-clif", "cargo-clif"] {
let wrapper_name = wrapper_base_name.replace("____", wrapper);
let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc);
let wrapper_path = DIST_DIR.to_path(dirs).join(&wrapper_name);
build_cargo_wrapper_cmd
.env("TOOLCHAIN_NAME", toolchain_name.clone())
.arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
.arg("-o")
.arg(&wrapper_path)
.arg("-Cstrip=debuginfo");
if let Some(rustup_toolchain_name) = &rustup_toolchain_name {
build_cargo_wrapper_cmd
.env("TOOLCHAIN_NAME", rustup_toolchain_name)
.env_remove("CARGO")
.env_remove("RUSTC")
.env_remove("RUSTDOC");
} else {
build_cargo_wrapper_cmd
.env_remove("TOOLCHAIN_NAME")
.env("CARGO", &bootstrap_host_compiler.cargo)
.env("RUSTC", &bootstrap_host_compiler.rustc)
.env("RUSTDOC", &bootstrap_host_compiler.rustdoc);
}
spawn_and_wait(build_cargo_wrapper_cmd);
try_hard_link(wrapper_path, BIN_DIR.to_path(dirs).join(wrapper_name));
}

47
build_system/mod.rs
View File

@ -78,6 +78,7 @@ pub(crate) fn main() {
let mut channel = "release";
let mut sysroot_kind = SysrootKind::Clif;
let mut use_unstable_features = true;
let mut frozen = false;
while let Some(arg) = args.next().as_deref() {
match arg {
"--out-dir" => {
@ -96,17 +97,38 @@ pub(crate) fn main() {
}
}
"--no-unstable-features" => use_unstable_features = false,
"--frozen" => frozen = true,
flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag),
arg => arg_error!("Unexpected argument {}", arg),
}
}
let bootstrap_host_compiler = Compiler::bootstrap_with_triple(
std::env::var("HOST_TRIPLE")
let rustup_toolchain_name = match (env::var("CARGO"), env::var("RUSTC"), env::var("RUSTDOC")) {
(Ok(_), Ok(_), Ok(_)) => None,
(Err(_), Err(_), Err(_)) => Some(rustc_info::get_toolchain_name()),
_ => {
eprintln!("All of CARGO, RUSTC and RUSTDOC need to be set or none must be set");
process::exit(1);
}
};
let bootstrap_host_compiler = {
let cargo = rustc_info::get_cargo_path();
let rustc = rustc_info::get_rustc_path();
let rustdoc = rustc_info::get_rustdoc_path();
let triple = std::env::var("HOST_TRIPLE")
.ok()
.or_else(|| config::get_value("host"))
.unwrap_or_else(|| rustc_info::get_host_triple()),
);
.unwrap_or_else(|| rustc_info::get_host_triple(&rustc));
Compiler {
cargo,
rustc,
rustdoc,
rustflags: String::new(),
rustdocflags: String::new(),
triple,
runner: vec![],
}
};
let target_triple = std::env::var("TARGET_TRIPLE")
.ok()
.or_else(|| config::get_value("target"))
@ -120,6 +142,7 @@ pub(crate) fn main() {
download_dir: out_dir.join("download"),
build_dir: out_dir.join("build"),
dist_dir: out_dir.join("dist"),
frozen,
};
path::RelPath::BUILD.ensure_exists(&dirs);
@ -134,7 +157,7 @@ pub(crate) fn main() {
}
if command == Command::Prepare {
prepare::prepare(&dirs);
prepare::prepare(&dirs, &bootstrap_host_compiler.rustc);
process::exit(0);
}
@ -158,6 +181,7 @@ pub(crate) fn main() {
sysroot_kind,
&cg_clif_dylib,
&bootstrap_host_compiler,
rustup_toolchain_name.as_deref(),
target_triple.clone(),
);
}
@ -166,7 +190,14 @@ pub(crate) fn main() {
eprintln!("Abi-cafe doesn't support cross-compilation");
process::exit(1);
}
abi_cafe::run(channel, sysroot_kind, &dirs, &cg_clif_dylib, &bootstrap_host_compiler);
abi_cafe::run(
channel,
sysroot_kind,
&dirs,
&cg_clif_dylib,
rustup_toolchain_name.as_deref(),
&bootstrap_host_compiler,
);
}
Command::Build => {
build_sysroot::build_sysroot(
@ -175,6 +206,7 @@ pub(crate) fn main() {
sysroot_kind,
&cg_clif_dylib,
&bootstrap_host_compiler,
rustup_toolchain_name.as_deref(),
target_triple,
);
}
@ -185,9 +217,10 @@ pub(crate) fn main() {
sysroot_kind,
&cg_clif_dylib,
&bootstrap_host_compiler,
rustup_toolchain_name.as_deref(),
target_triple,
);
bench::benchmark(&dirs);
bench::benchmark(&dirs, &bootstrap_host_compiler);
}
}
}

1
build_system/path.rs
View File

@ -9,6 +9,7 @@ pub(crate) struct Dirs {
pub(crate) download_dir: PathBuf,
pub(crate) build_dir: PathBuf,
pub(crate) dist_dir: PathBuf,
pub(crate) frozen: bool,
}
#[doc(hidden)]

24
build_system/prepare.rs
View File

@ -9,27 +9,19 @@
use super::tests::LIBCORE_TESTS_SRC;
use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait};
pub(crate) fn prepare(dirs: &Dirs) {
pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) {
RelPath::DOWNLOAD.ensure_fresh(dirs);
spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", "rustc", dirs));
prepare_stdlib(dirs);
spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", "rustc", dirs));
prepare_coretests(dirs);
spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", "rustc", dirs));
prepare_stdlib(dirs, rustc);
prepare_coretests(dirs, rustc);
super::tests::RAND_REPO.fetch(dirs);
spawn_and_wait(super::tests::RAND.fetch("cargo", "rustc", dirs));
super::tests::REGEX_REPO.fetch(dirs);
spawn_and_wait(super::tests::REGEX.fetch("cargo", "rustc", dirs));
super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", "rustc", dirs));
}
fn prepare_stdlib(dirs: &Dirs) {
let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
fn prepare_stdlib(dirs: &Dirs, rustc: &Path) {
let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust");
assert!(sysroot_src_orig.exists());
eprintln!("[COPY] stdlib src");
@ -44,7 +36,7 @@ fn prepare_stdlib(dirs: &Dirs) {
&SYSROOT_SRC.to_path(dirs).join("library"),
);
let rustc_version = get_rustc_version(Path::new("rustc"));
let rustc_version = get_rustc_version(rustc);
fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap();
eprintln!("[GIT] init");
@ -53,8 +45,8 @@ fn prepare_stdlib(dirs: &Dirs) {
apply_patches(dirs, "stdlib", &SYSROOT_SRC.to_path(dirs));
}
fn prepare_coretests(dirs: &Dirs) {
let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
fn prepare_coretests(dirs: &Dirs, rustc: &Path) {
let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust");
assert!(sysroot_src_orig.exists());
eprintln!("[COPY] coretests src");

18
build_system/rustc_info.rs
View File

@ -7,9 +7,9 @@ pub(crate) fn get_rustc_version(rustc: &Path) -> String {
String::from_utf8(version_info).unwrap()
}
pub(crate) fn get_host_triple() -> String {
pub(crate) fn get_host_triple(rustc: &Path) -> String {
let version_info =
Command::new("rustc").stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout;
Command::new(rustc).stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout;
String::from_utf8(version_info)
.unwrap()
.lines()
@ -34,6 +34,9 @@ pub(crate) fn get_toolchain_name() -> String {
}
pub(crate) fn get_cargo_path() -> PathBuf {
if let Ok(cargo) = std::env::var("CARGO") {
return PathBuf::from(cargo);
}
let cargo_path = Command::new("rustup")
.stderr(Stdio::inherit())
.args(&["which", "cargo"])
@ -44,6 +47,9 @@ pub(crate) fn get_cargo_path() -> PathBuf {
}
pub(crate) fn get_rustc_path() -> PathBuf {
if let Ok(rustc) = std::env::var("RUSTC") {
return PathBuf::from(rustc);
}
let rustc_path = Command::new("rustup")
.stderr(Stdio::inherit())
.args(&["which", "rustc"])
@ -54,6 +60,9 @@ pub(crate) fn get_rustc_path() -> PathBuf {
}
pub(crate) fn get_rustdoc_path() -> PathBuf {
if let Ok(rustdoc) = std::env::var("RUSTDOC") {
return PathBuf::from(rustdoc);
}
let rustc_path = Command::new("rustup")
.stderr(Stdio::inherit())
.args(&["which", "rustdoc"])
@ -73,8 +82,9 @@ pub(crate) fn get_default_sysroot(rustc: &Path) -> PathBuf {
Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned()
}
pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String {
let file_name = Command::new("rustc")
// FIXME call once for each target and pass result around in struct
pub(crate) fn get_file_name(rustc: &Path, crate_name: &str, crate_type: &str) -> String {
let file_name = Command::new(rustc)
.stderr(Stdio::inherit())
.args(&[
"--crate-name",

3
build_system/tests.rs
View File

@ -217,6 +217,7 @@ pub(crate) fn run_tests(
sysroot_kind: SysrootKind,
cg_clif_dylib: &Path,
bootstrap_host_compiler: &Compiler,
rustup_toolchain_name: Option<&str>,
target_triple: String,
) {
if config::get_bool("testsuite.no_sysroot") {
@ -226,6 +227,7 @@ pub(crate) fn run_tests(
SysrootKind::None,
cg_clif_dylib,
bootstrap_host_compiler,
rustup_toolchain_name,
target_triple.clone(),
);
@ -251,6 +253,7 @@ pub(crate) fn run_tests(
sysroot_kind,
cg_clif_dylib,
bootstrap_host_compiler,
rustup_toolchain_name,
target_triple.clone(),
);
// Rust's build system denies a couple of lints that trigger on several of the test

16
build_system/usage.txt
View File

@ -2,10 +2,10 @@ The build system of cg_clif.
USAGE:
./y.rs prepare [--out-dir DIR]
./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen]
./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen]
./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen]
./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen]
OPTIONS:
--debug
@ -26,9 +26,13 @@ OPTIONS:
Some features are not yet ready for production usage. This option will disable these
features. This includes the JIT mode and inline assembly support.
--frozen
Require Cargo.lock and cache are up to date
REQUIREMENTS:
* Rustup: The build system has a hard coded dependency on rustup to install the right nightly
version and make sure it is used where necessary.
* Rustup: By default rustup is used to install the right nightly version. If you don't want to
use rustup, you can manually install the nightly version indicated by rust-toolchain.toml and
point the CARGO, RUSTC and RUSTDOC env vars to the right executables.
* Git: `./y.rs prepare` uses git for applying patches and on Windows for downloading test repos.
* Curl and tar (non-Windows only): Used by `./y.rs prepare` to download a single commit for
repos. Git will be used to clone the whole repo when using Windows.

37
build_system/utils.rs
View File

@ -5,7 +5,6 @@
use std::process::{self, Command, Stdio};
use super::path::{Dirs, RelPath};
use super::rustc_info::{get_cargo_path, get_rustc_path, get_rustdoc_path};
#[derive(Clone, Debug)]
pub(crate) struct Compiler {
@ -19,18 +18,6 @@ pub(crate) struct Compiler {
}
impl Compiler {
pub(crate) fn bootstrap_with_triple(triple: String) -> Compiler {
Compiler {
cargo: get_cargo_path(),
rustc: get_rustc_path(),
rustdoc: get_rustdoc_path(),
rustflags: String::new(),
rustdocflags: String::new(),
triple,
runner: vec![],
}
}
pub(crate) fn set_cross_linker_and_runner(&mut self) {
match self.triple.as_str() {
"aarch64-unknown-linux-gnu" => {
@ -94,8 +81,11 @@ fn base_cmd(&self, command: &str, cargo: &Path, dirs: &Dirs) -> Command {
.arg("--manifest-path")
.arg(self.manifest_path(dirs))
.arg("--target-dir")
.arg(self.target_dir(dirs))
.arg("--frozen");
.arg(self.target_dir(dirs));
if dirs.frozen {
cmd.arg("--frozen");
}
cmd
}
@ -120,23 +110,6 @@ fn build_cmd(&self, command: &str, compiler: &Compiler, dirs: &Dirs) -> Command
cmd
}
#[must_use]
pub(crate) fn fetch(
&self,
cargo: impl AsRef<Path>,
rustc: impl AsRef<Path>,
dirs: &Dirs,
) -> Command {
let mut cmd = Command::new(cargo.as_ref());
cmd.env("RUSTC", rustc.as_ref())
.arg("fetch")
.arg("--manifest-path")
.arg(self.manifest_path(dirs));
cmd
}
pub(crate) fn clean(&self, dirs: &Dirs) {
let _ = fs::remove_dir_all(self.target_dir(dirs));
}

13
scripts/cargo-clif.rs
View File

@ -28,8 +28,13 @@ fn main() {
env::set_var("RUSTFLAGS", env::var("RUSTFLAGS").unwrap_or(String::new()) + &rustflags);
env::set_var("RUSTDOCFLAGS", env::var("RUSTDOCFLAGS").unwrap_or(String::new()) + &rustflags);
// Ensure that the right toolchain is used
env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
let cargo = if let Some(cargo) = option_env!("CARGO") {
cargo
} else {
// Ensure that the right toolchain is used
env::set_var("RUSTUP_TOOLCHAIN", option_env!("TOOLCHAIN_NAME").expect("TOOLCHAIN_NAME"));
"cargo"
};
let args: Vec<_> = match env::args().nth(1).as_deref() {
Some("jit") => {
@ -64,10 +69,10 @@ fn main() {
};
#[cfg(unix)]
panic!("Failed to spawn cargo: {}", Command::new("cargo").args(args).exec());
panic!("Failed to spawn cargo: {}", Command::new(cargo).args(args).exec());
#[cfg(not(unix))]
std::process::exit(
Command::new("cargo").args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
Command::new(cargo).args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
);
}

13
scripts/rustc-clif.rs
View File

@ -30,14 +30,19 @@ fn main() {
}
args.extend(passed_args);
// Ensure that the right toolchain is used
env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
let rustc = if let Some(rustc) = option_env!("RUSTC") {
rustc
} else {
// Ensure that the right toolchain is used
env::set_var("RUSTUP_TOOLCHAIN", option_env!("TOOLCHAIN_NAME").expect("TOOLCHAIN_NAME"));
"rustc"
};
#[cfg(unix)]
panic!("Failed to spawn rustc: {}", Command::new("rustc").args(args).exec());
panic!("Failed to spawn rustc: {}", Command::new(rustc).args(args).exec());
#[cfg(not(unix))]
std::process::exit(
Command::new("rustc").args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
Command::new(rustc).args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
);
}

13
scripts/rustdoc-clif.rs
View File

@ -30,14 +30,19 @@ fn main() {
}
args.extend(passed_args);
// Ensure that the right toolchain is used
env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
let rustdoc = if let Some(rustdoc) = option_env!("RUSTDOC") {
rustdoc
} else {
// Ensure that the right toolchain is used
env::set_var("RUSTUP_TOOLCHAIN", option_env!("TOOLCHAIN_NAME").expect("TOOLCHAIN_NAME"));
"rustdoc"
};
#[cfg(unix)]
panic!("Failed to spawn rustdoc: {}", Command::new("rustdoc").args(args).exec());
panic!("Failed to spawn rustdoc: {}", Command::new(rustdoc).args(args).exec());
#[cfg(not(unix))]
std::process::exit(
Command::new("rustdoc").args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
Command::new(rustdoc).args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
);
}