diff --git a/.gitignore b/.gitignore index b241bef9d1e..3e010204655 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ perf.data perf.data.old *.events *.string* +/y.bin /build /build_sysroot/sysroot_src /build_sysroot/compiler-builtins diff --git a/.vscode/settings.json b/.vscode/settings.json index 9009a532c54..cec110792ce 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -49,6 +49,23 @@ "cfg": [], }, ] + }, + { + "roots": ["./y.rs"], + "crates": [ + { + "root_module": "./y.rs", + "edition": "2018", + "deps": [{ "crate": 1, "name": "std" }], + "cfg": [], + }, + { + "root_module": "./build_sysroot/sysroot_src/library/std/src/lib.rs", + "edition": "2018", + "deps": [], + "cfg": [], + }, + ] } ] } diff --git a/build.sh b/build.sh deleted file mode 100755 index 086b9617fdf..00000000000 --- a/build.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Settings -export CHANNEL="release" -build_sysroot="clif" -target_dir='build' -while [[ $# != 0 ]]; do - case $1 in - "--debug") - export CHANNEL="debug" - ;; - "--sysroot") - build_sysroot=$2 - shift - ;; - "--target-dir") - target_dir=$2 - shift - ;; - *) - echo "Unknown flag '$1'" - echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]" - exit 1 - ;; - esac - shift -done - -# Build cg_clif -unset CARGO_TARGET_DIR -unamestr=$(uname) -if [[ "$unamestr" == 'Linux' || "$unamestr" == "FreeBSD" ]]; then - export RUSTFLAGS='-Clink-arg=-Wl,-rpath=$ORIGIN/../lib '$RUSTFLAGS -elif [[ "$unamestr" == 'Darwin' ]]; then - export RUSTFLAGS='-Csplit-debuginfo=unpacked -Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS -else - echo "Unsupported os $unamestr" - exit 1 -fi -if [[ "$CHANNEL" == "release" ]]; then - cargo build --release -else - cargo build -fi - -source scripts/ext_config.sh - -rm -rf "$target_dir" -mkdir "$target_dir" -mkdir "$target_dir"/bin "$target_dir"/lib -ln target/$CHANNEL/cg_clif{,_build_sysroot} "$target_dir"/bin -ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib -ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir" - -mkdir -p "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/" -mkdir -p "$target_dir/lib/rustlib/$HOST_TRIPLE/lib/" -if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then - cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/" -fi - -case "$build_sysroot" in - "none") - ;; - "llvm") - cp -r $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib "$target_dir/lib/rustlib/$TARGET_TRIPLE/" - if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then - cp -r $(rustc --print sysroot)/lib/rustlib/$HOST_TRIPLE/lib "$target_dir/lib/rustlib/$HOST_TRIPLE/" - fi - ;; - "clif") - echo "[BUILD] sysroot" - dir=$(pwd) - cd "$target_dir" - time "$dir/build_sysroot/build_sysroot.sh" - if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then - time TARGET_TRIPLE="$HOST_TRIPLE" "$dir/build_sysroot/build_sysroot.sh" - fi - cp lib/rustlib/*/lib/libstd-* lib/ - ;; - *) - echo "Unknown sysroot kind \`$build_sysroot\`." - echo "The allowed values are:" - echo " none A sysroot that doesn't contain the standard library" - echo " llvm Copy the sysroot from rustc compiled by cg_llvm" - echo " clif Build a new sysroot using cg_clif" - exit 1 -esac diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs new file mode 100644 index 00000000000..cdddeae2a33 --- /dev/null +++ b/build_system/build_backend.rs @@ -0,0 +1,41 @@ +use std::env; +use std::process::{self, Command}; + +pub(crate) fn build_backend(channel: &str) -> String { + let mut cmd = Command::new("cargo"); + cmd.arg("build"); + + match channel { + "debug" => {} + "release" => { + cmd.arg("--release"); + } + _ => unreachable!(), + } + + if cfg!(unix) { + if cfg!(target_os = "macos") { + cmd.env( + "RUSTFLAGS", + "-Csplit-debuginfo=unpacked \ + -Clink-arg=-Wl,-rpath,@loader_path/../lib \ + -Zosx-rpath-install-name" + .to_string() + + env::var("RUSTFLAGS").as_deref().unwrap_or(""), + ); + } else { + cmd.env( + "RUSTFLAGS", + "-Clink-arg=-Wl,-rpath=$ORIGIN/../lib ".to_string() + + env::var("RUSTFLAGS").as_deref().unwrap_or(""), + ); + } + } + + eprintln!("[BUILD] rustc_codegen_cranelift"); + if !cmd.spawn().unwrap().wait().unwrap().success() { + process::exit(1); + } + + crate::rustc_info::get_dylib_name("rustc_codegen_cranelift") +} diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs new file mode 100644 index 00000000000..d6ac19dd1cc --- /dev/null +++ b/build_system/build_sysroot.rs @@ -0,0 +1,128 @@ +use crate::{try_hard_link, SysrootKind}; +use std::env; +use std::fs; +use std::path::Path; +use std::process::{self, Command}; + +pub(crate) fn build_sysroot( + channel: &str, + sysroot_kind: SysrootKind, + target_dir: &Path, + cg_clif_dylib: String, + host_triple: &str, + target_triple: &str, +) { + if target_dir.exists() { + fs::remove_dir_all(target_dir).unwrap(); + } + fs::create_dir_all(target_dir.join("bin")).unwrap(); + fs::create_dir_all(target_dir.join("lib")).unwrap(); + + // Copy the backend + for file in ["cg_clif", "cg_clif_build_sysroot"] { + try_hard_link( + Path::new("target").join(channel).join(file), + target_dir.join("bin").join(file), + ); + } + + try_hard_link( + Path::new("target").join(channel).join(&cg_clif_dylib), + target_dir.join("lib").join(cg_clif_dylib), + ); + + // Copy supporting files + try_hard_link("rust-toolchain", target_dir.join("rust-toolchain")); + try_hard_link("scripts/config.sh", target_dir.join("config.sh")); + try_hard_link("scripts/cargo.sh", target_dir.join("cargo.sh")); + + let default_sysroot = crate::rustc_info::get_default_sysroot(); + + let rustlib = target_dir.join("lib").join("rustlib"); + let host_rustlib_lib = rustlib.join(host_triple).join("lib"); + let target_rustlib_lib = rustlib.join(target_triple).join("lib"); + fs::create_dir_all(&host_rustlib_lib).unwrap(); + fs::create_dir_all(&target_rustlib_lib).unwrap(); + + if target_triple == "x86_64-pc-windows-gnu" { + if !default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib").exists() { + eprintln!( + "The x86_64-pc-windows-gnu target needs to be installed first before it is possible \ + to compile a sysroot for it.", + ); + process::exit(1); + } + for file in fs::read_dir( + default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"), + ) + .unwrap() + { + let file = file.unwrap().path(); + if file.extension().map_or(true, |ext| ext.to_str().unwrap() != "o") { + continue; // only copy object files + } + try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap())); + } + } + + match sysroot_kind { + SysrootKind::None => {} // Nothing to do + SysrootKind::Llvm => { + for file in fs::read_dir( + default_sysroot.join("lib").join("rustlib").join(host_triple).join("lib"), + ) + .unwrap() + { + let file = file.unwrap().path(); + let file_name_str = file.file_name().unwrap().to_str().unwrap(); + if file_name_str.contains("rustc_") + || file_name_str.contains("chalk") + || file_name_str.contains("tracing") + || file_name_str.contains("regex") + { + // These are large crates that are part of the rustc-dev component and are not + // necessary to run regular programs. + continue; + } + try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap())); + } + + if target_triple != host_triple { + for file in fs::read_dir( + default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"), + ) + .unwrap() + { + let file = file.unwrap().path(); + try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap())); + } + } + } + SysrootKind::Clif => { + let cwd = env::current_dir().unwrap(); + + let mut cmd = Command::new(cwd.join("build_sysroot").join("build_sysroot.sh")); + cmd.current_dir(target_dir).env("TARGET_TRIPLE", target_triple); + eprintln!("[BUILD] sysroot"); + if !cmd.spawn().unwrap().wait().unwrap().success() { + process::exit(1); + } + + if host_triple != target_triple { + let mut cmd = Command::new(cwd.join("build_sysroot").join("build_sysroot.sh")); + cmd.current_dir(target_dir).env("TARGET_TRIPLE", host_triple); + eprintln!("[BUILD] sysroot"); + if !cmd.spawn().unwrap().wait().unwrap().success() { + process::exit(1); + } + } + + for file in fs::read_dir(host_rustlib_lib).unwrap() { + let file = file.unwrap().path(); + if file.file_name().unwrap().to_str().unwrap().contains("std-") { + try_hard_link(&file, target_dir.join("lib").join(file.file_name().unwrap())); + } + } + } + } +} diff --git a/build_system/rustc_info.rs b/build_system/rustc_info.rs new file mode 100644 index 00000000000..28222743022 --- /dev/null +++ b/build_system/rustc_info.rs @@ -0,0 +1,41 @@ +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +pub(crate) fn get_host_triple() -> String { + let version_info = + Command::new("rustc").stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout; + String::from_utf8(version_info) + .unwrap() + .lines() + .to_owned() + .find(|line| line.starts_with("host")) + .unwrap() + .split(":") + .nth(1) + .unwrap() + .trim() + .to_owned() +} + +pub(crate) fn get_default_sysroot() -> PathBuf { + let default_sysroot = Command::new("rustc") + .stderr(Stdio::inherit()) + .args(&["--print", "sysroot"]) + .output() + .unwrap() + .stdout; + Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned() +} + +pub(crate) fn get_dylib_name(crate_name: &str) -> String { + let dylib_name = Command::new("rustc") + .stderr(Stdio::inherit()) + .args(&["--crate-name", crate_name, "--crate-type", "dylib", "--print", "file-names", "-"]) + .output() + .unwrap() + .stdout; + let dylib_name = String::from_utf8(dylib_name).unwrap().trim().to_owned(); + assert!(!dylib_name.contains('\n')); + assert!(dylib_name.contains(crate_name)); + dylib_name +} diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index b52ca6ea2ad..c494e78050a 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -./build.sh +./y.rs build source build/config.sh echo "[SETUP] Rust fork" diff --git a/test.sh b/test.sh index e222adc7b80..a10924628bb 100755 --- a/test.sh +++ b/test.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash set -e -./build.sh --sysroot none "$@" +./y.rs build --sysroot none "$@" rm -r target/out || true scripts/tests.sh no_sysroot -./build.sh "$@" +./y.rs build "$@" scripts/tests.sh base_sysroot scripts/tests.sh extended_sysroot diff --git a/y.rs b/y.rs new file mode 100755 index 00000000000..7971e713082 --- /dev/null +++ b/y.rs @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +#![allow()] /*This line is ignored by bash +# This block is ignored by rustc +set -e +echo "[BUILD] y.rs" 1>&2 +rustc $0 -o ${0/.rs/.bin} -g +exec ${0/.rs/.bin} $@ +*/ + +//! The build system for cg_clif +//! +//! # Manual compilation +//! +//! If your system doesn't support shell scripts you can manually compile and run this file using +//! for example: +//! +//! ```shell +//! $ rustc y.rs -o build/y.bin +//! $ build/y.bin +//! ``` +//! +//! # Naming +//! +//! The name `y.rs` was chosen to not conflict with rustc's `x.py`. + +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process; + +#[path = "build_system/build_backend.rs"] +mod build_backend; +#[path = "build_system/build_sysroot.rs"] +mod build_sysroot; +#[path = "build_system/rustc_info.rs"] +mod rustc_info; + +fn usage() { + eprintln!("Usage:"); + eprintln!(" ./y.rs build [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]"); +} + +macro_rules! arg_error { + ($($err:tt)*) => {{ + eprintln!($($err)*); + usage(); + std::process::exit(1); + }}; +} + +enum Command { + Build, +} + +#[derive(Copy, Clone)] +enum SysrootKind { + None, + Clif, + Llvm, +} + +fn main() { + env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1"); + env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1"); + + let mut args = env::args().skip(1); + let command = match args.next().as_deref() { + Some("prepare") => { + if args.next().is_some() { + arg_error!("./x.rs prepare doesn't expect arguments"); + } + todo!(); + } + Some("build") => Command::Build, + Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag), + Some(command) => arg_error!("Unknown command {}", command), + None => { + usage(); + process::exit(0); + } + }; + + let mut target_dir = PathBuf::from("build"); + let mut channel = "release"; + let mut sysroot_kind = SysrootKind::Clif; + while let Some(arg) = args.next().as_deref() { + match arg { + "--target-dir" => { + target_dir = PathBuf::from(args.next().unwrap_or_else(|| { + arg_error!("--target-dir requires argument"); + })) + } + "--debug" => channel = "debug", + "--sysroot" => { + sysroot_kind = match args.next().as_deref() { + Some("none") => SysrootKind::None, + Some("clif") => SysrootKind::Clif, + Some("llvm") => SysrootKind::Llvm, + Some(arg) => arg_error!("Unknown sysroot kind {}", arg), + None => arg_error!("--sysroot requires argument"), + } + } + flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag), + arg => arg_error!("Unexpected argument {}", arg), + } + } + + let host_triple = if let Ok(host_triple) = std::env::var("HOST_TRIPLE") { + host_triple + } else { + rustc_info::get_host_triple() + }; + let target_triple = if let Ok(target_triple) = std::env::var("TARGET_TRIPLE") { + if target_triple != "" { + target_triple + } else { + host_triple.clone() // Empty target triple can happen on GHA + } + } else { + host_triple.clone() + }; + + let cg_clif_dylib = build_backend::build_backend(channel); + build_sysroot::build_sysroot( + channel, + sysroot_kind, + &target_dir, + cg_clif_dylib, + &host_triple, + &target_triple, + ); +} + +#[track_caller] +fn try_hard_link(src: impl AsRef, dst: impl AsRef) { + let src = src.as_ref(); + let dst = dst.as_ref(); + if let Err(_) = fs::hard_link(src, dst) { + fs::copy(src, dst).unwrap(); // Fallback to copying if hardlinking failed + } +}