Merge pull request #319 from GuillaumeGomez/build-system

Rustify prepare.sh command

.github/workflows/ci.yml

@@ -119,7 +119,7 @@ jobs:
 
     - name: Build
       run: |
-        ./prepare_build.sh
+        ./y.sh prepare --only-libcore
         ${{ matrix.libgccjit_version.env_extra }} ./build.sh ${{ matrix.libgccjit_version.extra }}
         ${{ matrix.libgccjit_version.env_extra }} cargo test ${{ matrix.libgccjit_version.extra }}
         ./clean_all.sh
@@ -128,7 +128,7 @@ jobs:
       run: |
         git config --global user.email "user@example.com"
         git config --global user.name "User"
-        ./prepare.sh
+        ./y.sh prepare
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile

.github/workflows/release.yml

@@ -88,7 +88,7 @@ jobs:
 
     - name: Build
       run: |
-        ./prepare_build.sh
+        ./y.sh prepare --only-libcore
         ./build.sh --release --release-sysroot
         cargo test
         ./clean_all.sh
@@ -97,7 +97,7 @@ jobs:
       run: |
         git config --global user.email "user@example.com"
         git config --global user.name "User"
-        ./prepare.sh
+        ./y.sh prepare
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile

.github/workflows/stdarch.yml

@@ -102,7 +102,7 @@ jobs:
 
     - name: Build
       run: |
-        ./prepare_build.sh
+        ./y.sh prepare --only-libcore
         ./build.sh --release --release-sysroot
         cargo test
 
@@ -115,7 +115,7 @@ jobs:
       run: |
         git config --global user.email "user@example.com"
         git config --global user.name "User"
-        ./prepare.sh
+        ./y.sh prepare
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile

.gitignore

@@ -25,3 +25,4 @@ tools/llvmint
 tools/llvmint-2
 # The `llvm` folder is generated by the `tools/generate_intrinsics.py` script to update intrinsics.
 llvm
+build_system/target

Readme.md

@@ -65,7 +65,7 @@ $ export RUST_COMPILER_RT_ROOT="$PWD/llvm/compiler-rt"
 Then you can run commands like this:
 
 ```bash
-$ ./prepare.sh # download and patch sysroot src and install hyperfine for benchmarking
+$ ./y.sh prepare # download and patch sysroot src and install hyperfine for benchmarking
 $ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) ./build.sh --release
 ```
 

build_sysroot/prepare_sysroot_src.sh

@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-set -e
-cd $(dirname "$0")
-
-SRC_DIR=$(dirname $(rustup which rustc))"/../lib/rustlib/src/rust/"
-DST_DIR="sysroot_src"
-
-if [ ! -e $SRC_DIR ]; then
-    echo "Please install rust-src component"
-    exit 1
-fi
-
-rm -rf $DST_DIR
-mkdir -p $DST_DIR/library
-cp -r $SRC_DIR/library $DST_DIR/
-
-pushd $DST_DIR
-echo "[GIT] init"
-git init
-echo "[GIT] add"
-git add .
-echo "[GIT] commit"
-
-# This is needed on systems where nothing is configured.
-# git really needs something here, or it will fail.
-# Even using --author is not enough.
-git config user.email || git config user.email "none@example.com"
-git config user.name || git config user.name "None"
-
-git commit -m "Initial commit" -q
-for file in $(ls ../../patches/ | grep -v patcha); do
-    echo "[GIT] apply" $file
-    git apply ../../patches/$file
-    git add -A
-    git commit --no-gpg-sign -m "Patch $file"
-done
-popd
-
-echo "Successfully prepared libcore for building"

build_system/Cargo.lock

@@ -0,0 +1,7 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "y"
+version = "0.1.0"

build_system/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "y"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "y"
+path = "src/main.rs"

build_system/src/build.rs

@@ -0,0 +1,3 @@
+pub fn run() -> Result<(), String> {
+    Ok(())
+}

build_system/src/main.rs

@@ -0,0 +1,49 @@
+use std::env;
+use std::process;
+
+mod build;
+mod prepare;
+mod rustc_info;
+mod utils;
+
+macro_rules! arg_error {
+    ($($err:tt)*) => {{
+        eprintln!($($err)*);
+        usage();
+        std::process::exit(1);
+    }};
+}
+
+fn usage() {
+    // println!("{}", include_str!("usage.txt"));
+}
+
+pub enum Command {
+    Prepare,
+    Build,
+}
+
+fn main() {
+    if env::var("RUST_BACKTRACE").is_err() {
+        env::set_var("RUST_BACKTRACE", "1");
+    }
+
+    let command = match env::args().nth(1).as_deref() {
+        Some("prepare") => Command::Prepare,
+        Some("build") => Command::Build,
+        Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
+        Some(command) => arg_error!("Unknown command {}", command),
+        None => {
+            usage();
+            process::exit(0);
+        }
+    };
+
+    if let Err(e) = match command {
+        Command::Prepare => prepare::run(),
+        Command::Build => build::run(),
+    } {
+        eprintln!("Command failed to run: {e:?}");
+        process::exit(1);
+    }
+}

build_system/src/prepare.rs

@@ -0,0 +1,175 @@
+use crate::rustc_info::get_rustc_path;
+use crate::utils::{cargo_install, git_clone, run_command, run_command_with_output, walk_dir};
+
+use std::fs;
+use std::path::Path;
+
+fn prepare_libcore(sysroot_path: &Path) -> Result<(), String> {
+    let rustc_path = match get_rustc_path() {
+        Some(path) => path,
+        None => return Err("`rustc` path not found".to_owned()),
+    };
+
+    let parent = match rustc_path.parent() {
+        Some(path) => path,
+        None => return Err(format!("No parent for `{}`", rustc_path.display())),
+    };
+
+    let rustlib_dir =
+        parent
+            .join("../lib/rustlib/src/rust")
+            .canonicalize()
+            .map_err(|e| format!("Failed to canonicalize path: {e:?}"))?;
+    if !rustlib_dir.is_dir() {
+        return Err("Please install `rust-src` component".to_owned());
+    }
+
+    let sysroot_dir = sysroot_path.join("sysroot_src");
+    if sysroot_dir.is_dir() {
+        if let Err(e) = fs::remove_dir_all(&sysroot_dir) {
+            return Err(format!("Failed to remove `{}`: {:?}", sysroot_dir.display(), e));
+        }
+    }
+
+    let sysroot_library_dir = sysroot_dir.join("library");
+    fs::create_dir_all(&sysroot_library_dir)
+        .map_err(|e| format!(
+            "Failed to create folder `{}`: {e:?}",
+            sysroot_library_dir.display(),
+        ))?;
+
+    run_command(&[&"cp", &"-r", &rustlib_dir.join("library"), &sysroot_dir], None)?;
+
+    println!("[GIT] init (cwd): `{}`", sysroot_dir.display());
+    run_command(&[&"git", &"init"], Some(&sysroot_dir))?;
+    println!("[GIT] add (cwd): `{}`", sysroot_dir.display());
+    run_command(&[&"git", &"add", &"."], Some(&sysroot_dir))?;
+    println!("[GIT] commit (cwd): `{}`", sysroot_dir.display());
+
+    // This is needed on systems where nothing is configured.
+    // git really needs something here, or it will fail.
+    // Even using --author is not enough.
+    run_command(&[&"git", &"config", &"user.email", &"none@example.com"], Some(&sysroot_dir))?;
+    run_command(&[&"git", &"config", &"user.name", &"None"], Some(&sysroot_dir))?;
+    run_command(&[&"git", &"config", &"core.autocrlf", &"false"], Some(&sysroot_dir))?;
+    run_command(&[&"git", &"config", &"commit.gpgSign", &"false"], Some(&sysroot_dir))?;
+    run_command(&[&"git", &"commit", &"-m", &"Initial commit", &"-q"], Some(&sysroot_dir))?;
+
+    let mut patches = Vec::new();
+    walk_dir("patches", |_| Ok(()), |file_path: &Path| {
+        patches.push(file_path.to_path_buf());
+        Ok(())
+    })?;
+    patches.sort();
+    for file_path in patches {
+        println!("[GIT] apply `{}`", file_path.display());
+        let path = Path::new("../..").join(file_path);
+        run_command_with_output(&[&"git", &"apply", &path], Some(&sysroot_dir))?;
+        run_command_with_output(&[&"git", &"add", &"-A"], Some(&sysroot_dir))?;
+        run_command_with_output(
+            &[&"git", &"commit", &"--no-gpg-sign", &"-m", &format!("Patch {}", path.display())],
+            Some(&sysroot_dir),
+        )?;
+    }
+    println!("Successfully prepared libcore for building");
+    Ok(())
+}
+
+// build with cg_llvm for perf comparison
+fn build_raytracer(repo_dir: &Path) -> Result<(), String> {
+    run_command(&[&"cargo", &"build"], Some(repo_dir))?;
+    let mv_target = repo_dir.join("raytracer_cg_llvm");
+    if mv_target.is_file() {
+        std::fs::remove_file(&mv_target)
+            .map_err(|e| format!("Failed to remove file `{}`: {e:?}", mv_target.display()))?;
+    }
+    run_command(&[&"mv", &"target/debug/main", &"raytracer_cg_llvm"], Some(repo_dir))?;
+    Ok(())
+}
+
+fn clone_and_setup<F>(repo_url: &str, checkout_commit: &str, extra: Option<F>) -> Result<(), String>
+where
+    F: Fn(&Path) -> Result<(), String>,
+{
+    let clone_result = git_clone(repo_url, None)?;
+    if !clone_result.ran_clone {
+        println!("`{}` has already been cloned", clone_result.repo_name);
+    }
+    let repo_path = Path::new(&clone_result.repo_name);
+    run_command(&[&"git", &"checkout", &"--", &"."], Some(&repo_path))?;
+    run_command(&[&"git", &"checkout", &checkout_commit], Some(&repo_path))?;
+    let filter = format!("-{}-", clone_result.repo_name);
+    walk_dir("crate_patches", |_| Ok(()), |file_path| {
+        let s = file_path.as_os_str().to_str().unwrap();
+        if s.contains(&filter) && s.ends_with(".patch") {
+            run_command_with_output(
+                &[&"git", &"am", &file_path.canonicalize().unwrap()],
+                Some(&repo_path),
+            )?;
+        }
+        Ok(())
+    })?;
+    if let Some(extra) = extra {
+        extra(&repo_path)?;
+    }
+    Ok(())
+}
+
+struct PrepareArg {
+    only_libcore: bool,
+}
+
+impl PrepareArg {
+    fn new() -> Result<Option<Self>, String> {
+        let mut only_libcore = false;
+
+        for arg in std::env::args().skip(2) {
+            match arg.as_str() {
+                "--only-libcore" => only_libcore = true,
+                "--help" => {
+                    Self::usage();
+                    return Ok(None)
+                }
+                a => return Err(format!("Unknown argument `{a}`")),
+            }
+        }
+        Ok(Some(Self {
+            only_libcore,
+        }))
+    }
+
+    fn usage() {
+        println!(r#"
+`prepare` command help:
+
+    --only-libcore  : Only setup libcore and don't clone other repositories
+    --help          : Show this help
+"#)
+    }
+}
+
+pub fn run() -> Result<(), String> {
+    let args = match PrepareArg::new()? {
+        Some(a) => a,
+        None => return Ok(()),
+    };
+    let sysroot_path = Path::new("build_sysroot");
+    prepare_libcore(sysroot_path)?;
+
+    if !args.only_libcore {
+        cargo_install("hyperfine")?;
+
+        let to_clone = &[
+            ("https://github.com/rust-random/rand.git", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", None),
+            ("https://github.com/rust-lang/regex.git", "341f207c1071f7290e3f228c710817c280c8dca1", None),
+            ("https://github.com/ebobby/simple-raytracer", "804a7a21b9e673a482797aa289a18ed480e4d813", Some(build_raytracer)),
+        ];
+
+        for (repo_url, checkout_commit, cb) in to_clone {
+            clone_and_setup(repo_url, checkout_commit, *cb)?;
+        }
+    }
+
+    println!("Successfully ran `prepare`");
+    Ok(())
+}

build_system/src/rustc_info.rs

@@ -0,0 +1,12 @@
+use std::path::{Path, PathBuf};
+
+use crate::utils::run_command;
+
+pub fn get_rustc_path() -> Option<PathBuf> {
+    if let Ok(rustc) = std::env::var("RUSTC") {
+        return Some(PathBuf::from(rustc));
+    }
+    run_command(&[&"rustup", &"which", &"rustc"], None)
+        .ok()
+        .map(|out| Path::new(String::from_utf8(out.stdout).unwrap().trim()).to_owned())
+}

build_system/src/utils.rs

@@ -0,0 +1,143 @@
+use std::ffi::OsStr;
+use std::fmt::Debug;
+use std::fs;
+use std::path::Path;
+use std::process::{Command, ExitStatus, Output};
+
+fn get_command_inner(input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>) -> Command {
+    let (cmd, args) = match input {
+        [] => panic!("empty command"),
+        [cmd, args @ ..] => (cmd, args),
+    };
+    let mut command = Command::new(cmd);
+    command.args(args);
+    if let Some(cwd) = cwd {
+        command.current_dir(cwd);
+    }
+    command
+}
+
+fn check_exit_status(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    exit_status: ExitStatus,
+) -> Result<(), String> {
+    if exit_status.success() {
+        Ok(())
+    } else {
+        Err(format!(
+            "Command `{}`{} exited with status {:?}",
+            input.iter()
+                .map(|s| s.as_ref().to_str().unwrap())
+                .collect::<Vec<_>>()
+                .join(" "),
+            cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display()))
+                .unwrap_or_default(),
+            exit_status.code(),
+        ))
+    }
+}
+
+fn command_error<D: Debug>(input: &[&dyn AsRef<OsStr>], cwd: &Option<&Path>, error: D) -> String {
+    format!(
+        "Command `{}`{} failed to run: {error:?}",
+        input.iter()
+            .map(|s| s.as_ref().to_str().unwrap())
+            .collect::<Vec<_>>()
+            .join(" "),
+        cwd.as_ref()
+            .map(|cwd| format!(
+                " (running in folder `{}`)",
+                cwd.display(),
+            ))
+            .unwrap_or_default(),
+    )
+}
+
+pub fn run_command(input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>) -> Result<Output, String> {
+    let output = get_command_inner(input, cwd)
+        .output()
+        .map_err(|e| command_error(input, &cwd, e))?;
+    check_exit_status(input, cwd, output.status)?;
+    Ok(output)
+}
+
+pub fn run_command_with_output(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+) -> Result<(), String> {
+    let exit_status = get_command_inner(input, cwd).spawn()
+        .map_err(|e| command_error(input, &cwd, e))?
+        .wait()
+        .map_err(|e| command_error(input, &cwd, e))?;
+    check_exit_status(input, cwd, exit_status)?;
+    Ok(())
+}
+
+pub fn cargo_install(to_install: &str) -> Result<(), String> {
+    let output = run_command(&[&"cargo", &"install", &"--list"], None)?;
+
+    let to_install_needle = format!("{to_install} ");
+    // cargo install --list returns something like this:
+    //
+    // mdbook-toc v0.8.0:
+    //     mdbook-toc
+    // rust-reduce v0.1.0:
+    //     rust-reduce
+    //
+    // We are only interested into the command name so we only look for lines ending with `:`.
+    if String::from_utf8(output.stdout)
+        .unwrap()
+        .lines()
+        .any(|line| line.ends_with(':') && line.starts_with(&to_install_needle))
+    {
+        return Ok(());
+    }
+    // We voluntarily ignore this error.
+    if run_command_with_output(&[&"cargo", &"install", &to_install], None).is_err() {
+        println!("Skipping installation of `{to_install}`");
+    }
+    Ok(())
+}
+
+pub struct CloneResult {
+    pub ran_clone: bool,
+    pub repo_name: String,
+}
+
+pub fn git_clone(to_clone: &str, dest: Option<&Path>) -> Result<CloneResult, String> {
+    let repo_name = to_clone.split('/').last().unwrap();
+    let repo_name = match repo_name.strip_suffix(".git") {
+        Some(n) => n.to_owned(),
+        None => repo_name.to_owned(),
+    };
+
+    let dest = dest
+        .map(|dest| dest.join(&repo_name))
+        .unwrap_or_else(|| Path::new(&repo_name).into());
+    if dest.is_dir() {
+        return Ok(CloneResult { ran_clone: false, repo_name });
+    }
+
+    run_command_with_output(&[&"git", &"clone", &to_clone, &dest], None)?;
+    Ok(CloneResult { ran_clone: true, repo_name })
+}
+
+pub fn walk_dir<P, D, F>(dir: P, mut dir_cb: D, mut file_cb: F) -> Result<(), String>
+where
+    P: AsRef<Path>,
+    D: FnMut(&Path) -> Result<(), String>,
+    F: FnMut(&Path) -> Result<(), String>,
+{
+    let dir = dir.as_ref();
+    for entry in fs::read_dir(dir).map_err(|e| format!("Failed to read dir `{}`: {e:?}", dir.display()))? {
+        let entry = entry.map_err(|e| format!("Failed to read entry in `{}`: {e:?}", dir.display()))?;
+        let entry_path = entry.path();
+        if entry_path.is_dir() {
+            dir_cb(&entry_path)?;
+        } else {
+            file_cb(&entry_path)?;
+        }
+    }
+    Ok(())
+}

prepare.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-set -e
-set -v
-
-source prepare_build.sh
-
-cargo install hyperfine || echo "Skipping hyperfine install"
-
-git clone https://github.com/rust-random/rand.git || echo "rust-random/rand has already been cloned"
-pushd rand
-git checkout -- .
-git checkout 0f933f9c7176e53b2a3c7952ded484e1783f0bf1
-git am ../crate_patches/*-rand-*.patch
-popd
-
-git clone https://github.com/rust-lang/regex.git || echo "rust-lang/regex has already been cloned"
-pushd regex
-git checkout -- .
-git checkout 341f207c1071f7290e3f228c710817c280c8dca1
-popd
-
-git clone https://github.com/ebobby/simple-raytracer || echo "ebobby/simple-raytracer has already been cloned"
-pushd simple-raytracer
-git checkout -- .
-git checkout 804a7a21b9e673a482797aa289a18ed480e4d813
-
-# build with cg_llvm for perf comparison
-cargo build
-mv target/debug/main raytracer_cg_llvm
-popd

prepare_build.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -e
-set -v
-
-./build_sysroot/prepare_sysroot_src.sh

rustup.sh

@@ -16,7 +16,7 @@ case $1 in
         done
 
         ./clean_all.sh
-        ./prepare.sh
+        ./y.sh prepare
         ;;
     "commit")
         git add rust-toolchain

y.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+echo "[BUILD] build system" 1>&2
+mkdir -p build_system/target
+rustc build_system/src/main.rs -o build_system/target/y -Cdebuginfo=1 --edition 2021
+exec ./build_system/target/y "$@"