diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 841e1a0870e..e6d3375fb1b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,4 +51,13 @@ jobs: export COMPILE_RUNS=2 export RUN_RUNS=2 - ./test.sh --release + ./test.sh + + - name: Package prebuilt cg_clif + run: tar cvfJ cg_clif.tar.xz build + + - name: Upload prebuilt cg_clif + uses: actions/upload-artifact@v2 + with: + name: cg_clif-${{ runner.os }} + path: cg_clif.tar.xz diff --git a/.gitignore b/.gitignore index 0da9927b479..18196bce009 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ perf.data perf.data.old *.events *.string* -/build_sysroot/sysroot +/build /build_sysroot/sysroot_src /rust /rand diff --git a/Readme.md b/Readme.md index ca380703a7f..f8a5e13ed54 100644 --- a/Readme.md +++ b/Readme.md @@ -2,7 +2,10 @@ > ⚠⚠⚠ Certain kinds of FFI don't work yet. ⚠⚠⚠ -The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift). This has the potential to improve compilation times in debug mode. If your project doesn't use any of the things listed under "Not yet supported", it should work fine. If not please open an issue. +The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift). +This has the potential to improve compilation times in debug mode. +If your project doesn't use any of the things listed under "Not yet supported", it should work fine. +If not please open an issue. ## Building and testing @@ -10,40 +13,45 @@ The goal of this project is to create an alternative codegen backend for the rus $ git clone https://github.com/bjorn3/rustc_codegen_cranelift.git $ cd rustc_codegen_cranelift $ ./prepare.sh # download and patch sysroot src and install hyperfine for benchmarking -$ ./test.sh --release +$ ./build.sh ``` -If you want to only build but not test you should replace the last command with: +To run the test suite replace the last command with: ```bash -$ cargo build --release -$ ./build_sysroot/build_sysroot.sh +$ ./test.sh ``` +This will implicitly build cg_clif too. Both `build.sh` and `test.sh` accept a `--debug` argument to +build in debug mode. + +Alternatively you can download a pre built version from [GHA]. It is listed in the artifacts section +of workflow runs. Unfortunately due to GHA restrictions you need to be logged in to access it. + +[GHA]: https://github.com/bjorn3/rustc_codegen_cranelift/actions?query=branch%3Amaster+event%3Apush+is%3Asuccess + ## Usage rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. -Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `test.sh`). +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`). ### Cargo In the directory with your project (where you can do the usual `cargo build`), run: ```bash -$ $cg_clif_dir/cargo.sh run +$ $cg_clif_dir/build/cargo.sh run ``` This should build and run your project with rustc_codegen_cranelift instead of the usual LLVM backend. -If you compiled cg_clif in debug mode (aka you didn't pass `--release` to `./test.sh`) you should set `CHANNEL="debug"`. - ### Rustc > You should prefer using the Cargo method. ```bash -$ $cg_clif_dir/target/release/cg_clif my_crate.rs +$ $cg_clif_dir/build/cg_clif my_crate.rs ``` ### Jit mode @@ -54,13 +62,13 @@ In jit mode cg_clif will immediately execute your code without creating an execu > The jit mode will probably need cargo integration to make this possible. ```bash -$ $cg_clif_dir/cargo.sh jit +$ $cg_clif_dir/build/cargo.sh jit ``` or ```bash -$ $cg_clif_dir/target/release/cg_clif --jit my_crate.rs +$ $cg_clif_dir/build/cg_clif --jit my_crate.rs ``` ### Shell @@ -69,7 +77,7 @@ These are a few functions that allow you to easily run rust code from the shell ```bash function jit_naked() { - echo "$@" | $cg_clif_dir/target/release/cg_clif - --jit + echo "$@" | $cg_clif_dir/build/cg_clif - --jit } function jit() { diff --git a/build.sh b/build.sh new file mode 100755 index 00000000000..f9a87e68a04 --- /dev/null +++ b/build.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + +# Settings +export CHANNEL="release" +build_sysroot=1 +target_dir='build' +while [[ $# != 0 ]]; do + case $1 in + "--debug") + export CHANNEL="debug" + ;; + "--without-sysroot") + build_sysroot=0 + ;; + "--target-dir") + target_dir=$2 + shift + ;; + *) + echo "Unknown flag '$1'" + echo "Usage: ./build.sh [--debug] [--without-sysroot] [--target-dir DIR]" + ;; + esac + shift +done + +# Build cg_clif +export RUSTFLAGS="-Zrun_dsymutil=no" +if [[ "$CHANNEL" == "release" ]]; then + cargo build --release +else + cargo build +fi + +rm -rf $target_dir +mkdir $target_dir +cp -a target/$CHANNEL/cg_clif{,_build_sysroot} target/$CHANNEL/*rustc_codegen_cranelift* $target_dir/ +cp -a rust-toolchain scripts/config.sh scripts/cargo.sh $target_dir + +if [[ "$build_sysroot" == "1" ]]; then + echo "[BUILD] sysroot" + export CG_CLIF_INCR_CACHE_DISABLED=1 + dir=$(pwd) + cd $target_dir + time $dir/build_sysroot/build_sysroot.sh +fi diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh index 7557f74b286..1d87562a6a4 100755 --- a/build_sysroot/build_sysroot.sh +++ b/build_sysroot/build_sysroot.sh @@ -3,29 +3,25 @@ # Requires the CHANNEL env var to be set to `debug` or `release.` set -e -cd $(dirname "$0") -if [ -z $CHANNEL ]; then -export CHANNEL='release' -fi +source ./config.sh -pushd ../ >/dev/null -source ./scripts/config.sh -popd >/dev/null - -# We expect the target dir in the default location. Guard against the user changing it. -export CARGO_TARGET_DIR=target - -# Cleanup for previous run -# v Clean target dir except for build scripts and incremental cache -rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/null || true -rm -r sysroot/ 2>/dev/null || true +dir=$(pwd) # Use rustc with cg_clif as hotpluggable backend instead of the custom cg_clif driver so that # build scripts are still compiled using cg_llvm. -export RUSTC=$(pwd)/../"target/"$CHANNEL"/cg_clif_build_sysroot" +export RUSTC=$dir"/cg_clif_build_sysroot" export RUSTFLAGS=$RUSTFLAGS" --clif" +cd $(dirname "$0") + +# Cleanup for previous run +# v Clean target dir except for build scripts and incremental cache +#rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/null || true + +# We expect the target dir in the default location. Guard against the user changing it. +export CARGO_TARGET_DIR=target + # Build libs export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked -Cpanic=abort" if [[ "$1" != "--debug" ]]; then @@ -39,5 +35,5 @@ else fi # Copy files to sysroot -mkdir -p sysroot/lib/rustlib/$TARGET_TRIPLE/lib/ -cp -r target/$TARGET_TRIPLE/$sysroot_channel/deps/* sysroot/lib/rustlib/$TARGET_TRIPLE/lib/ +mkdir -p $dir/sysroot/lib/rustlib/$TARGET_TRIPLE/lib/ +cp -a target/$TARGET_TRIPLE/$sysroot_channel/deps/* $dir/sysroot/lib/rustlib/$TARGET_TRIPLE/lib/ diff --git a/build_sysroot/prepare_sysroot_src.sh b/build_sysroot/prepare_sysroot_src.sh index 14aa77478f5..d0fb09ce745 100755 --- a/build_sysroot/prepare_sysroot_src.sh +++ b/build_sysroot/prepare_sysroot_src.sh @@ -12,7 +12,7 @@ fi rm -rf $DST_DIR mkdir -p $DST_DIR/library -cp -r $SRC_DIR/library $DST_DIR/ +cp -a $SRC_DIR/library $DST_DIR/ pushd $DST_DIR echo "[GIT] init" diff --git a/clean_all.sh b/clean_all.sh index 3003a0ea2d1..5a69c862d01 100755 --- a/clean_all.sh +++ b/clean_all.sh @@ -1,5 +1,5 @@ #!/bin/bash --verbose set -e -rm -rf target/ build_sysroot/{sysroot/,sysroot_src/,target/} perf.data{,.old} +rm -rf target/ build/ build_sysroot/{sysroot_src/,target/} perf.data{,.old} rm -rf rand/ regex/ simple-raytracer/ diff --git a/cargo.sh b/scripts/cargo.sh similarity index 51% rename from cargo.sh rename to scripts/cargo.sh index cebc3e67363..e63daa40f35 100755 --- a/cargo.sh +++ b/scripts/cargo.sh @@ -1,19 +1,13 @@ #!/bin/bash -if [ -z $CHANNEL ]; then -export CHANNEL='release' -fi - -pushd $(dirname "$0") >/dev/null -source scripts/config.sh +dir=$(dirname "$0") +source $dir/config.sh # read nightly compiler from rust-toolchain file -TOOLCHAIN=$(cat rust-toolchain) - -popd >/dev/null +TOOLCHAIN=$(cat $dir/rust-toolchain) cmd=$1 -shift +shift || true if [[ "$cmd" = "jit" ]]; then cargo +${TOOLCHAIN} rustc $@ -- --jit diff --git a/scripts/config.sh b/scripts/config.sh index 530b7f242a0..af181f4f724 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -39,18 +39,19 @@ echo export RUSTC_WRAPPER= fi -export RUSTC=$(pwd)/"target/"$CHANNEL"/cg_clif" +dir=$(cd $(dirname "$BASH_SOURCE"); pwd) + +export RUSTC=$dir"/cg_clif" export RUSTFLAGS=$linker export RUSTDOCFLAGS=$linker' -Ztrim-diagnostic-paths=no -Cpanic=abort -Zpanic-abort-tests '\ -'-Zcodegen-backend='$(pwd)'/target/'$CHANNEL'/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$(pwd)'/build_sysroot/sysroot' +'-Zcodegen-backend='$dir'/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir'/sysroot' # FIXME remove once the atomic shim is gone if [[ `uname` == 'Darwin' ]]; then export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup" fi -export LD_LIBRARY_PATH="$(pwd)/target/out:$(pwd)/build_sysroot/sysroot/lib/rustlib/"$TARGET_TRIPLE"/lib:\ -$(pwd)/target/"$CHANNEL":$(rustc --print sysroot)/lib" +export LD_LIBRARY_PATH="$dir:$(rustc --print sysroot)/lib:$dir/target/out:$dir/sysroot/lib/rustlib/"$TARGET_TRIPLE"/lib" export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH export CG_CLIF_DISPLAY_CG_TIME=1 diff --git a/scripts/filter_profile.rs b/scripts/filter_profile.rs index c70c3ec47f3..3327c10089d 100755 --- a/scripts/filter_profile.rs +++ b/scripts/filter_profile.rs @@ -1,9 +1,8 @@ #!/bin/bash #![forbid(unsafe_code)]/* This line is ignored by bash # This block is ignored by rustc -CHANNEL="release" pushd $(dirname "$0")/../ -source scripts/config.sh +source build/config.sh popd PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS --jit $0 #*/ diff --git a/scripts/rustup.sh b/scripts/rustup.sh index 38991d6d47d..541b3c6563b 100755 --- a/scripts/rustup.sh +++ b/scripts/rustup.sh @@ -26,6 +26,15 @@ case $1 in git add rust-toolchain build_sysroot/Cargo.lock git commit -m "Rustup to $(rustc -V)" ;; + "push") + cg_clif=$(pwd) + pushd ../rust + branch=update_cg_clif-$(date +%Y-%m-%d) + git checkout -b $branch + git subtree pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master + git push -u my $branch + popd + ;; *) echo "Unknown command '$1'" echo "Usage: ./rustup.sh prepare|commit" diff --git a/scripts/tests.sh b/scripts/tests.sh old mode 100644 new mode 100755 index 7d1e488ac3a..d941b73c81b --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -1,65 +1,71 @@ -function no_sysroot_tests() { - RUSTC=$RUSTC" "$RUSTFLAGS" -L crate=target/out --out-dir target/out -Cdebuginfo=2" +#!/bin/bash +set -e + +source build/config.sh +export CG_CLIF_INCR_CACHE_DISABLED=1 +MY_RUSTC=$RUSTC" "$RUSTFLAGS" -L crate=target/out --out-dir target/out -Cdebuginfo=2" + +function no_sysroot_tests() { echo "[BUILD] mini_core" - $RUSTC example/mini_core.rs --crate-name mini_core --crate-type lib,dylib --target $TARGET_TRIPLE + $MY_RUSTC example/mini_core.rs --crate-name mini_core --crate-type lib,dylib --target $TARGET_TRIPLE echo "[BUILD] example" - $RUSTC example/example.rs --crate-type lib --target $TARGET_TRIPLE + $MY_RUSTC example/example.rs --crate-type lib --target $TARGET_TRIPLE if [[ "$JIT_SUPPORTED" = "1" ]]; then echo "[JIT] mini_core_hello_world" - CG_CLIF_JIT_ARGS="abc bcd" $RUSTC --jit example/mini_core_hello_world.rs --cfg jit --target $HOST_TRIPLE + CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC --jit example/mini_core_hello_world.rs --cfg jit --target $HOST_TRIPLE else echo "[JIT] mini_core_hello_world (skipped)" fi echo "[AOT] mini_core_hello_world" - $RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target $TARGET_TRIPLE + $MY_RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target $TARGET_TRIPLE $RUN_WRAPPER ./target/out/mini_core_hello_world abc bcd # (echo "break set -n main"; echo "run"; sleep 1; echo "si -c 10"; sleep 1; echo "frame variable") | lldb -- ./target/out/mini_core_hello_world abc bcd echo "[AOT] arbitrary_self_types_pointers_and_wrappers" - $RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target $TARGET_TRIPLE + $MY_RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target $TARGET_TRIPLE $RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers } function base_sysroot_tests() { echo "[AOT] alloc_example" - $RUSTC example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE + $MY_RUSTC example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE $RUN_WRAPPER ./target/out/alloc_example if [[ "$JIT_SUPPORTED" = "1" ]]; then echo "[JIT] std_example" - $RUSTC --jit example/std_example.rs --target $HOST_TRIPLE + $MY_RUSTC --jit example/std_example.rs --target $HOST_TRIPLE else echo "[JIT] std_example (skipped)" fi echo "[AOT] dst_field_align" # FIXME Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed. - $RUSTC example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target $TARGET_TRIPLE + $MY_RUSTC example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target $TARGET_TRIPLE $RUN_WRAPPER ./target/out/dst_field_align || (echo $?; false) echo "[AOT] std_example" - $RUSTC example/std_example.rs --crate-type bin --target $TARGET_TRIPLE + $MY_RUSTC example/std_example.rs --crate-type bin --target $TARGET_TRIPLE $RUN_WRAPPER ./target/out/std_example arg echo "[AOT] subslice-patterns-const-eval" - $RUSTC example/subslice-patterns-const-eval.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE + $MY_RUSTC example/subslice-patterns-const-eval.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE $RUN_WRAPPER ./target/out/subslice-patterns-const-eval echo "[AOT] track-caller-attribute" - $RUSTC example/track-caller-attribute.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE + $MY_RUSTC example/track-caller-attribute.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE $RUN_WRAPPER ./target/out/track-caller-attribute echo "[AOT] mod_bench" - $RUSTC example/mod_bench.rs --crate-type bin --target $TARGET_TRIPLE + $MY_RUSTC example/mod_bench.rs --crate-type bin --target $TARGET_TRIPLE $RUN_WRAPPER ./target/out/mod_bench pushd rand rm -r ./target || true - ../cargo.sh test --workspace + ../build/cargo.sh test --workspace popd } @@ -69,7 +75,7 @@ function extended_sysroot_tests() { echo "[BENCH COMPILE] ebobby/simple-raytracer" hyperfine --runs ${RUN_RUNS:-10} --warmup 1 --prepare "cargo clean" \ "RUSTC=rustc RUSTFLAGS='' cargo build" \ - "../cargo.sh build" + "../build/cargo.sh build" echo "[BENCH RUN] ebobby/simple-raytracer" cp ./target/debug/main ./raytracer_cg_clif @@ -85,18 +91,33 @@ function extended_sysroot_tests() { pushd build_sysroot/sysroot_src/library/core/tests echo "[TEST] libcore" rm -r ./target || true - ../../../../../cargo.sh test + ../../../../../build/cargo.sh test popd pushd regex echo "[TEST] rust-lang/regex example shootout-regex-dna" - ../cargo.sh clean + ../build/cargo.sh clean # Make sure `[codegen mono items] start` doesn't poison the diff - ../cargo.sh build --example shootout-regex-dna - cat examples/regexdna-input.txt | ../cargo.sh run --example shootout-regex-dna | grep -v "Spawned thread" > res.txt + ../build/cargo.sh build --example shootout-regex-dna + cat examples/regexdna-input.txt | ../build/cargo.sh run --example shootout-regex-dna | grep -v "Spawned thread" > res.txt diff -u res.txt examples/regexdna-output.txt echo "[TEST] rust-lang/regex tests" - ../cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options + ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q popd } + +case "$1" in + "no_sysroot") + no_sysroot_tests + ;; + "base_sysroot") + base_sysroot_tests + ;; + "extended_sysroot") + extended_sysroot_tests + ;; + *) + echo "unknown test suite" + ;; +esac diff --git a/src/bin/cg_clif.rs b/src/bin/cg_clif.rs index 9a661dcbcc5..6253022ade2 100644 --- a/src/bin/cg_clif.rs +++ b/src/bin/cg_clif.rs @@ -31,11 +31,6 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks { .unwrap() .parent() .unwrap() - .parent() - .unwrap() - .parent() - .unwrap() - .join("build_sysroot") .join("sysroot"), ); } diff --git a/test.sh b/test.sh index e7a0d6ab4e1..3cdd4119d79 100755 --- a/test.sh +++ b/test.sh @@ -1,29 +1,15 @@ #!/bin/bash set -e -# Build cg_clif export RUSTFLAGS="-Zrun_dsymutil=no" -if [[ "$1" == "--release" ]]; then - export CHANNEL='release' - cargo build --release -else - export CHANNEL='debug' - cargo build --bin cg_clif -fi -# Config -source scripts/config.sh -source scripts/tests.sh -export CG_CLIF_INCR_CACHE_DISABLED=1 +./build.sh --without-sysroot $@ -# Cleanup rm -r target/out || true -no_sysroot_tests +scripts/tests.sh no_sysroot -echo "[BUILD] sysroot" -time ./build_sysroot/build_sysroot.sh +./build.sh $@ -base_sysroot_tests - -extended_sysroot_tests +scripts/tests.sh base_sysroot +scripts/tests.sh extended_sysroot