From aa868a8a212656a965b25762fb3a45f0bd64666c Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 09:45:00 +0200 Subject: [PATCH 01/11] dependency management --- Cargo.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 639fd50c750..6f7570edda5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,12 +40,15 @@ rustc_version = { version = "0.2.3", optional = true } env_logger = "0.6" log = "0.4" shell-escape = "0.1.4" +hex = "0.3.2" +rand = "0.6" # A noop dependency that changes in the Rust repository, it's a bit of a hack. # See the `src/tools/rustc-workspace-hack/README.md` file in `rust-lang/rust` # for more information. rustc-workspace-hack = "1.0.0" -hex = "0.3.2" -rand = "0.6" +# Depend on num-traits with default features to avoid having to rebuild +# between "cargo build" and "cargo intall". +num-traits = "*" [build-dependencies] vergen = "3" From c2791fc56abba235c4d0d56a746e24ac2cb778bf Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 12:51:59 +0200 Subject: [PATCH 02/11] add 'miri' script to help build, test and run miri --- miri | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 miri diff --git a/miri b/miri new file mode 100755 index 00000000000..fc8fdb7b5b2 --- /dev/null +++ b/miri @@ -0,0 +1,36 @@ +#!/bin/sh +set -e +TARGET=$(rustc --print target-spec-json -Z unstable-options | jq '.["llvm-target"]' -r) +SYSROOT=$(rustc --print sysroot) +export RUSTFLAGS="-C link-args=-Wl,-rpath,$SYSROOT/lib/rustlib/$TARGET/lib -C debug-assertions" + +COMMAND="$1" +shift + +case "$COMMAND" in +install) + exec cargo install --path "$(dirname "$0")" --force --locked --offline + ;; +build|test|run) + # Basic build + cargo build --release + + # We we want to just build, we are done. + if [ "$COMMAND" = "build" ]; then exit 0; fi + + # Get ourselves a sysroot + if [ -n "$MIRI_SYSROOT" ]; then + # sysroot already set + true + elif rustc --print sysroot | egrep -q 'build/[^/]+/stage'; then + # a local build, we have a proper libstd in $SYSROOT + true + else + # we have to build a sysroot + cargo run --release --bin cargo-miri -- miri setup + export MIRI_SYSROOT=$HOME/.cache/miri/HOST + fi + + exec cargo "$COMMAND" --release "$@" + ;; +esac From 6420293af4b87bca51c51661de1aa53d2158cdb5 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 12:59:52 +0200 Subject: [PATCH 03/11] adjust readme to miri build script --- README.md | 47 +++++++++++------------------------------------ 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index d3a06dea6d2..09b23b0db12 100644 --- a/README.md +++ b/README.md @@ -141,42 +141,28 @@ version of `rustc` that, instead of compiling your code, runs it. It accepts all the same flags as `rustc` (though the ones only affecting code generation and linking obviously will have no effect) [and more][miri-flags]. -To run the Miri driver, you need to have the `MIRI_SYSROOT` environment variable -set to an appropriate sysroot. You can generate such a sysroot with the -following incantation: - -``` -cargo run --bin cargo-miri -- miri setup -``` - -This basically runs the `cargo-miri` binary (which backs the `cargo miri` -subcommand) with `cargo`, and asks it to `setup`. It should in the end print -the directory where the libstd was built. In the following, we will assume it -is `~/.cache/miri/HOST`; you may have to adjust that if you are not using Linux. - -Now you can run the driver directly using +Running the Miri driver requires some fiddling with environment variables, so the `miri` script helps you do that. +For example, you can run the driver on a particular file by doing ```sh -MIRI_SYSROOT=~/.cache/miri/HOST cargo run tests/run-pass/format.rs # or whatever test you like +./miri run tests/run-pass/format.rs # or whatever test you like ``` and you can run the test suite using ``` -cargo test +./miri test ``` -We recommend adding the `--release` flag to make tests run faster. - -`cargo test --release FILTER` only runs those tests that contain `FILTER` in -their filename (including the base directory, e.g. `cargo test --release fail` -will run all compile-fail tests). +`./miri test FILTER` only runs those tests that contain `FILTER` in their +filename (including the base directory, e.g. `./miri test fail` will run all +compile-fail tests). You can get a trace of which MIR statements are being executed by setting the `MIRI_LOG` environment variable. For example: ```sh -MIRI_LOG=info cargo run tests/run-pass/vecs.rs +MIRI_LOG=info ./miri run tests/run-pass/vecs.rs ``` Setting `MIRI_LOG` like this will configure logging for Miri itself as well as @@ -185,7 +171,7 @@ can also do more targeted configuration, e.g. the following helps debug the stacked borrows implementation: ```sh -MIRI_LOG=rustc_mir::interpret=info,miri::stacked_borrows cargo run tests/run-pass/vecs.rs +MIRI_LOG=rustc_mir::interpret=info,miri::stacked_borrows ./miri run tests/run-pass/vecs.rs ``` In addition, you can set `MIRI_BACKTRACE=1` to get a backtrace of where an @@ -199,7 +185,7 @@ is probably easier to test it with the cargo wrapper. You can install your development version of Miri using ``` -cargo install --path . --force +./miri install ``` and then you can use it as if it was installed by `rustup`. Make sure you use @@ -235,18 +221,7 @@ rustup override set custom ``` With this, you should now have a working development setup! See -[above][testing-miri] for how to proceed working with the Miri driver. Notice -that rustc's sysroot is already built for Miri in this case, so you can set -`MIRI_SYSROOT=$(rustc --print sysroot)`. - -Running `cargo miri` in this setup is a bit more complicated, because the Miri -binary you just created needs help to find the libraries it links against. On -Linux, you can set the rpath to make this "just work": - -```sh -export RUSTFLAGS="-C link-args=-Wl,-rpath,$(rustc --print sysroot)/lib/rustlib/x86_64-unknown-linux-gnu/lib" -cargo install --path . --force -``` +[above][testing-miri] for how to proceed working with the Miri driver. ### Miri `-Z` flags and environment variables [miri-flags]: #miri--z-flags-and-environment-variables From ab1f60c9105f41f0ba3d6e7b87674828dcabc025 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 13:04:18 +0200 Subject: [PATCH 04/11] comments --- miri | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/miri b/miri index fc8fdb7b5b2..bc8bd59e5dd 100755 --- a/miri +++ b/miri @@ -2,6 +2,8 @@ set -e TARGET=$(rustc --print target-spec-json -Z unstable-options | jq '.["llvm-target"]' -r) SYSROOT=$(rustc --print sysroot) +# We set the rpath so that Miri finds the private rustc libraries it needs. +# We enable debug-assertions to get tracing. export RUSTFLAGS="-C link-args=-Wl,-rpath,$SYSROOT/lib/rustlib/$TARGET/lib -C debug-assertions" COMMAND="$1" @@ -22,8 +24,8 @@ build|test|run) if [ -n "$MIRI_SYSROOT" ]; then # sysroot already set true - elif rustc --print sysroot | egrep -q 'build/[^/]+/stage'; then - # a local build, we have a proper libstd in $SYSROOT + elif echo "$SYSROOT" | egrep -q 'build/[^/]+/stage'; then + # a local rustc build, assume we have a proper libstd in $SYSROOT true else # we have to build a sysroot From eb85ced8c730ad6a0f0518dfd7db3166a18e8108 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 13:08:47 +0200 Subject: [PATCH 05/11] improve backtraces --- miri | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/miri b/miri index bc8bd59e5dd..99ae7c5811e 100755 --- a/miri +++ b/miri @@ -4,7 +4,8 @@ TARGET=$(rustc --print target-spec-json -Z unstable-options | jq '.["llvm-target SYSROOT=$(rustc --print sysroot) # We set the rpath so that Miri finds the private rustc libraries it needs. # We enable debug-assertions to get tracing. -export RUSTFLAGS="-C link-args=-Wl,-rpath,$SYSROOT/lib/rustlib/$TARGET/lib -C debug-assertions" +# We enable line-only debuginfo for backtraces. +export RUSTFLAGS="-C link-args=-Wl,-rpath,$SYSROOT/lib/rustlib/$TARGET/lib -C debug-assertions -C debuginfo=1" COMMAND="$1" shift From cf96396fccbad2cce0f4a710d26951083a9864e6 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 14:40:27 +0200 Subject: [PATCH 06/11] make miri script smarter: auto-determine MIRI_SYSROOT, handle MIRI_TEST_TARGET --- README.md | 10 +++--- miri | 83 ++++++++++++++++++++++++++++++++----------- src/bin/cargo-miri.rs | 5 ++- travis.sh | 19 +++------- 4 files changed, 78 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 09b23b0db12..130072825e1 100644 --- a/README.md +++ b/README.md @@ -141,14 +141,16 @@ version of `rustc` that, instead of compiling your code, runs it. It accepts all the same flags as `rustc` (though the ones only affecting code generation and linking obviously will have no effect) [and more][miri-flags]. -Running the Miri driver requires some fiddling with environment variables, so the `miri` script helps you do that. -For example, you can run the driver on a particular file by doing +Running the Miri driver requires some fiddling with environment variables, so +the `miri` script helps you do that. For example, you can run the driver on a +particular file by doing ```sh -./miri run tests/run-pass/format.rs # or whatever test you like +./miri run tests/run-pass/format.rs +./miri run tests/run-pass/hello.rs --target i686-unknown-linux-gnu ``` -and you can run the test suite using +and you can run the test suite using: ``` ./miri test diff --git a/miri b/miri index 99ae7c5811e..1d090c8827f 100755 --- a/miri +++ b/miri @@ -7,33 +7,76 @@ SYSROOT=$(rustc --print sysroot) # We enable line-only debuginfo for backtraces. export RUSTFLAGS="-C link-args=-Wl,-rpath,$SYSROOT/lib/rustlib/$TARGET/lib -C debug-assertions -C debuginfo=1" +## Helper functions + +# Build a sysroot and set MIRI_SYSROOT to use it. Arguments are passed to `cargo miri setup`. +build_sysroot() { + # Build once, for the user to see. + cargo run --release --bin cargo-miri -- miri setup "$@" + # Call again, to just set env var. + eval $(cargo run --release -q --bin cargo-miri -- miri setup --env "$@") + export MIRI_SYSROOT +} + +# Prepare and set MIRI_SYSROOT. Respects `MIRI_TEST_TARGET` and takes into account +# locally built vs. distributed rustc. +find_sysroot() { + # Get ourselves a sysroot + if [ -n "$MIRI_SYSROOT" ]; then + # Sysroot already set, use that. + true + elif echo "$SYSROOT" | egrep -q 'build/[^/]+/stage'; then + # A local rustc build. + if [ -n "$MIRI_TEST_TARGET" ]; then + # Foreign targets still need a build. Use the rustc sources. + export XARGO_RUST_SRC="$SYSROOT/../../../src" + build_sysroot --target "$MIRI_TEST_TARGET" + else + # Assume we have a proper host libstd in $SYSROOT. + true + fi + else + # We have to build a sysroot either way. + if [ -n "$MIRI_TEST_TARGET" ]; then + build_sysroot --target "$MIRI_TEST_TARGET" + else + build_sysroot + fi + fi +} + +## Main + COMMAND="$1" shift case "$COMMAND" in install) - exec cargo install --path "$(dirname "$0")" --force --locked --offline + # "--locked" to respect the Cargo.lock file if it exists, + # "--offline" to avoid querying the registry (for yanked packages). + exec cargo "$COMMAND" --path "$(dirname "$0")" --force --locked --offline "$@" ;; -build|test|run) - # Basic build - cargo build --release - - # We we want to just build, we are done. - if [ "$COMMAND" = "build" ]; then exit 0; fi - - # Get ourselves a sysroot - if [ -n "$MIRI_SYSROOT" ]; then - # sysroot already set - true - elif echo "$SYSROOT" | egrep -q 'build/[^/]+/stage'; then - # a local rustc build, assume we have a proper libstd in $SYSROOT - true - else - # we have to build a sysroot - cargo run --release --bin cargo-miri -- miri setup - export MIRI_SYSROOT=$HOME/.cache/miri/HOST +build) + # Build, and let caller control flags. + exec cargo "$COMMAND" --release "$@" + ;; +test|run) + # In "run" mode, scan for "--target" to set the "MIRI_TEST_TARGET" env var so + # that we set the MIRI_SYSROOT up the right way. + if [ "$COMMAND" = "run" ] && [ -z "$MIRI_TEST_TARGET" ]; then + for ARG in "$@"; do + if [ "$LAST_ARG" = "--target" ]; then + # Found it! + export MIRI_TEST_TARGET="$ARG" + break + fi + LAST_ARG="$ARG" + done fi - + # First build and get a sysroot. + cargo build --release + find_sysroot + # Then run the actual command. exec cargo "$COMMAND" --release "$@" ;; esac diff --git a/src/bin/cargo-miri.rs b/src/bin/cargo-miri.rs index 5dc7a75fc76..55c53e7361a 100644 --- a/src/bin/cargo-miri.rs +++ b/src/bin/cargo-miri.rs @@ -243,6 +243,7 @@ path = "lib.rs" File::create(dir.join("lib.rs")).unwrap(); // Run xargo. let target = get_arg_flag_value("--target"); + let print_env = !ask_user && has_arg_flag("--env"); // whether we just print the necessary environment variable let mut command = Command::new("xargo"); command.arg("build").arg("-q") .current_dir(&dir) @@ -265,7 +266,9 @@ path = "lib.rs" }; let sysroot = if is_host { dir.join("HOST") } else { PathBuf::from(dir) }; std::env::set_var("MIRI_SYSROOT", &sysroot); - if !ask_user { + if print_env { + println!("MIRI_SYSROOT={}", sysroot.display()); + } else if !ask_user { println!("A libstd for Miri is now available in `{}`", sysroot.display()); } } diff --git a/travis.sh b/travis.sh index 84f9c408dcb..dc98ca44ae5 100755 --- a/travis.sh +++ b/travis.sh @@ -3,36 +3,27 @@ set -euo pipefail # Determine configuration if [ "$TRAVIS_OS_NAME" == osx ]; then - MIRI_SYSROOT_BASE=~/Library/Caches/miri.miri.miri/ FOREIGN_TARGET=i686-apple-darwin else - MIRI_SYSROOT_BASE=~/.cache/miri/ FOREIGN_TARGET=i686-unknown-linux-gnu fi # Prepare echo "Build and install miri" -cargo build --release --all-features --all-targets -cargo install --all-features --force --path . -echo - -echo "Get ourselves a MIR-full libstd for the host and a foreign architecture" -cargo miri setup -cargo miri setup --target "$FOREIGN_TARGET" +./miri build --all-features --all-targets +./miri install echo # Test function run_tests { - cargo test --release --all-features - test-cargo-miri/run-test.py + ./miri test + test-cargo-miri/run-test.py } echo "Test host architecture" -export MIRI_SYSROOT="$MIRI_SYSROOT_BASE"/HOST run_tests echo echo "Test foreign architecture ($FOREIGN_TARGET)" -export MIRI_SYSROOT="$MIRI_SYSROOT_BASE" MIRI_TEST_TARGET="$FOREIGN_TARGET" -run_tests +MIRI_TEST_TARGET="$FOREIGN_TARGET" run_tests echo From 7f3a298f6db4acdd33e29f5c4dc20cd91efa1859 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 15:02:17 +0200 Subject: [PATCH 07/11] improve macOS-compatibility by being more awful --- miri | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/miri b/miri index 1d090c8827f..7c88309c447 100755 --- a/miri +++ b/miri @@ -1,6 +1,7 @@ #!/bin/sh set -e -TARGET=$(rustc --print target-spec-json -Z unstable-options | jq '.["llvm-target"]' -r) +# I'd love to use `jq` for parsing the JSON properly, but macOS is totally underequipped for this kind of work. +TARGET=$(rustc --print target-spec-json -Z unstable-options | grep llvm-target | cut -d '"' -f 4) SYSROOT=$(rustc --print sysroot) # We set the rpath so that Miri finds the private rustc libraries it needs. # We enable debug-assertions to get tracing. From a358590679df59d57ca66524f162c5b2f8f2bf18 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 15:05:55 +0200 Subject: [PATCH 08/11] fix indentation --- miri | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miri b/miri index 7c88309c447..effbd023f19 100755 --- a/miri +++ b/miri @@ -33,8 +33,8 @@ find_sysroot() { export XARGO_RUST_SRC="$SYSROOT/../../../src" build_sysroot --target "$MIRI_TEST_TARGET" else - # Assume we have a proper host libstd in $SYSROOT. - true + # Assume we have a proper host libstd in $SYSROOT. + true fi else # We have to build a sysroot either way. From 3a837cdedd782dab48b98eb488c4c8f90c013cd2 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 15:19:53 +0200 Subject: [PATCH 09/11] add comment --- travis.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/travis.sh b/travis.sh index dc98ca44ae5..4df12613fb8 100755 --- a/travis.sh +++ b/travis.sh @@ -17,6 +17,8 @@ echo # Test function run_tests { ./miri test + # "miri test" has built the sysroot for us, now this should pass without + # any interactive questions. test-cargo-miri/run-test.py } From aa888c55936a473d5ff0c4f474a42bf94237a77d Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 15:24:37 +0200 Subject: [PATCH 10/11] remove redundant RUSTFLAGS --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fd8c1260215..1ba55e0c724 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,6 @@ env: global: - RUST_TEST_NOCAPTURE=1 - RUST_BACKTRACE=1 - - RUSTFLAGS="-C debug-assertions" before_script: # Linux: install extra stuff for cross-compilation From 0b6b7dad8f1aaeca31a8c298fa8fcae3831a69bf Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 27 May 2019 15:35:48 +0200 Subject: [PATCH 11/11] fix more indentation --- miri | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/miri b/miri index effbd023f19..a5f7271b246 100755 --- a/miri +++ b/miri @@ -37,7 +37,7 @@ find_sysroot() { true fi else - # We have to build a sysroot either way. + # A normal toolchain. We have to build a sysroot either way. if [ -n "$MIRI_TEST_TARGET" ]; then build_sysroot --target "$MIRI_TEST_TARGET" else