Auto merge of #1757 - RalfJung:rustdoc, r=RalfJung

add rustdoc support

`@teryror` did all the work in https://github.com/rust-lang/miri/pull/1671; I just finished things up and fixed conflicts. Also thanks to `@hyd-dev` for preemptively fixing a sysroot issue that would have taken me some time to diagnose.

Fixes https://github.com/rust-lang/miri/issues/584

README.md

@@ -290,6 +290,10 @@ different Miri binaries, and as such worth documenting:
   that the compiled `rlib`s are compatible with Miri.
   When set while running `cargo-miri`, it indicates that we are part of a sysroot
   build (for which some crates need special treatment).
+* `MIRI_CALLED_FROM_RUSTDOC` when set to any value tells `cargo-miri` that it is
+  running as a child process of `rustdoc`, which invokes it twice for each doc-test
+  and requires special treatment, most notably a check-only build before interpretation.
+  This is set by `cargo-miri` itself when running as a `rustdoc`-wrapper.
 * `MIRI_CWD` when set to any value tells the Miri driver to change to the given
   directory after loading all the source files, but before commencing
   interpretation. This is useful if the interpreted program wants a different

cargo-miri/bin.rs

@@ -1,8 +1,8 @@
 use std::env;
 use std::ffi::OsString;
 use std::fs::{self, File};
-use std::io::{self, BufRead, BufReader, BufWriter, Write};
 use std::iter::TakeWhile;
+use std::io::{self, BufRead, BufReader, BufWriter, Read, Write};
 use std::ops::Not;
 use std::path::{Path, PathBuf};
 use std::process::Command;
@@ -46,6 +46,24 @@ struct CrateRunEnv {
     env: Vec<(OsString, OsString)>,
     /// The current working directory.
     current_dir: OsString,
+    /// The contents passed via standard input.
+    stdin: Vec<u8>,
+}
+
+impl CrateRunEnv {
+    /// Gather all the information we need.
+    fn collect(args: env::Args, capture_stdin: bool) -> Self {
+        let args = args.collect();
+        let env = env::vars_os().collect();
+        let current_dir = env::current_dir().unwrap().into_os_string();
+
+        let mut stdin = Vec::new();
+        if capture_stdin {
+            std::io::stdin().lock().read_to_end(&mut stdin).expect("cannot read stdin");
+        }
+
+        CrateRunEnv { args, env, current_dir, stdin }
+    }
 }
 
 /// The information Miri needs to run a crate. Stored as JSON when the crate is "compiled".
@@ -58,14 +76,6 @@ enum CrateRunInfo {
 }
 
 impl CrateRunInfo {
-    /// Gather all the information we need.
-    fn collect(args: env::Args) -> Self {
-        let args = args.collect();
-        let env = env::vars_os().collect();
-        let current_dir = env::current_dir().unwrap().into_os_string();
-        Self::RunWith(CrateRunEnv { args, env, current_dir })
-    }
-
     fn store(&self, filename: &Path) {
         let file = File::create(filename)
             .unwrap_or_else(|_| show_error(format!("cannot create `{}`", filename.display())));
@@ -155,6 +165,13 @@ fn forward_patched_extern_arg(args: &mut impl Iterator<Item = String>, cmd: &mut
     }
 }
 
+fn forward_miri_sysroot(cmd: &mut Command) {
+    let sysroot =
+        env::var_os("MIRI_SYSROOT").expect("the wrapper should have set MIRI_SYSROOT");
+    cmd.arg("--sysroot");
+    cmd.arg(sysroot);
+}
+
 /// Returns the path to the `miri` binary
 fn find_miri() -> PathBuf {
     if let Some(path) = env::var_os("MIRI") {
@@ -190,6 +207,22 @@ fn exec(mut cmd: Command) {
     }
 }
 
+/// Execute the command and pipe `input` into its stdin.
+/// If it fails, fail this process with the same exit code.
+/// Otherwise, continue.
+fn exec_with_pipe(mut cmd: Command, input: &[u8]) {
+    cmd.stdin(std::process::Stdio::piped());
+    let mut child = cmd.spawn().expect("failed to spawn process");
+    {
+        let stdin = child.stdin.as_mut().expect("failed to open stdin");
+        stdin.write_all(input).expect("failed to write out test source");
+    }
+    let exit_status = child.wait().expect("failed to run command");
+    if exit_status.success().not() {
+        std::process::exit(exit_status.code().unwrap_or(-1))
+    }
+}
+
 fn xargo_version() -> Option<(u32, u32, u32)> {
     let out = xargo_check().arg("--version").output().ok()?;
     if !out.status.success() {
@@ -548,7 +581,7 @@ fn phase_cargo_miri(mut args: env::Args) {
     // us in order to skip them.
     cmd.env(&host_runner_env_name, &cargo_miri_path);
 
-    // Set rustdoc to us as well, so we can make it do nothing (see issue #584).
+    // Set rustdoc to us as well, so we can run doctests.
     cmd.env("RUSTDOC", &cargo_miri_path);
 
     // Run cargo.
@@ -591,17 +624,22 @@ fn phase_cargo_rustc(mut args: env::Args) {
     }
 
     fn out_filename(prefix: &str, suffix: &str) -> PathBuf {
-        let mut path = PathBuf::from(get_arg_flag_value("--out-dir").unwrap());
-        path.push(format!(
-            "{}{}{}{}",
-            prefix,
-            get_arg_flag_value("--crate-name").unwrap(),
-            // This is technically a `-C` flag but the prefix seems unique enough...
-            // (and cargo passes this before the filename so it should be unique)
-            get_arg_flag_value("extra-filename").unwrap_or(String::new()),
-            suffix,
-        ));
-        path
+        if let Some(out_dir) = get_arg_flag_value("--out-dir") {
+            let mut path = PathBuf::from(out_dir);
+            path.push(format!(
+                "{}{}{}{}",
+                prefix,
+                get_arg_flag_value("--crate-name").unwrap(),
+                // This is technically a `-C` flag but the prefix seems unique enough...
+                // (and cargo passes this before the filename so it should be unique)
+                get_arg_flag_value("extra-filename").unwrap_or(String::new()),
+                suffix,
+            ));
+            path
+        } else {
+            let out_file = get_arg_flag_value("-o").unwrap();
+            PathBuf::from(out_file)
+        }
     }
 
     let verbose = std::env::var_os("MIRI_VERBOSE").is_some();
@@ -631,12 +669,43 @@ fn phase_cargo_rustc(mut args: env::Args) {
     let runnable_crate = !print && is_runnable_crate();
 
     if runnable_crate && target_crate {
+        let inside_rustdoc = env::var_os("MIRI_CALLED_FROM_RUSTDOC").is_some();
         // This is the binary or test crate that we want to interpret under Miri.
         // But we cannot run it here, as cargo invoked us as a compiler -- our stdin and stdout are not
         // like we want them.
         // Instead of compiling, we write JSON into the output file with all the relevant command-line flags
         // and environment variables; this is used when cargo calls us again in the CARGO_TARGET_RUNNER phase.
-        store_json(CrateRunInfo::collect(args));
+        let env = CrateRunEnv::collect(args, inside_rustdoc);
+
+        // Rustdoc expects us to exit with an error code if the test is marked as `compile_fail`,
+        // just creating the JSON file is not enough: we need to detect syntax errors,
+        // so we need to run Miri with `MIRI_BE_RUSTC` for a check-only build.
+        if inside_rustdoc {
+            let mut cmd = miri();
+
+            // Ensure --emit argument for a check-only build is present.
+            // We cannot use the usual helpers since we need to check specifically in `env.args`.
+            if let Some(i) = env.args.iter().position(|arg| arg.starts_with("--emit=")) {
+                // For `no_run` tests, rustdoc passes a `--emit` flag; make sure it has the right shape.
+                assert_eq!(env.args[i], "--emit=metadata");
+            } else {
+                // For all other kinds of tests, we can just add our flag.
+                cmd.arg("--emit=metadata");
+            }
+
+            cmd.args(&env.args);
+            cmd.env("MIRI_BE_RUSTC", "1");
+
+            if verbose {
+                eprintln!("[cargo-miri rustc] captured input:\n{}", std::str::from_utf8(&env.stdin).unwrap());
+                eprintln!("[cargo-miri rustc] {:?}", cmd);
+            }
+
+            exec_with_pipe(cmd, &env.stdin);
+        }
+
+        store_json(CrateRunInfo::RunWith(env));
+
         return;
     }
 
@@ -681,10 +750,7 @@ fn phase_cargo_rustc(mut args: env::Args) {
         }
 
         // Use our custom sysroot.
-        let sysroot =
-            env::var_os("MIRI_SYSROOT").expect("the wrapper should have set MIRI_SYSROOT");
-        cmd.arg("--sysroot");
-        cmd.arg(sysroot);
+        forward_miri_sysroot(&mut cmd);
     } else {
         // For host crates or when we are printing, just forward everything.
         cmd.args(args);
@@ -772,11 +838,10 @@ fn phase_cargo_runner(binary: &Path, binary_args: env::Args) {
             cmd.arg(arg);
         }
     }
-    // Set sysroot.
-    let sysroot =
-        env::var_os("MIRI_SYSROOT").expect("the wrapper should have set MIRI_SYSROOT");
-    cmd.arg("--sysroot");
-    cmd.arg(sysroot);
+    if env::var_os("MIRI_CALLED_FROM_RUSTDOC").is_none() {
+        // Set sysroot (if we are inside rustdoc, we already did that in `phase_cargo_rustdoc`).
+        forward_miri_sysroot(&mut cmd);
+    }
     // Respect `MIRIFLAGS`.
     if let Ok(a) = env::var("MIRIFLAGS") {
         // This code is taken from `RUSTFLAGS` handling in cargo.
@@ -801,6 +866,77 @@ fn phase_cargo_runner(binary: &Path, binary_args: env::Args) {
     if verbose {
         eprintln!("[cargo-miri runner] {:?}", cmd);
     }
+
+    if std::env::var_os("MIRI_CALLED_FROM_RUSTDOC").is_some() {
+        exec_with_pipe(cmd, &info.stdin)
+    } else {
+        exec(cmd)
+    }
+}
+
+fn phase_cargo_rustdoc(fst_arg: &str, mut args: env::Args) {
+    let verbose = std::env::var_os("MIRI_VERBOSE").is_some();
+
+    // phase_cargo_miri sets the RUSTDOC env var to ourselves, so we can't use that here;
+    // just default to a straight-forward invocation for now:
+    let mut cmd = Command::new("rustdoc");
+
+    // Because of the way the main function is structured, we have to take the first argument spearately
+    // from the rest; to simplify the following argument patching loop, we'll just skip that one.
+    // This is fine for now, because cargo will never pass --extern arguments in the first position,
+    // but we should defensively assert that this will work.
+    let extern_flag = "--extern";
+    assert!(fst_arg != extern_flag);
+    cmd.arg(fst_arg);
+
+    let runtool_flag = "--runtool";
+    // `crossmode` records if *any* argument matches `runtool_flag`; here we check the first one.
+    let mut crossmode = fst_arg == runtool_flag;
+    while let Some(arg) = args.next() {
+        if arg == extern_flag {
+            // Patch --extern arguments to use *.rmeta files, since phase_cargo_rustc only creates stub *.rlib files.
+            forward_patched_extern_arg(&mut args, &mut cmd);
+        } else if arg == runtool_flag {
+            // An existing --runtool flag indicates cargo is running in cross-target mode, which we don't support.
+            // Note that this is only passed when cargo is run with the unstable -Zdoctest-xcompile flag;
+            // otherwise, we won't be called as rustdoc at all.
+            crossmode = true;
+            break;
+        } else {
+            cmd.arg(arg);
+        }
+    }
+
+    if crossmode {
+        show_error(format!("cross-interpreting doc-tests is not currently supported by Miri."));
+    }
+
+    // For each doc-test, rustdoc starts two child processes: first the test is compiled,
+    // then the produced executable is invoked. We want to reroute both of these to cargo-miri,
+    // such that the first time we'll enter phase_cargo_rustc, and phase_cargo_runner second.
+    //
+    // rustdoc invokes the test-builder by forwarding most of its own arguments, which makes
+    // it difficult to determine when phase_cargo_rustc should run instead of phase_cargo_rustdoc.
+    // Furthermore, the test code is passed via stdin, rather than a temporary file, so we need
+    // to let phase_cargo_rustc know to expect that. We'll use this environment variable as a flag:
+    cmd.env("MIRI_CALLED_FROM_RUSTDOC", "1");
+
+    // The `--test-builder` and `--runtool` arguments are unstable rustdoc features,
+    // which are disabled by default. We first need to enable them explicitly:
+    cmd.arg("-Z").arg("unstable-options");
+
+    // rustdoc needs to know the right sysroot.
+    forward_miri_sysroot(&mut cmd);
+
+    // Make rustdoc call us back.
+    let cargo_miri_path = std::env::current_exe().expect("current executable path invalid");
+    cmd.arg("--test-builder").arg(&cargo_miri_path); // invoked by forwarding most arguments
+    cmd.arg("--runtool").arg(&cargo_miri_path); // invoked with just a single path argument
+
+    if verbose {
+        eprintln!("[cargo-miri rustdoc] {:?}", cmd);
+    }
+
     exec(cmd)
 }
 
@@ -817,6 +953,29 @@ fn main() {
         return;
     }
 
+    // The way rustdoc invokes rustc is indistuingishable from the way cargo invokes rustdoc by the
+    // arguments alone. `phase_cargo_rustdoc` sets this environment variable to let us disambiguate.
+    let invoked_by_rustdoc = env::var_os("MIRI_CALLED_FROM_RUSTDOC").is_some();
+    if invoked_by_rustdoc {
+        // ...however, we then also see this variable when rustdoc invokes us as the testrunner!
+        // The runner is invoked as `$runtool ($runtool-arg)* output_file`;
+        // since we don't specify any runtool-args, and rustdoc supplies multiple arguments to
+        // the test-builder unconditionally, we can just check the number of remaining arguments:
+        if args.len() == 1 {
+            let arg = args.next().unwrap();
+            let binary = Path::new(&arg);
+            if binary.exists() {
+                phase_cargo_runner(binary, args);
+            } else {
+                show_error(format!("`cargo-miri` called with non-existing path argument `{}` in rustdoc mode; please invoke this binary through `cargo miri`", arg));
+            }
+        } else {
+            phase_cargo_rustc(args);
+        }
+
+        return;
+    }
+
     // Dispatch to `cargo-miri` phase. There are three phases:
     // - When we are called via `cargo miri`, we run as the frontend and invoke the underlying
     //   cargo. We set RUSTC_WRAPPER and CARGO_TARGET_RUNNER to ourselves.
@@ -829,16 +988,15 @@ fn main() {
         Some("miri") => phase_cargo_miri(args),
         Some("rustc") => phase_cargo_rustc(args),
         Some(arg) => {
-            // We have to distinguish the "runner" and "rustfmt" cases.
+            // We have to distinguish the "runner" and "rustdoc" cases.
             // As runner, the first argument is the binary (a file that should exist, with an absolute path);
-            // as rustfmt, the first argument is a flag (`--something`).
+            // as rustdoc, the first argument is a flag (`--something`).
             let binary = Path::new(arg);
             if binary.exists() {
                 assert!(!arg.starts_with("--")); // not a flag
                 phase_cargo_runner(binary, args);
             } else if arg.starts_with("--") {
-                // We are rustdoc.
-                eprintln!("Running doctests is not currently supported by Miri.")
+                phase_cargo_rustdoc(arg, args);
             } else {
                 show_error(format!("`cargo-miri` called with unexpected first argument `{}`; please only invoke this binary through `cargo miri`", arg));
             }

test-cargo-miri/run-test.py

@@ -5,7 +5,7 @@ Assumes the `MIRI_SYSROOT` env var to be set appropriately,
 and the working directory to contain the cargo-miri-test project.
 '''
 
-import sys, subprocess, os
+import sys, subprocess, os, re
 
 CGREEN  = '\33[32m'
 CBOLD   = '\33[1m'
@@ -23,6 +23,10 @@ def cargo_miri(cmd, quiet = True):
         args += ["--target", os.environ['MIRI_TEST_TARGET']]
     return args
 
+def normalize_stdout(str):
+    str = str.replace("src\\", "src/") # normalize paths across platforms
+    return re.sub("finished in \d+\.\d\ds", "finished in $TIME", str)
+
 def test(name, cmd, stdout_ref, stderr_ref, stdin=b'', env={}):
     print("Testing {}...".format(name))
     ## Call `cargo miri`, capture all output
@@ -38,7 +42,7 @@ def test(name, cmd, stdout_ref, stderr_ref, stdin=b'', env={}):
     (stdout, stderr) = p.communicate(input=stdin)
     stdout = stdout.decode("UTF-8")
     stderr = stderr.decode("UTF-8")
-    if p.returncode == 0 and stdout == open(stdout_ref).read() and stderr == open(stderr_ref).read():
+    if p.returncode == 0 and normalize_stdout(stdout) == open(stdout_ref).read() and stderr == open(stderr_ref).read():
         # All good!
         return
     # Show output
@@ -101,26 +105,27 @@ def test_cargo_miri_run():
 def test_cargo_miri_test():
     # rustdoc is not run on foreign targets
     is_foreign = 'MIRI_TEST_TARGET' in os.environ
-    rustdoc_ref = "test.stderr-empty.ref" if is_foreign else "test.stderr-rustdoc.ref"
+    default_ref = "test.cross-target.stdout.ref" if is_foreign else "test.default.stdout.ref"
+    filter_ref = "test.filter.cross-target.stdout.ref" if is_foreign else "test.filter.stdout.ref"
 
     test("`cargo miri test`",
         cargo_miri("test"),
-        "test.default.stdout.ref", rustdoc_ref,
+        default_ref, "test.stderr-empty.ref",
         env={'MIRIFLAGS': "-Zmiri-seed=feed"},
     )
-    test("`cargo miri test` (no isolation)",
-        cargo_miri("test"),
-        "test.default.stdout.ref", rustdoc_ref,
+    test("`cargo miri test` (no isolation, no doctests)",
+        cargo_miri("test") + ["--bins", "--tests"], # no `--lib`, we disabled that in `Cargo.toml`
+        "test.cross-target.stdout.ref", "test.stderr-empty.ref",
         env={'MIRIFLAGS': "-Zmiri-disable-isolation"},
     )
     test("`cargo miri test` (raw-ptr tracking)",
         cargo_miri("test"),
-        "test.default.stdout.ref", rustdoc_ref,
+        default_ref, "test.stderr-empty.ref",
         env={'MIRIFLAGS': "-Zmiri-track-raw-pointers"},
     )
     test("`cargo miri test` (with filter)",
         cargo_miri("test") + ["--", "--format=pretty", "le1"],
-        "test.filter.stdout.ref", rustdoc_ref,
+        filter_ref, "test.stderr-empty.ref",
     )
     test("`cargo miri test` (test target)",
         cargo_miri("test") + ["--test", "test", "--", "--format=pretty"],
@@ -138,6 +143,7 @@ def test_cargo_miri_test():
 
 os.chdir(os.path.dirname(os.path.realpath(__file__)))
 os.environ["RUST_TEST_NOCAPTURE"] = "0" # this affects test output, so make sure it is not set
+os.environ["RUST_TEST_THREADS"] = "1" # avoid non-deterministic output due to concurrent test runs
 
 target_str = " for target {}".format(os.environ['MIRI_TEST_TARGET']) if 'MIRI_TEST_TARGET' in os.environ else ""
 print(CGREEN + CBOLD + "## Running `cargo miri` tests{}".format(target_str) + CEND)

test-cargo-miri/src/lib.rs

@@ -2,6 +2,12 @@
 /// ```rust
 /// assert!(cargo_miri_test::make_true());
 /// ```
+/// ```rust,no_run
+/// assert!(cargo_miri_test::make_true());
+/// ```
+/// ```rust,compile_fail
+/// assert!(cargo_miri_test::make_true() == 5);
+/// ```
 pub fn make_true() -> bool {
     issue_1567::use_the_dependency();
     issue_1705::use_the_dependency();

test-cargo-miri/test.cross-target.stdout.ref

@@ -0,0 +1,10 @@
+
+running 1 test
+.
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out
+
+
+running 7 tests
+..i....
+test result: ok. 6 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out
+

test-cargo-miri/test.default.stdout.ref

@@ -8,3 +8,11 @@ running 7 tests
 ..i....
 test result: ok. 6 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out
 
+
+running 3 tests
+test src/lib.rs - make_true (line 2) ... ok
+test src/lib.rs - make_true (line 5) ... ok
+test src/lib.rs - make_true (line 8) ... ok
+
+test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+

test-cargo-miri/test.filter.cross-target.stdout.ref

@@ -0,0 +1,11 @@
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 1 filtered out
+
+
+running 1 test
+test simple1 ... ok
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 6 filtered out
+

test-cargo-miri/test.filter.stdout.ref

@@ -9,3 +9,8 @@ test simple1 ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 6 filtered out
 
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 3 filtered out; finished in $TIME
+

test-cargo-miri/test.stderr-rustdoc.ref

@@ -1 +0,0 @@
-Running doctests is not currently supported by Miri.