Rollup merge of #45783 - kennytm:compiler-test-fixes, r=alexcrichton

Miscellaneous changes for CI, Docker and compiletest.

This PR contains 7 independent commits that improves interaction with CI, Docker and compiletest.

1. a4e5c91cb8 — Forces a newline every 100 dots when testing in quiet mode. Prevents spurious timeouts when abusing the CI to test Android jobs.

2. 1b5aaf22e8 — Use vault.centos.org for dist-powerpc64le-linux, see #45744.

3. 33400fbbcd — Modify `src/ci/docker/run.sh` so that the docker images can be run from Docker Toolbox for Windows on Windows 7. I haven't checked the behavior of the newer Docker for Windows on Windows 10. Also, "can run" does not mean all the test can pass successfully (the UDP tests failed last time I checked)

4. d517668a08 — Don't emit a real warning the linker segfault, which affects UI tests like https://github.com/rust-lang/rust/pull/45489#issuecomment-340134944. Log it instead.

5. 51e2247948 — During run-pass, trim the output if stdout/stderr exceeds 416 KB (top 160 KB + bottom 256 KB). This is an attempt to avoid spurious failures like https://github.com/rust-lang/rust/pull/45384#issuecomment-341755788

6. 9cfdabaf3c — Force `gem update --system` before deploy. This is an attempt to prevent spurious error #44159.

7. eee10cc482 — Tries to print the crash log on macOS on failure. This is an attempt to debug #45230.

.travis.yml

@@ -253,7 +253,14 @@ after_failure:
 
   # Random attempt at debugging currently. Just poking around in here to see if
   # anything shows up.
-  - ls $HOME/Library/Logs/DiagnosticReports/
+  - ls -lat $HOME/Library/Logs/DiagnosticReports/
+  - find $HOME/Library/Logs/DiagnosticReports/ ! \(
+      -name '*.stage2-*.crash'
+      -name 'com.apple.CoreSimulator.CoreSimulatorService-*.crash'
+    \)
+      -exec echo -e travis_fold":start:crashlog\n\033[31;1m" {} "\033[0m" \;
+      -exec head -750 {} \;
+      -exec echo travis_fold":"end:crashlog \;
 
   # attempt to debug anything killed by the oom killer on linux, just to see if
   # it happened
@@ -286,6 +293,7 @@ before_deploy:
           rm -rf obj/build/dist/doc &&
           cp -r obj/build/dist/* deploy/$TRAVIS_COMMIT;
       fi
+  - travis_retry gem update --system
 
 deploy:
   - provider: s3

src/Cargo.lock

@@ -348,7 +348,9 @@ dependencies = [
  "getopts 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]

src/ci/docker/README.md

@@ -22,6 +22,48 @@ Images will output artifacts in an `obj` dir at the root of a repository.
 - `scripts` contains files shared by docker images
 - `disabled` contains images that are not built on travis
 
+## Docker Toolbox on Windows
+
+For Windows before Windows 10, the docker images can be run on Windows via
+[Docker Toolbox]. There are several preparation needs to be made before running
+a Docker image.
+
+1. Stop the virtual machine from the terminal with `docker-machine stop`
+
+2. If your Rust source is placed outside of `C:\Users\**`, e.g. if you place the
+    repository in the `E:\rust` folder, please add a shared folder from
+    VirtualBox by:
+
+    1. Select the "default" virtual machine inside VirtualBox, then click
+        "Settings"
+    2. Go to "Shared Folders", click "Add shared foldrer" (the folder icon with
+        a plus sign), fill in the following information, then click "OK":
+
+        * Folder path: `E:\rust`
+        * Folder name: `e/rust`
+        * Read-only: ☐ *unchecked*
+        * Auto-mount: ☑ *checked*
+        * Make Permanant: ☑ *checked*
+
+3. VirtualBox might not support creating symbolic links inside a shared folder
+    by default. You can enable it manually by running these from `cmd.exe`:
+
+    ```bat
+    cd "C:\Program Files\Oracle\VirtualBox"
+    VBoxManage setextradata default VBoxInternal2/SharedFoldersEnableSymlinksCreate/e/rust 1
+    ::                                                                              ^~~~~~
+    ::                                                                              folder name
+    ```
+
+4. Restart the virtual machine from terminal with `docker-machine start`.
+
+To run the image,
+
+1. Launch the "Docker Quickstart Terminal".
+2. Execute `./src/ci/docker/run.sh $image_name` as explained at the beginning.
+
+[Docker Toolbox]: https://www.docker.com/products/docker-toolbox
+
 ## Cross toolchains
 
 A number of these images take quite a long time to compile as they're building
@@ -137,7 +179,7 @@ For targets: `armv7-unknown-linux-gnueabihf`
     libraries like jemalloc. See the mk/cfg/arm(v7)-uknown-linux-gnueabi{,hf}.mk
     file in Rust's source code.
 
-## `aarch64-linux-gnu.config`
+### `aarch64-linux-gnu.config`
 
 For targets: `aarch64-unknown-linux-gnu`
 
@@ -150,7 +192,7 @@ For targets: `aarch64-unknown-linux-gnu`
 - C compiler > gcc version = 5.2.0
 - C compiler > C++ = ENABLE -- to cross compile LLVM
 
-## `powerpc-linux-gnu.config`
+### `powerpc-linux-gnu.config`
 
 For targets: `powerpc-unknown-linux-gnu`
 
@@ -165,7 +207,7 @@ For targets: `powerpc-unknown-linux-gnu`
 - C compiler > gcc version = 4.9.3
 - C compiler > C++ = ENABLE -- to cross compile LLVM
 
-## `powerpc64-linux-gnu.config`
+### `powerpc64-linux-gnu.config`
 
 For targets: `powerpc64-unknown-linux-gnu`
 
@@ -184,7 +226,7 @@ For targets: `powerpc64-unknown-linux-gnu`
 
 (+) These CPU options match the configuration of the toolchains in RHEL6.
 
-## `s390x-linux-gnu.config`
+### `s390x-linux-gnu.config`
 
 For targets: `s390x-unknown-linux-gnu`
 

src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh

@@ -23,9 +23,9 @@ SYSROOT=/usr/local/$TARGET/sysroot
 mkdir -p $SYSROOT
 pushd $SYSROOT
 
-centos_base=http://mirror.centos.org/altarch/7/os/ppc64le/Packages
-glibc_v=2.17-196.el7
-kernel_v=3.10.0-693.el7
+centos_base=http://vault.centos.org/altarch/7.3.1611/os/ppc64le/Packages/
+glibc_v=2.17-157.el7
+kernel_v=3.10.0-514.el7
 for package in glibc{,-devel,-headers}-$glibc_v kernel-headers-$kernel_v; do
   curl $centos_base/$package.ppc64le.rpm | \
     rpm2cpio - | cpio -idm

src/ci/docker/run.sh

@@ -11,6 +11,8 @@
 
 set -e
 
+export MSYS_NO_PATHCONV=1
+
 script=`cd $(dirname $0) && pwd`/`basename $0`
 image=$1
 
@@ -25,12 +27,19 @@ travis_fold start build_docker
 travis_time_start
 
 if [ -f "$docker_dir/$image/Dockerfile" ]; then
+    dockerfile="$docker_dir/$image/Dockerfile"
+    if [ -x /usr/bin/cygpath ]; then
+        context="`cygpath -w $docker_dir`"
+        dockerfile="`cygpath -w $dockerfile`"
+    else
+        context="$docker_dir"
+    fi
     retry docker \
       build \
       --rm \
       -t rust-ci \
-      -f "$docker_dir/$image/Dockerfile" \
-      "$docker_dir"
+      -f "$dockerfile" \
+      "$context"
 elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
     if [ -n "$TRAVIS_OS_NAME" ]; then
         echo Cannot run disabled images on travis!

src/librustc_trans/back/link.rs

@@ -671,11 +671,12 @@ fn link_natively(sess: &Session,
             break
         }
 
-        sess.struct_warn("looks like the linker segfaulted when we tried to \
-                          call it, automatically retrying again")
-            .note(&format!("{:?}", cmd))
-            .note(&out)
-            .emit();
+        warn!(
+            "looks like the linker segfaulted when we tried to call it, \
+             automatically retrying again. cmd = {:?}, out = {}.",
+            cmd,
+            out,
+        );
     }
 
     match prog {

src/libtest/lib.rs

@@ -71,6 +71,7 @@ use std::thread;
 use std::time::{Instant, Duration};
 
 const TEST_WARN_TIMEOUT_S: u64 = 60;
+const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode
 
 // to be used by rustc to compile tests in libtest
 pub mod test {
@@ -614,7 +615,14 @@ impl<T: Write> ConsoleTestState<T> {
     pub fn write_short_result(&mut self, verbose: &str, quiet: &str, color: term::color::Color)
                               -> io::Result<()> {
         if self.quiet {
-            self.write_pretty(quiet, color)
+            self.write_pretty(quiet, color)?;
+            if self.current_test_count() % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 {
+                // we insert a new line every 100 dots in order to flush the
+                // screen when dealing with line-buffered output (e.g. piping to
+                // `stamp` in the rust CI).
+                self.write_plain("\n")?;
+            }
+            Ok(())
         } else {
             self.write_pretty(verbose, color)?;
             self.write_plain("\n")
@@ -771,9 +779,12 @@ impl<T: Write> ConsoleTestState<T> {
         Ok(())
     }
 
+    fn current_test_count(&self) -> usize {
+        self.passed + self.failed + self.ignored + self.measured + self.allowed_fail
+    }
+
     pub fn write_run_finish(&mut self) -> io::Result<bool> {
-        assert!(self.passed + self.failed + self.ignored + self.measured +
-                    self.allowed_fail == self.total);
+        assert!(self.current_test_count() == self.total);
 
         if self.options.display_output {
             self.write_outputs()?;

src/tools/compiletest/Cargo.toml

@@ -11,3 +11,7 @@ getopts = "0.2"
 log = "0.3"
 rustc-serialize = "0.3"
 libc = "0.2"
+
+[target.'cfg(windows)'.dependencies]
+miow = "0.2"
+winapi = "0.2"

src/tools/compiletest/src/main.rs

@@ -11,10 +11,11 @@
 #![crate_name = "compiletest"]
 
 #![feature(test)]
+#![feature(slice_rotate)]
 
 #![deny(warnings)]
 
-#[cfg(any(target_os = "macos", target_os = "ios"))]
+#[cfg(unix)]
 extern crate libc;
 extern crate test;
 extern crate getopts;
@@ -47,6 +48,7 @@ pub mod runtest;
 pub mod common;
 pub mod errors;
 mod raise_fd_limit;
+mod read2;
 
 fn main() {
     env_logger::init().unwrap();

src/tools/compiletest/src/read2.rs

@@ -0,0 +1,208 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// FIXME: This is a complete copy of `cargo/src/cargo/util/read2.rs`
+// Consider unify the read2() in libstd, cargo and this to prevent further code duplication.
+
+pub use self::imp::read2;
+
+#[cfg(not(any(unix, windows)))]
+mod imp {
+    use std::io::{self, Read};
+    use std::process::{ChildStdout, ChildStderr};
+
+    pub fn read2(out_pipe: ChildStdout,
+                 err_pipe: ChildStderr,
+                 data: &mut FnMut(bool, &mut Vec<u8>, bool)) -> io::Result<()> {
+        let mut buffer = Vec::new();
+        out_pipe.read_to_end(&mut buffer)?;
+        data(true, &mut buffer, true);
+        buffer.clear();
+        err_pipe.read_to_end(&mut buffer)?;
+        data(false, &mut buffer, true);
+        Ok(())
+    }
+}
+
+#[cfg(unix)]
+mod imp {
+    use std::io::prelude::*;
+    use std::io;
+    use std::mem;
+    use std::os::unix::prelude::*;
+    use std::process::{ChildStdout, ChildStderr};
+    use libc;
+
+    pub fn read2(mut out_pipe: ChildStdout,
+                 mut err_pipe: ChildStderr,
+                 data: &mut FnMut(bool, &mut Vec<u8>, bool)) -> io::Result<()> {
+        unsafe {
+            libc::fcntl(out_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK);
+            libc::fcntl(err_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK);
+        }
+
+        let mut out_done = false;
+        let mut err_done = false;
+        let mut out = Vec::new();
+        let mut err = Vec::new();
+
+        let mut fds: [libc::pollfd; 2] = unsafe { mem::zeroed() };
+        fds[0].fd = out_pipe.as_raw_fd();
+        fds[0].events = libc::POLLIN;
+        fds[1].fd = err_pipe.as_raw_fd();
+        fds[1].events = libc::POLLIN;
+        loop {
+            // wait for either pipe to become readable using `select`
+            let r = unsafe { libc::poll(fds.as_mut_ptr(), 2, -1) };
+            if r == -1 {
+                let err = io::Error::last_os_error();
+                if err.kind() == io::ErrorKind::Interrupted {
+                    continue
+                }
+                return Err(err)
+            }
+
+            // Read as much as we can from each pipe, ignoring EWOULDBLOCK or
+            // EAGAIN. If we hit EOF, then this will happen because the underlying
+            // reader will return Ok(0), in which case we'll see `Ok` ourselves. In
+            // this case we flip the other fd back into blocking mode and read
+            // whatever's leftover on that file descriptor.
+            let handle = |res: io::Result<_>| {
+                match res {
+                    Ok(_) => Ok(true),
+                    Err(e) => {
+                        if e.kind() == io::ErrorKind::WouldBlock {
+                            Ok(false)
+                        } else {
+                            Err(e)
+                        }
+                    }
+                }
+            };
+            if !out_done && fds[0].revents != 0 && handle(out_pipe.read_to_end(&mut out))? {
+                out_done = true;
+            }
+            data(true, &mut out, out_done);
+            if !err_done && fds[1].revents != 0 && handle(err_pipe.read_to_end(&mut err))? {
+                err_done = true;
+            }
+            data(false, &mut err, err_done);
+
+            if out_done && err_done {
+                return Ok(())
+            }
+        }
+    }
+}
+
+#[cfg(windows)]
+mod imp {
+    extern crate miow;
+    extern crate winapi;
+
+    use std::io;
+    use std::os::windows::prelude::*;
+    use std::process::{ChildStdout, ChildStderr};
+    use std::slice;
+
+    use self::miow::iocp::{CompletionPort, CompletionStatus};
+    use self::miow::pipe::NamedPipe;
+    use self::miow::Overlapped;
+    use self::winapi::ERROR_BROKEN_PIPE;
+
+    struct Pipe<'a> {
+        dst: &'a mut Vec<u8>,
+        overlapped: Overlapped,
+        pipe: NamedPipe,
+        done: bool,
+    }
+
+    pub fn read2(out_pipe: ChildStdout,
+                 err_pipe: ChildStderr,
+                 data: &mut FnMut(bool, &mut Vec<u8>, bool)) -> io::Result<()> {
+        let mut out = Vec::new();
+        let mut err = Vec::new();
+
+        let port = CompletionPort::new(1)?;
+        port.add_handle(0, &out_pipe)?;
+        port.add_handle(1, &err_pipe)?;
+
+        unsafe {
+            let mut out_pipe = Pipe::new(out_pipe, &mut out);
+            let mut err_pipe = Pipe::new(err_pipe, &mut err);
+
+            out_pipe.read()?;
+            err_pipe.read()?;
+
+            let mut status = [CompletionStatus::zero(), CompletionStatus::zero()];
+
+            while !out_pipe.done || !err_pipe.done {
+                for status in port.get_many(&mut status, None)? {
+                    if status.token() == 0 {
+                        out_pipe.complete(status);
+                        data(true, out_pipe.dst, out_pipe.done);
+                        out_pipe.read()?;
+                    } else {
+                        err_pipe.complete(status);
+                        data(false, err_pipe.dst, err_pipe.done);
+                        err_pipe.read()?;
+                    }
+                }
+            }
+
+            Ok(())
+        }
+    }
+
+    impl<'a> Pipe<'a> {
+        unsafe fn new<P: IntoRawHandle>(p: P, dst: &'a mut Vec<u8>) -> Pipe<'a> {
+            Pipe {
+                dst: dst,
+                pipe: NamedPipe::from_raw_handle(p.into_raw_handle()),
+                overlapped: Overlapped::zero(),
+                done: false,
+            }
+        }
+
+        unsafe fn read(&mut self) -> io::Result<()> {
+            let dst = slice_to_end(self.dst);
+            match self.pipe.read_overlapped(dst, self.overlapped.raw()) {
+                Ok(_) => Ok(()),
+                Err(e) => {
+                    if e.raw_os_error() == Some(ERROR_BROKEN_PIPE as i32) {
+                        self.done = true;
+                        Ok(())
+                    } else {
+                        Err(e)
+                    }
+                }
+            }
+        }
+
+        unsafe fn complete(&mut self, status: &CompletionStatus) {
+            let prev = self.dst.len();
+            self.dst.set_len(prev + status.bytes_transferred() as usize);
+            if status.bytes_transferred() == 0 {
+                self.done = true;
+            }
+        }
+    }
+
+    unsafe fn slice_to_end(v: &mut Vec<u8>) -> &mut [u8] {
+        if v.capacity() == 0 {
+            v.reserve(16);
+        }
+        if v.capacity() == v.len() {
+            v.reserve(1);
+        }
+        slice::from_raw_parts_mut(v.as_mut_ptr().offset(v.len() as isize),
+                                  v.capacity() - v.len())
+    }
+}

src/tools/compiletest/src/runtest.rs

@@ -29,7 +29,7 @@ use std::fmt;
 use std::io::prelude::*;
 use std::io::{self, BufReader};
 use std::path::{Path, PathBuf};
-use std::process::{Command, Output, ExitStatus, Stdio};
+use std::process::{Command, Output, ExitStatus, Stdio, Child};
 use std::str;
 
 use extract_gdb_version;
@@ -1344,12 +1344,14 @@ actual:\n\
         if let Some(input) = input {
             child.stdin.as_mut().unwrap().write_all(input.as_bytes()).unwrap();
         }
-        let Output { status, stdout, stderr } = child.wait_with_output().unwrap();
+
+        let Output { status, stdout, stderr } = read2_abbreviated(child)
+            .expect("failed to read output");
 
         let result = ProcRes {
             status,
-            stdout: String::from_utf8(stdout).unwrap(),
-            stderr: String::from_utf8(stderr).unwrap(),
+            stdout: String::from_utf8_lossy(&stdout).into_owned(),
+            stderr: String::from_utf8_lossy(&stderr).into_owned(),
             cmdline,
         };
 
@@ -1635,7 +1637,9 @@ actual:\n\
         cmd.arg("-a").arg("-u");
         cmd.arg(filename);
         cmd.arg("-nobanner");
-        let output = match cmd.output() {
+        cmd.stdout(Stdio::piped());
+        cmd.stderr(Stdio::piped());
+        let output = match cmd.spawn().and_then(read2_abbreviated) {
             Ok(output) => output,
             Err(_) => return,
         };
@@ -2095,6 +2099,8 @@ actual:\n\
 
         let mut cmd = Command::new(make);
         cmd.current_dir(&self.testpaths.file)
+           .stdout(Stdio::piped())
+           .stderr(Stdio::piped())
            .env("TARGET", &self.config.target)
            .env("PYTHON", &self.config.docck_python)
            .env("S", src_root)
@@ -2143,7 +2149,7 @@ actual:\n\
             }
         }
 
-        let output = cmd.output().expect("failed to spawn `make`");
+        let output = cmd.spawn().and_then(read2_abbreviated).expect("failed to spawn `make`");
         if !output.status.success() {
             let res = ProcRes {
                 status: output.status,
@@ -2536,3 +2542,76 @@ fn nocomment_mir_line(line: &str) -> &str {
         line
     }
 }
+
+fn read2_abbreviated(mut child: Child) -> io::Result<Output> {
+    use std::mem::replace;
+    use read2::read2;
+
+    const HEAD_LEN: usize = 160 * 1024;
+    const TAIL_LEN: usize = 256 * 1024;
+
+    enum ProcOutput {
+        Full(Vec<u8>),
+        Abbreviated {
+            head: Vec<u8>,
+            skipped: usize,
+            tail: Box<[u8]>,
+        }
+    }
+
+    impl ProcOutput {
+        fn extend(&mut self, data: &[u8]) {
+            let new_self = match *self {
+                ProcOutput::Full(ref mut bytes) => {
+                    bytes.extend_from_slice(data);
+                    let new_len = bytes.len();
+                    if new_len <= HEAD_LEN + TAIL_LEN {
+                        return;
+                    }
+                    let tail = bytes.split_off(new_len - TAIL_LEN).into_boxed_slice();
+                    let head = replace(bytes, Vec::new());
+                    let skipped = new_len - HEAD_LEN - TAIL_LEN;
+                    ProcOutput::Abbreviated { head, skipped, tail }
+                }
+                ProcOutput::Abbreviated { ref mut skipped, ref mut tail, .. } => {
+                    *skipped += data.len();
+                    if data.len() <= TAIL_LEN {
+                        tail[..data.len()].copy_from_slice(data);
+                        tail.rotate(data.len());
+                    } else {
+                        tail.copy_from_slice(&data[(data.len() - TAIL_LEN)..]);
+                    }
+                    return;
+                }
+            };
+            *self = new_self;
+        }
+
+        fn into_bytes(self) -> Vec<u8> {
+            match self {
+                ProcOutput::Full(bytes) => bytes,
+                ProcOutput::Abbreviated { mut head, skipped, tail } => {
+                    write!(&mut head, "\n\n<<<<<< SKIPPED {} BYTES >>>>>>\n\n", skipped).unwrap();
+                    head.extend_from_slice(&tail);
+                    head
+                }
+            }
+        }
+    }
+
+    let mut stdout = ProcOutput::Full(Vec::new());
+    let mut stderr = ProcOutput::Full(Vec::new());
+
+    drop(child.stdin.take());
+    read2(child.stdout.take().unwrap(), child.stderr.take().unwrap(), &mut |is_stdout, data, _| {
+        if is_stdout { &mut stdout } else { &mut stderr }.extend(data);
+        data.clear();
+    })?;
+    let status = child.wait()?;
+
+    Ok(Output {
+        status,
+        stdout: stdout.into_bytes(),
+        stderr: stderr.into_bytes(),
+    })
+}

src/tools/tidy/src/bins.rs

@@ -31,9 +31,9 @@ pub fn check(path: &Path, bad: &mut bool) {
     if let Ok(mut file) = fs::File::open("/proc/version") {
         let mut contents = String::new();
         file.read_to_string(&mut contents).unwrap();
-        // Probably on Windows Linux Subsystem, all files will be marked as
-        // executable, so skip checking.
-        if contents.contains("Microsoft") {
+        // Probably on Windows Linux Subsystem or Docker via VirtualBox,
+        // all files will be marked as executable, so skip checking.
+        if contents.contains("Microsoft") || contents.contains("boot2docker") {
             return;
         }
     }