rust/src/librustc_llvm/build.rs
Alex Crichton 43e8ac27d9 rustc: Persist LLVM's Linker in Fat LTO
This commit updates our Fat LTO logic to tweak our custom wrapper around LLVM's
"link modules" functionality. Previously whenever the
`LLVMRustLinkInExternalBitcode` function was called it would call LLVM's
`Linker::linkModules` wrapper. Internally this would crate an instance of a
`Linker` which internally creates an instance of an `IRMover`. Unfortunately for
us the creation of `IRMover` is somewhat O(n) with the input module. This means
that every time we linked a module it was O(n) with respect to the entire module
we had built up!

Now the modules we build up during LTO are quite large, so this quickly started
creating an O(n^2) problem for us! Discovered in #48025 it turns out this has
always been a problem and we just haven't noticed it. It became particularly
worse recently though due to most libraries having 16x more object files than
they previously did (1 -> 16).

This commit fixes this performance issue by preserving the `Linker` instance
across all links into the main LLVM module. This means we only create one
`IRMover` and allows LTO to progress much speedier.

From the `cargo-cache` project in #48025 a **full build** locally when from
5m15s to 2m24s. Looking at the timing logs each object file was linked in in
single-digit millisecond rather than hundreds, clearly being a nice improvement!

Closes #48025
2018-02-12 09:11:06 -08:00

270 lines
10 KiB
Rust

// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
extern crate cc;
extern crate build_helper;
use std::process::Command;
use std::env;
use std::path::{PathBuf, Path};
use build_helper::output;
fn detect_llvm_link() -> (&'static str, &'static str) {
// Force the link mode we want, preferring static by default, but
// possibly overridden by `configure --enable-llvm-link-shared`.
if env::var_os("LLVM_LINK_SHARED").is_some() {
("dylib", "--link-shared")
} else {
("static", "--link-static")
}
}
fn main() {
let target = env::var("TARGET").expect("TARGET was not set");
let llvm_config = env::var_os("LLVM_CONFIG")
.map(PathBuf::from)
.unwrap_or_else(|| {
if let Some(dir) = env::var_os("CARGO_TARGET_DIR").map(PathBuf::from) {
let to_test = dir.parent()
.unwrap()
.parent()
.unwrap()
.join(&target)
.join("llvm/bin/llvm-config");
if Command::new(&to_test).output().is_ok() {
return to_test;
}
}
PathBuf::from("llvm-config")
});
println!("cargo:rerun-if-changed={}", llvm_config.display());
println!("cargo:rerun-if-env-changed=LLVM_CONFIG");
// Test whether we're cross-compiling LLVM. This is a pretty rare case
// currently where we're producing an LLVM for a different platform than
// what this build script is currently running on.
//
// In that case, there's no guarantee that we can actually run the target,
// so the build system works around this by giving us the LLVM_CONFIG for
// the host platform. This only really works if the host LLVM and target
// LLVM are compiled the same way, but for us that's typically the case.
//
// We *want* detect this cross compiling situation by asking llvm-config
// what it's host-target is. If that's not the TARGET, then we're cross
// compiling. Unfortunately `llvm-config` seems either be buggy, or we're
// misconfiguring it, because the `i686-pc-windows-gnu` build of LLVM will
// report itself with a `--host-target` of `x86_64-pc-windows-gnu`. This
// tricks us into thinking we're doing a cross build when we aren't, so
// havoc ensues.
//
// In any case, if we're cross compiling, this generally just means that we
// can't trust all the output of llvm-config becaues it might be targeted
// for the host rather than the target. As a result a bunch of blocks below
// are gated on `if !is_crossed`
let target = env::var("TARGET").expect("TARGET was not set");
let host = env::var("HOST").expect("HOST was not set");
let is_crossed = target != host;
let mut optional_components =
vec!["x86", "arm", "aarch64", "mips", "powerpc",
"systemz", "jsbackend", "webassembly", "msp430", "sparc", "nvptx"];
let mut version_cmd = Command::new(&llvm_config);
version_cmd.arg("--version");
let version_output = output(&mut version_cmd);
let mut parts = version_output.split('.').take(2)
.filter_map(|s| s.parse::<u32>().ok());
let (major, _minor) =
if let (Some(major), Some(minor)) = (parts.next(), parts.next()) {
(major, minor)
} else {
(3, 9)
};
if major > 3 {
optional_components.push("hexagon");
}
// FIXME: surely we don't need all these components, right? Stuff like mcjit
// or interpreter the compiler itself never uses.
let required_components = &["ipo",
"bitreader",
"bitwriter",
"linker",
"asmparser",
"mcjit",
"lto",
"interpreter",
"instrumentation"];
let components = output(Command::new(&llvm_config).arg("--components"));
let mut components = components.split_whitespace().collect::<Vec<_>>();
components.retain(|c| optional_components.contains(c) || required_components.contains(c));
for component in required_components {
if !components.contains(component) {
panic!("require llvm component {} but wasn't found", component);
}
}
for component in components.iter() {
println!("cargo:rustc-cfg=llvm_component=\"{}\"", component);
}
// Link in our own LLVM shims, compiled with the same flags as LLVM
let mut cmd = Command::new(&llvm_config);
cmd.arg("--cxxflags");
let cxxflags = output(&mut cmd);
let mut cfg = cc::Build::new();
cfg.warnings(false);
for flag in cxxflags.split_whitespace() {
// Ignore flags like `-m64` when we're doing a cross build
if is_crossed && flag.starts_with("-m") {
continue;
}
// -Wdate-time is not supported by the netbsd cross compiler
if is_crossed && target.contains("netbsd") && flag.contains("date-time") {
continue;
}
cfg.flag(flag);
}
for component in &components {
let mut flag = String::from("LLVM_COMPONENT_");
flag.push_str(&component.to_uppercase());
cfg.define(&flag, None);
}
println!("cargo:rerun-if-changed-env=LLVM_RUSTLLVM");
if env::var_os("LLVM_RUSTLLVM").is_some() {
cfg.define("LLVM_RUSTLLVM", None);
}
build_helper::rerun_if_changed_anything_in_dir(Path::new("../rustllvm"));
cfg.file("../rustllvm/PassWrapper.cpp")
.file("../rustllvm/RustWrapper.cpp")
.file("../rustllvm/ArchiveWrapper.cpp")
.file("../rustllvm/Linker.cpp")
.cpp(true)
.cpp_link_stdlib(None) // we handle this below
.compile("rustllvm");
let (llvm_kind, llvm_link_arg) = detect_llvm_link();
// Link in all LLVM libraries, if we're uwring the "wrong" llvm-config then
// we don't pick up system libs because unfortunately they're for the host
// of llvm-config, not the target that we're attempting to link.
let mut cmd = Command::new(&llvm_config);
cmd.arg(llvm_link_arg).arg("--libs");
if !is_crossed {
cmd.arg("--system-libs");
}
cmd.args(&components);
for lib in output(&mut cmd).split_whitespace() {
let name = if lib.starts_with("-l") {
&lib[2..]
} else if lib.starts_with("-") {
&lib[1..]
} else if Path::new(lib).exists() {
// On MSVC llvm-config will print the full name to libraries, but
// we're only interested in the name part
let name = Path::new(lib).file_name().unwrap().to_str().unwrap();
name.trim_right_matches(".lib")
} else if lib.ends_with(".lib") {
// Some MSVC libraries just come up with `.lib` tacked on, so chop
// that off
lib.trim_right_matches(".lib")
} else {
continue;
};
// Don't need or want this library, but LLVM's CMake build system
// doesn't provide a way to disable it, so filter it here even though we
// may or may not have built it. We don't reference anything from this
// library and it otherwise may just pull in extra dependencies on
// libedit which we don't want
if name == "LLVMLineEditor" {
continue;
}
let kind = if name.starts_with("LLVM") {
llvm_kind
} else {
"dylib"
};
println!("cargo:rustc-link-lib={}={}", kind, name);
}
// LLVM ldflags
//
// If we're a cross-compile of LLVM then unfortunately we can't trust these
// ldflags (largely where all the LLVM libs are located). Currently just
// hack around this by replacing the host triple with the target and pray
// that those -L directories are the same!
let mut cmd = Command::new(&llvm_config);
cmd.arg(llvm_link_arg).arg("--ldflags");
for lib in output(&mut cmd).split_whitespace() {
if lib.starts_with("-LIBPATH:") {
println!("cargo:rustc-link-search=native={}", &lib[9..]);
} else if is_crossed {
if lib.starts_with("-L") {
println!("cargo:rustc-link-search=native={}",
lib[2..].replace(&host, &target));
}
} else if lib.starts_with("-l") {
println!("cargo:rustc-link-lib={}", &lib[2..]);
} else if lib.starts_with("-L") {
println!("cargo:rustc-link-search=native={}", &lib[2..]);
}
}
let llvm_static_stdcpp = env::var_os("LLVM_STATIC_STDCPP");
let stdcppname = if target.contains("openbsd") {
// llvm-config on OpenBSD doesn't mention stdlib=libc++
"c++"
} else if target.contains("freebsd") {
"c++"
} else if target.contains("netbsd") && llvm_static_stdcpp.is_some() {
// NetBSD uses a separate library when relocation is required
"stdc++_pic"
} else {
"stdc++"
};
// C++ runtime library
if !target.contains("msvc") {
if let Some(s) = llvm_static_stdcpp {
assert!(!cxxflags.contains("stdlib=libc++"));
let path = PathBuf::from(s);
println!("cargo:rustc-link-search=native={}",
path.parent().unwrap().display());
println!("cargo:rustc-link-lib=static={}", stdcppname);
} else if cxxflags.contains("stdlib=libc++") {
println!("cargo:rustc-link-lib=c++");
} else {
println!("cargo:rustc-link-lib={}", stdcppname);
}
}
// LLVM requires symbols from this library, but apparently they're not printed
// during llvm-config?
if target.contains("windows-gnu") {
println!("cargo:rustc-link-lib=static-nobundle=gcc_s");
println!("cargo:rustc-link-lib=static-nobundle=pthread");
}
}