mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rollup merge of #56944 - alexcrichton:less-thin2, r=michaelwoerister

Browse Source 
bootstrap: Link LLVM as a dylib with ThinLTO

When building a distributed compiler on Linux where we use ThinLTO to
create the LLVM shared object this commit switches the compiler to
dynamically linking that LLVM artifact instead of statically linking to
LLVM. The primary goal here is to reduce CI compile times, avoiding two+
ThinLTO builds of all of LLVM. By linking dynamically to LLVM we'll
reuse the one ThinLTO step done by LLVM's build itself.

Lots of discussion about this change can be found [here] and down. A
perf run will show whether this is worth it or not!

[here]: https://github.com/rust-lang/rust/pull/53245#issuecomment-417015334
This commit is contained in:
Mazdak Farrokhzad 2018-12-24 13:29:28 +01:00 committed by GitHub
commit f1051b574c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 18 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/bootstrap/check.rs
View File

@ -48,7 +48,6 @@ fn run(self, builder: &Builder) {
builder.info(&format!("Checking std artifacts ({} -> {})", &compiler.host, target));
run_cargo(builder,
&mut cargo,
vec![],
&libstd_stamp(builder, compiler, target),
true);
@ -95,7 +94,6 @@ fn run(self, builder: &Builder) {
builder.info(&format!("Checking compiler artifacts ({} -> {})", &compiler.host, target));
run_cargo(builder,
&mut cargo,
vec![],
&librustc_stamp(builder, compiler, target),
true);
@ -146,7 +144,6 @@ fn run(self, builder: &Builder) {
let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
run_cargo(builder,
&mut cargo,
vec![],
&codegen_backend_stamp(builder, compiler, target, backend),
true);
}
@ -184,7 +181,6 @@ fn run(self, builder: &Builder) {
builder.info(&format!("Checking test artifacts ({} -> {})", &compiler.host, target));
run_cargo(builder,
&mut cargo,
vec![],
&libtest_stamp(builder, compiler, target),
true);
@ -232,7 +228,6 @@ fn run(self, builder: &Builder) {
println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target);
run_cargo(builder,
&mut cargo,
vec![],
&rustdoc_stamp(builder, compiler, target),
true);

49
src/bootstrap/compile.rs
View File

@ -29,6 +29,7 @@
use filetime::FileTime;
use serde_json;
use crate::dist;
use crate::util::{exe, libdir, is_dylib};
use crate::{Compiler, Mode, GitRepo};
use crate::native;
@ -114,7 +115,6 @@ fn run(self, builder: &Builder) {
&compiler.host, target));
run_cargo(builder,
&mut cargo,
vec![],
&libstd_stamp(builder, compiler, target),
false);
@ -375,7 +375,6 @@ fn run(self, builder: &Builder) {
&compiler.host, target));
run_cargo(builder,
&mut cargo,
vec![],
&libtest_stamp(builder, compiler, target),
false);
@ -503,7 +502,6 @@ fn run(self, builder: &Builder) {
compiler.stage, &compiler.host, target));
run_cargo(builder,
&mut cargo,
vec![],
&librustc_stamp(builder, compiler, target),
false);
@ -646,47 +644,18 @@ fn run(self, builder: &Builder) {
let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "rustc");
let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "build");
cargo.arg("--manifest-path")
.arg(builder.src.join("src/librustc_codegen_llvm/Cargo.toml"));
rustc_cargo_env(builder, &mut cargo);
let features = build_codegen_backend(&builder, &mut cargo, &compiler, target, backend);
let mut cargo_tails_args = vec![];
if builder.config.llvm_thin_lto {
cargo_tails_args.push("--".to_string());
let num_jobs = builder.jobs();
if !target.contains("msvc") {
// Here we assume that the linker is clang. If it's not, there'll
// be linker errors.
cargo_tails_args.push("-Clink-arg=-fuse-ld=lld".to_string());
cargo_tails_args.push("-Clink-arg=-flto=thin".to_string());
if builder.config.llvm_optimize {
cargo_tails_args.push("-Clink-arg=-O2".to_string());
}
// Let's make LLD respect the `-j` option.
let num_jobs_arg = format!("-Clink-arg=-Wl,--thinlto-jobs={}", num_jobs);
cargo_tails_args.push(num_jobs_arg);
} else {
// Here we assume that the linker is lld-link.exe. lld-link.exe
// does not need the extra arguments except for num_jobs
let num_jobs_arg = format!("-Clink-arg=/opt:lldltojobs={}", num_jobs);
cargo_tails_args.push(num_jobs_arg);
}
}
let tmp_stamp = out_dir.join(".tmp.stamp");
let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
let files = run_cargo(builder,
cargo.arg("--features").arg(features),
cargo_tails_args,
&tmp_stamp,
false);
if builder.config.dry_run {
@ -759,7 +728,9 @@ pub fn build_codegen_backend(builder: &Builder,
"libstdc++.a");
cargo.env("LLVM_STATIC_STDCPP", file);
}
if builder.config.llvm_link_shared {
if builder.config.llvm_link_shared ||
(builder.config.llvm_thin_lto && backend != "emscripten")
{
cargo.env("LLVM_LINK_SHARED", "1");
}
}
@ -999,6 +970,8 @@ fn run(self, builder: &Builder) -> Compiler {
copy_lld_to_sysroot(builder, target_compiler, &lld_install);
}
dist::maybe_install_llvm_dylib(builder, target_compiler.host, &sysroot);
// Link the compiler binary itself into place
let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host);
let rustc = out_dir.join(exe("rustc_binary", &*host));
@ -1025,7 +998,6 @@ pub fn add_to_sysroot(builder: &Builder, sysroot_dst: &Path, stamp: &Path) {
pub fn run_cargo(builder: &Builder,
cargo: &mut Command,
tail_args: Vec<String>,
stamp: &Path,
is_check: bool)
-> Vec<PathBuf>
@ -1048,7 +1020,7 @@ pub fn run_cargo(builder: &Builder,
// files we need to probe for later.
let mut deps = Vec::new();
let mut toplevel = Vec::new();
let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| {
let ok = stream_cargo(builder, cargo, &mut |msg| {
let filenames = match msg {
CargoMessage::CompilerArtifact { filenames, .. } => filenames,
_ => return,
@ -1173,7 +1145,6 @@ pub fn run_cargo(builder: &Builder,
pub fn stream_cargo(
builder: &Builder,
cargo: &mut Command,
tail_args: Vec<String>,
cb: &mut dyn FnMut(CargoMessage),
) -> bool {
if builder.config.dry_run {
@ -1184,10 +1155,6 @@ pub fn stream_cargo(
cargo.arg("--message-format").arg("json")
.stdout(Stdio::piped());
for arg in tail_args {
cargo.arg(arg);
}
builder.verbose(&format!("running: {:?}", cargo));
let mut child = match cargo.spawn() {
Ok(child) => child,

8
src/bootstrap/dist.rs
View File

@ -1888,13 +1888,13 @@ fn run(self, builder: &Builder) {
// LLVM tools are linked dynamically.
// Note: This function does no yet support Windows but we also don't support
// linking LLVM tools dynamically on Windows yet.
fn maybe_install_llvm_dylib(builder: &Builder,
target: Interned<String>,
image: &Path) {
pub fn maybe_install_llvm_dylib(builder: &Builder,
target: Interned<String>,
sysroot: &Path) {
let src_libdir = builder
.llvm_out(target)
.join("lib");
let dst_libdir = image.join("lib/rustlib").join(&*target).join("lib");
let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
t!(fs::create_dir_all(&dst_libdir));
if target.contains("apple-darwin") {

2
src/bootstrap/tool.rs
View File

@ -87,7 +87,7 @@ fn run(self, builder: &Builder) -> Option<PathBuf> {
let _folder = builder.fold_output(|| format!("stage{}-{}", compiler.stage, tool));
builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
let mut duplicates = Vec::new();
let is_expected = compile::stream_cargo(builder, &mut cargo, vec![], &mut |msg| {
let is_expected = compile::stream_cargo(builder, &mut cargo, &mut |msg| {
// Only care about big things like the RLS/Cargo for now
match tool {
| "rls"

9
src/librustc_driver/lib.rs
View File

@ -218,14 +218,7 @@ pub fn run<F>(run_compiler: F) -> isize
}
fn load_backend_from_dylib(path: &Path) -> fn() -> Box<dyn CodegenBackend> {
// Note that we're specifically using `open_global_now` here rather than
// `open`, namely we want the behavior on Unix of RTLD_GLOBAL and RTLD_NOW,
// where NOW means "bind everything right now" because we don't want
// surprises later on and RTLD_GLOBAL allows the symbols to be made
// available for future dynamic libraries opened. This is currently used by
// loading LLVM and then making its symbols available for other dynamic
// libraries.
let lib = DynamicLibrary::open_global_now(path).unwrap_or_else(|err| {
let lib = DynamicLibrary::open(Some(path)).unwrap_or_else(|err| {
let err = format!("couldn't load codegen backend {:?}: {:?}", path, err);
early_error(ErrorOutputType::default(), &err);
});

4
src/librustc_llvm/build.rs
View File

@ -142,6 +142,10 @@ fn main() {
continue;
}
if flag.starts_with("-flto") {
continue;
}
// -Wdate-time is not supported by the netbsd cross compiler
if is_crossed && target.contains("netbsd") && flag.contains("date-time") {
continue;

28
src/test/run-make-fulldeps/llvm-pass/Makefile
View File

@ -1,28 +0,0 @@
-include ../tools.mk
ifeq ($(UNAME),Darwin)
PLUGIN_FLAGS := -C link-args=-Wl,-undefined,dynamic_lookup
endif
ifeq ($(findstring stage1,$(RUST_BUILD_STAGE)),stage1)
# ignore stage1
all:
else
# Windows doesn't correctly handle include statements with escaping paths,
# so this test will not get run on Windows.
ifdef IS_WINDOWS
all:
else
all: $(call NATIVE_STATICLIB,llvm-function-pass) $(call NATIVE_STATICLIB,llvm-module-pass)
$(RUSTC) plugin.rs -C prefer-dynamic $(PLUGIN_FLAGS)
$(RUSTC) main.rs
$(TMPDIR)/libllvm-function-pass.o:
$(CXX) $(CFLAGS) $(LLVM_CXXFLAGS) -c llvm-function-pass.so.cc -o $(TMPDIR)/libllvm-function-pass.o
$(TMPDIR)/libllvm-module-pass.o:
$(CXX) $(CFLAGS) $(LLVM_CXXFLAGS) -c llvm-module-pass.so.cc -o $(TMPDIR)/libllvm-module-pass.o
endif
endif

56
src/test/run-make-fulldeps/llvm-pass/llvm-function-pass.so.cc
View File

@ -1,56 +0,0 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include "llvm/Pass.h"
#include "llvm/IR/Function.h"
using namespace llvm;
namespace {
class TestLLVMPass : public FunctionPass {
public:
static char ID;
TestLLVMPass() : FunctionPass(ID) { }
bool runOnFunction(Function &F) override;
StringRef getPassName() const override {
return "Some LLVM pass";
}
};
}
bool TestLLVMPass::runOnFunction(Function &F) {
// A couple examples of operations that previously caused segmentation faults
// https://github.com/rust-lang/rust/issues/31067
for (auto N = F.begin(); N != F.end(); ++N) {
/* code */
}
LLVMContext &C = F.getContext();
IntegerType *Int8Ty = IntegerType::getInt8Ty(C);
PointerType::get(Int8Ty, 0);
return true;
}
char TestLLVMPass::ID = 0;
static RegisterPass<TestLLVMPass> RegisterAFLPass(
"some-llvm-function-pass", "Some LLVM pass");

55
src/test/run-make-fulldeps/llvm-pass/llvm-module-pass.so.cc
View File

@ -1,55 +0,0 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include "llvm/IR/Module.h"
using namespace llvm;
namespace {
class TestLLVMPass : public ModulePass {
public:
static char ID;
TestLLVMPass() : ModulePass(ID) { }
bool runOnModule(Module &M) override;
StringRef getPassName() const override {
return "Some LLVM pass";
}
};
}
bool TestLLVMPass::runOnModule(Module &M) {
// A couple examples of operations that previously caused segmentation faults
// https://github.com/rust-lang/rust/issues/31067
for (auto F = M.begin(); F != M.end(); ++F) {
/* code */
}
LLVMContext &C = M.getContext();
IntegerType *Int8Ty = IntegerType::getInt8Ty(C);
PointerType::get(Int8Ty, 0);
return true;
}
char TestLLVMPass::ID = 0;
static RegisterPass<TestLLVMPass> RegisterAFLPass(
"some-llvm-module-pass", "Some LLVM pass");

14
src/test/run-make-fulldeps/llvm-pass/main.rs
View File

@ -1,14 +0,0 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#![feature(plugin)]
#![plugin(some_plugin)]
fn main() {}

28
src/test/run-make-fulldeps/llvm-pass/plugin.rs
View File

@ -1,28 +0,0 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#![feature(plugin_registrar, rustc_private)]
#![crate_type = "dylib"]
#![crate_name = "some_plugin"]
extern crate rustc;
extern crate rustc_plugin;
#[link(name = "llvm-function-pass", kind = "static")]
#[link(name = "llvm-module-pass", kind = "static")]
extern {}
use rustc_plugin::registry::Registry;
#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
reg.register_llvm_pass("some-llvm-function-pass");
reg.register_llvm_pass("some-llvm-module-pass");
}