From 5d531aeaf46451f999aea0f7c9ba08016b9ee3eb Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 16 Sep 2019 13:34:57 -0700 Subject: [PATCH] rustc: Convert `dependency_formats` to a query This commit converts a field of `Session`, `dependency_formats`, into a query of `TyCtxt`. This information then also needed to be threaded through to other remaining portions of the linker, but it's relatively straightforward. The only change here is that instead of `HashMap` the data structure changed to `Vec<(CrateType, T)>` to make it easier to deal with in queries. --- src/librustc/middle/dependency_format.rs | 378 +-------------------- src/librustc/query/mod.rs | 4 + src/librustc/session/config.rs | 2 +- src/librustc/session/mod.rs | 3 - src/librustc_codegen_ssa/back/link.rs | 61 ++-- src/librustc_codegen_ssa/back/linker.rs | 12 +- src/librustc_codegen_ssa/back/write.rs | 2 +- src/librustc_codegen_ssa/base.rs | 3 +- src/librustc_codegen_ssa/lib.rs | 2 + src/librustc_interface/passes.rs | 4 - src/librustc_metadata/cstore_impl.rs | 5 + src/librustc_metadata/dependency_format.rs | 371 ++++++++++++++++++++ src/librustc_metadata/encoder.rs | 26 +- src/librustc_metadata/lib.rs | 1 + 14 files changed, 455 insertions(+), 419 deletions(-) create mode 100644 src/librustc_metadata/dependency_format.rs diff --git a/src/librustc/middle/dependency_format.rs b/src/librustc/middle/dependency_format.rs index 96b99fe4cdc..8b2bf55ccc1 100644 --- a/src/librustc/middle/dependency_format.rs +++ b/src/librustc/middle/dependency_format.rs @@ -1,64 +1,10 @@ -//! Resolution of mixing rlibs and dylibs +//! Type definitions for learning about the dependency formats of all upstream +//! crates (rlibs/dylibs/oh my). //! -//! When producing a final artifact, such as a dynamic library, the compiler has -//! a choice between linking an rlib or linking a dylib of all upstream -//! dependencies. The linking phase must guarantee, however, that a library only -//! show up once in the object file. For example, it is illegal for library A to -//! be statically linked to B and C in separate dylibs, and then link B and C -//! into a crate D (because library A appears twice). -//! -//! The job of this module is to calculate what format each upstream crate -//! should be used when linking each output type requested in this session. This -//! generally follows this set of rules: -//! -//! 1. Each library must appear exactly once in the output. -//! 2. Each rlib contains only one library (it's just an object file) -//! 3. Each dylib can contain more than one library (due to static linking), -//! and can also bring in many dynamic dependencies. -//! -//! With these constraints in mind, it's generally a very difficult problem to -//! find a solution that's not "all rlibs" or "all dylibs". I have suspicions -//! that NP-ness may come into the picture here... -//! -//! The current selection algorithm below looks mostly similar to: -//! -//! 1. If static linking is required, then require all upstream dependencies -//! to be available as rlibs. If not, generate an error. -//! 2. If static linking is requested (generating an executable), then -//! attempt to use all upstream dependencies as rlibs. If any are not -//! found, bail out and continue to step 3. -//! 3. Static linking has failed, at least one library must be dynamically -//! linked. Apply a heuristic by greedily maximizing the number of -//! dynamically linked libraries. -//! 4. Each upstream dependency available as a dynamic library is -//! registered. The dependencies all propagate, adding to a map. It is -//! possible for a dylib to add a static library as a dependency, but it -//! is illegal for two dylibs to add the same static library as a -//! dependency. The same dylib can be added twice. Additionally, it is -//! illegal to add a static dependency when it was previously found as a -//! dylib (and vice versa) -//! 5. After all dynamic dependencies have been traversed, re-traverse the -//! remaining dependencies and add them statically (if they haven't been -//! added already). -//! -//! While not perfect, this algorithm should help support use-cases such as leaf -//! dependencies being static while the larger tree of inner dependencies are -//! all dynamic. This isn't currently very well battle tested, so it will likely -//! fall short in some use cases. -//! -//! Currently, there is no way to specify the preference of linkage with a -//! particular library (other than a global dynamic/static switch). -//! Additionally, the algorithm is geared towards finding *any* solution rather -//! than finding a number of solutions (there are normally quite a few). - -use crate::hir::def_id::CrateNum; +//! For all the gory details, see the provider of the `dependency_formats` +//! query. use crate::session::config; -use crate::ty::TyCtxt; -use crate::middle::cstore::{self, DepKind}; -use crate::middle::cstore::LinkagePreference::{self, RequireStatic, RequireDynamic}; -use crate::util::nodemap::FxHashMap; -use rustc_target::spec::PanicStrategy; /// A list of dependencies for a certain crate type. /// @@ -71,324 +17,12 @@ /// A mapping of all required dependencies for a particular flavor of output. /// /// This is local to the tcx, and is generally relevant to one session. -pub type Dependencies = FxHashMap; +pub type Dependencies = Vec<(config::CrateType, DependencyList)>; -#[derive(Copy, Clone, PartialEq, Debug)] +#[derive(Copy, Clone, PartialEq, Debug, HashStable)] pub enum Linkage { NotLinked, IncludedFromDylib, Static, Dynamic, } - -pub fn calculate(tcx: TyCtxt<'_>) { - let sess = &tcx.sess; - let fmts = sess.crate_types.borrow().iter().map(|&ty| { - let linkage = calculate_type(tcx, ty); - verify_ok(tcx, &linkage); - (ty, linkage) - }).collect::>(); - sess.abort_if_errors(); - sess.dependency_formats.set(fmts); -} - -fn calculate_type(tcx: TyCtxt<'_>, ty: config::CrateType) -> DependencyList { - let sess = &tcx.sess; - - if !sess.opts.output_types.should_codegen() { - return Vec::new(); - } - - let preferred_linkage = match ty { - // cdylibs must have all static dependencies. - config::CrateType::Cdylib => Linkage::Static, - - // Generating a dylib without `-C prefer-dynamic` means that we're going - // to try to eagerly statically link all dependencies. This is normally - // done for end-product dylibs, not intermediate products. - config::CrateType::Dylib if !sess.opts.cg.prefer_dynamic => Linkage::Static, - config::CrateType::Dylib => Linkage::Dynamic, - - // If the global prefer_dynamic switch is turned off, or the final - // executable will be statically linked, prefer static crate linkage. - config::CrateType::Executable if !sess.opts.cg.prefer_dynamic || - sess.crt_static() => Linkage::Static, - config::CrateType::Executable => Linkage::Dynamic, - - // proc-macro crates are mostly cdylibs, but we also need metadata. - config::CrateType::ProcMacro => Linkage::Static, - - // No linkage happens with rlibs, we just needed the metadata (which we - // got long ago), so don't bother with anything. - config::CrateType::Rlib => Linkage::NotLinked, - - // staticlibs must have all static dependencies. - config::CrateType::Staticlib => Linkage::Static, - }; - - if preferred_linkage == Linkage::NotLinked { - // If the crate is not linked, there are no link-time dependencies. - return Vec::new(); - } - - if preferred_linkage == Linkage::Static { - // Attempt static linkage first. For dylibs and executables, we may be - // able to retry below with dynamic linkage. - if let Some(v) = attempt_static(tcx) { - return v; - } - - // Staticlibs, cdylibs, and static executables must have all static - // dependencies. If any are not found, generate some nice pretty errors. - if ty == config::CrateType::Cdylib || ty == config::CrateType::Staticlib || - (ty == config::CrateType::Executable && sess.crt_static() && - !sess.target.target.options.crt_static_allows_dylibs) { - for &cnum in tcx.crates().iter() { - if tcx.dep_kind(cnum).macros_only() { continue } - let src = tcx.used_crate_source(cnum); - if src.rlib.is_some() { continue } - sess.err(&format!("crate `{}` required to be available in rlib format, \ - but was not found in this form", - tcx.crate_name(cnum))); - } - return Vec::new(); - } - } - - let mut formats = FxHashMap::default(); - - // Sweep all crates for found dylibs. Add all dylibs, as well as their - // dependencies, ensuring there are no conflicts. The only valid case for a - // dependency to be relied upon twice is for both cases to rely on a dylib. - for &cnum in tcx.crates().iter() { - if tcx.dep_kind(cnum).macros_only() { continue } - let name = tcx.crate_name(cnum); - let src = tcx.used_crate_source(cnum); - if src.dylib.is_some() { - info!("adding dylib: {}", name); - add_library(tcx, cnum, RequireDynamic, &mut formats); - let deps = tcx.dylib_dependency_formats(cnum); - for &(depnum, style) in deps.iter() { - info!("adding {:?}: {}", style, tcx.crate_name(depnum)); - add_library(tcx, depnum, style, &mut formats); - } - } - } - - // Collect what we've got so far in the return vector. - let last_crate = tcx.crates().len(); - let mut ret = (1..last_crate+1).map(|cnum| { - match formats.get(&CrateNum::new(cnum)) { - Some(&RequireDynamic) => Linkage::Dynamic, - Some(&RequireStatic) => Linkage::IncludedFromDylib, - None => Linkage::NotLinked, - } - }).collect::>(); - - // Run through the dependency list again, and add any missing libraries as - // static libraries. - // - // If the crate hasn't been included yet and it's not actually required - // (e.g., it's an allocator) then we skip it here as well. - for &cnum in tcx.crates().iter() { - let src = tcx.used_crate_source(cnum); - if src.dylib.is_none() && - !formats.contains_key(&cnum) && - tcx.dep_kind(cnum) == DepKind::Explicit { - assert!(src.rlib.is_some() || src.rmeta.is_some()); - info!("adding staticlib: {}", tcx.crate_name(cnum)); - add_library(tcx, cnum, RequireStatic, &mut formats); - ret[cnum.as_usize() - 1] = Linkage::Static; - } - } - - // We've gotten this far because we're emitting some form of a final - // artifact which means that we may need to inject dependencies of some - // form. - // - // Things like allocators and panic runtimes may not have been activated - // quite yet, so do so here. - activate_injected_dep(*sess.injected_panic_runtime.get(), &mut ret, - &|cnum| tcx.is_panic_runtime(cnum)); - - // When dylib B links to dylib A, then when using B we must also link to A. - // It could be the case, however, that the rlib for A is present (hence we - // found metadata), but the dylib for A has since been removed. - // - // For situations like this, we perform one last pass over the dependencies, - // making sure that everything is available in the requested format. - for (cnum, kind) in ret.iter().enumerate() { - let cnum = CrateNum::new(cnum + 1); - let src = tcx.used_crate_source(cnum); - match *kind { - Linkage::NotLinked | - Linkage::IncludedFromDylib => {} - Linkage::Static if src.rlib.is_some() => continue, - Linkage::Dynamic if src.dylib.is_some() => continue, - kind => { - let kind = match kind { - Linkage::Static => "rlib", - _ => "dylib", - }; - sess.err(&format!("crate `{}` required to be available in {} format, \ - but was not found in this form", - tcx.crate_name(cnum), kind)); - } - } - } - - ret -} - -fn add_library( - tcx: TyCtxt<'_>, - cnum: CrateNum, - link: LinkagePreference, - m: &mut FxHashMap, -) { - match m.get(&cnum) { - Some(&link2) => { - // If the linkages differ, then we'd have two copies of the library - // if we continued linking. If the linkages are both static, then we - // would also have two copies of the library (static from two - // different locations). - // - // This error is probably a little obscure, but I imagine that it - // can be refined over time. - if link2 != link || link == RequireStatic { - tcx.sess.struct_err(&format!("cannot satisfy dependencies so `{}` only \ - shows up once", tcx.crate_name(cnum))) - .help("having upstream crates all available in one format \ - will likely make this go away") - .emit(); - } - } - None => { m.insert(cnum, link); } - } -} - -fn attempt_static(tcx: TyCtxt<'_>) -> Option { - let sess = &tcx.sess; - let crates = cstore::used_crates(tcx, RequireStatic); - if !crates.iter().by_ref().all(|&(_, ref p)| p.is_some()) { - return None - } - - // All crates are available in an rlib format, so we're just going to link - // everything in explicitly so long as it's actually required. - let last_crate = tcx.crates().len(); - let mut ret = (1..last_crate+1).map(|cnum| { - if tcx.dep_kind(CrateNum::new(cnum)) == DepKind::Explicit { - Linkage::Static - } else { - Linkage::NotLinked - } - }).collect::>(); - - // Our allocator/panic runtime may not have been linked above if it wasn't - // explicitly linked, which is the case for any injected dependency. Handle - // that here and activate them. - activate_injected_dep(*sess.injected_panic_runtime.get(), &mut ret, - &|cnum| tcx.is_panic_runtime(cnum)); - - Some(ret) -} - -// Given a list of how to link upstream dependencies so far, ensure that an -// injected dependency is activated. This will not do anything if one was -// transitively included already (e.g., via a dylib or explicitly so). -// -// If an injected dependency was not found then we're guaranteed the -// metadata::creader module has injected that dependency (not listed as -// a required dependency) in one of the session's field. If this field is not -// set then this compilation doesn't actually need the dependency and we can -// also skip this step entirely. -fn activate_injected_dep(injected: Option, - list: &mut DependencyList, - replaces_injected: &dyn Fn(CrateNum) -> bool) { - for (i, slot) in list.iter().enumerate() { - let cnum = CrateNum::new(i + 1); - if !replaces_injected(cnum) { - continue - } - if *slot != Linkage::NotLinked { - return - } - } - if let Some(injected) = injected { - let idx = injected.as_usize() - 1; - assert_eq!(list[idx], Linkage::NotLinked); - list[idx] = Linkage::Static; - } -} - -// After the linkage for a crate has been determined we need to verify that -// there's only going to be one allocator in the output. -fn verify_ok(tcx: TyCtxt<'_>, list: &[Linkage]) { - let sess = &tcx.sess; - if list.len() == 0 { - return - } - let mut panic_runtime = None; - for (i, linkage) in list.iter().enumerate() { - if let Linkage::NotLinked = *linkage { - continue - } - let cnum = CrateNum::new(i + 1); - - if tcx.is_panic_runtime(cnum) { - if let Some((prev, _)) = panic_runtime { - let prev_name = tcx.crate_name(prev); - let cur_name = tcx.crate_name(cnum); - sess.err(&format!("cannot link together two \ - panic runtimes: {} and {}", - prev_name, cur_name)); - } - panic_runtime = Some((cnum, tcx.panic_strategy(cnum))); - } - } - - // If we found a panic runtime, then we know by this point that it's the - // only one, but we perform validation here that all the panic strategy - // compilation modes for the whole DAG are valid. - if let Some((cnum, found_strategy)) = panic_runtime { - let desired_strategy = sess.panic_strategy(); - - // First up, validate that our selected panic runtime is indeed exactly - // our same strategy. - if found_strategy != desired_strategy { - sess.err(&format!("the linked panic runtime `{}` is \ - not compiled with this crate's \ - panic strategy `{}`", - tcx.crate_name(cnum), - desired_strategy.desc())); - } - - // Next up, verify that all other crates are compatible with this panic - // strategy. If the dep isn't linked, we ignore it, and if our strategy - // is abort then it's compatible with everything. Otherwise all crates' - // panic strategy must match our own. - for (i, linkage) in list.iter().enumerate() { - if let Linkage::NotLinked = *linkage { - continue - } - if desired_strategy == PanicStrategy::Abort { - continue - } - let cnum = CrateNum::new(i + 1); - let found_strategy = tcx.panic_strategy(cnum); - let is_compiler_builtins = tcx.is_compiler_builtins(cnum); - if is_compiler_builtins || desired_strategy == found_strategy { - continue - } - - sess.err(&format!("the crate `{}` is compiled with the \ - panic strategy `{}` which is \ - incompatible with this crate's \ - strategy of `{}`", - tcx.crate_name(cnum), - found_strategy.desc(), - desired_strategy.desc())); - } - } -} diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index c7260945295..4a9fa57f345 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -630,6 +630,10 @@ -> &'tcx [(CrateNum, LinkagePreference)] { desc { "dylib dependency formats of crate" } } + + query dependency_formats(_: CrateNum) -> Lrc { + desc { "get the linkage format of all dependencies" } + } } Codegen { diff --git a/src/librustc/session/config.rs b/src/librustc/session/config.rs index 5eda3df3781..a9a20a92434 100644 --- a/src/librustc/session/config.rs +++ b/src/librustc/session/config.rs @@ -687,7 +687,7 @@ pub enum EntryFnType { impl_stable_hash_via_hash!(EntryFnType); -#[derive(Copy, PartialEq, PartialOrd, Clone, Ord, Eq, Hash, Debug)] +#[derive(Copy, PartialEq, PartialOrd, Clone, Ord, Eq, Hash, Debug, HashStable)] pub enum CrateType { Executable, Dylib, diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index a24fed8f21c..8aca861d1f6 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -7,7 +7,6 @@ use crate::lint; use crate::lint::builtin::BuiltinLintDiagnostics; -use crate::middle::dependency_format; use crate::session::config::{OutputType, PrintRequest, SwitchWithOptPath}; use crate::session::search_paths::{PathKind, SearchPath}; use crate::util::nodemap::{FxHashMap, FxHashSet}; @@ -91,7 +90,6 @@ pub struct Session { pub plugin_llvm_passes: OneThread>>, pub plugin_attributes: Lock>, pub crate_types: Once>, - pub dependency_formats: Once, /// The `crate_disambiguator` is constructed out of all the `-C metadata` /// arguments passed to the compiler. Its value together with the crate-name /// forms a unique global identifier for the crate. It is used to allow @@ -1247,7 +1245,6 @@ fn build_session_( plugin_llvm_passes: OneThread::new(RefCell::new(Vec::new())), plugin_attributes: Lock::new(Vec::new()), crate_types: Once::new(), - dependency_formats: Once::new(), crate_disambiguator: Once::new(), features: Once::new(), recursion_limit: Once::new(), diff --git a/src/librustc_codegen_ssa/back/link.rs b/src/librustc_codegen_ssa/back/link.rs index 9b044d9b453..3b7ae5e33d5 100644 --- a/src/librustc_codegen_ssa/back/link.rs +++ b/src/librustc_codegen_ssa/back/link.rs @@ -219,15 +219,24 @@ pub fn get_linker(sess: &Session, linker: &Path, flavor: LinkerFlavor) -> (PathB (linker.to_path_buf(), cmd) } -pub fn each_linked_rlib(sess: &Session, - info: &CrateInfo, - f: &mut dyn FnMut(CrateNum, &Path)) -> Result<(), String> { +pub fn each_linked_rlib( + info: &CrateInfo, + f: &mut dyn FnMut(CrateNum, &Path), +) -> Result<(), String> { let crates = info.used_crates_static.iter(); - let fmts = sess.dependency_formats.borrow(); - let fmts = fmts.get(&config::CrateType::Executable) - .or_else(|| fmts.get(&config::CrateType::Staticlib)) - .or_else(|| fmts.get(&config::CrateType::Cdylib)) - .or_else(|| fmts.get(&config::CrateType::ProcMacro)); + let mut fmts = None; + for (ty, list) in info.dependency_formats.iter() { + match ty { + config::CrateType::Executable | + config::CrateType::Staticlib | + config::CrateType::Cdylib | + config::CrateType::ProcMacro => { + fmts = Some(list); + break; + } + _ => {} + } + } let fmts = match fmts { Some(f) => f, None => return Err("could not find formats for rlibs".to_string()) @@ -406,7 +415,7 @@ fn link_staticlib<'a, B: ArchiveBuilder<'a>>(sess: &'a Session, tempdir); let mut all_native_libs = vec![]; - let res = each_linked_rlib(sess, &codegen_results.crate_info, &mut |cnum, path| { + let res = each_linked_rlib(&codegen_results.crate_info, &mut |cnum, path| { let name = &codegen_results.crate_info.crate_name[&cnum]; let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; @@ -1294,11 +1303,13 @@ pub fn add_local_native_libraries(cmd: &mut dyn Linker, // Rust crates are not considered at all when creating an rlib output. All // dependencies will be linked when producing the final output (instead of // the intermediate rlib version) -fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>(cmd: &mut dyn Linker, - sess: &'a Session, - codegen_results: &CodegenResults, - crate_type: config::CrateType, - tmpdir: &Path) { +fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( + cmd: &mut dyn Linker, + sess: &'a Session, + codegen_results: &CodegenResults, + crate_type: config::CrateType, + tmpdir: &Path, +) { // All of the heavy lifting has previously been accomplished by the // dependency_format module of the compiler. This is just crawling the // output of that module, adding crates as necessary. @@ -1307,8 +1318,10 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>(cmd: &mut dyn Linker, // will slurp up the object files inside), and linking to a dynamic library // involves just passing the right -l flag. - let formats = sess.dependency_formats.borrow(); - let data = formats.get(&crate_type).unwrap(); + let (_, data) = codegen_results.crate_info.dependency_formats + .iter() + .find(|(ty, _)| *ty == crate_type) + .expect("failed to find crate type in dependency format list"); // Invoke get_used_crates to ensure that we get a topological sorting of // crates. @@ -1620,10 +1633,12 @@ fn add_dynamic_crate(cmd: &mut dyn Linker, sess: &Session, cratepath: &Path) { // generic function calls a native function, then the generic function must // be instantiated in the target crate, meaning that the native symbol must // also be resolved in the target crate. -pub fn add_upstream_native_libraries(cmd: &mut dyn Linker, - sess: &Session, - codegen_results: &CodegenResults, - crate_type: config::CrateType) { +pub fn add_upstream_native_libraries( + cmd: &mut dyn Linker, + sess: &Session, + codegen_results: &CodegenResults, + crate_type: config::CrateType, +) { // Be sure to use a topological sorting of crates because there may be // interdependencies between native libraries. When passing -nodefaultlibs, // for example, almost all native libraries depend on libc, so we have to @@ -1633,8 +1648,10 @@ pub fn add_upstream_native_libraries(cmd: &mut dyn Linker, // This passes RequireStatic, but the actual requirement doesn't matter, // we're just getting an ordering of crate numbers, we're not worried about // the paths. - let formats = sess.dependency_formats.borrow(); - let data = formats.get(&crate_type).unwrap(); + let (_, data) = codegen_results.crate_info.dependency_formats + .iter() + .find(|(ty, _)| *ty == crate_type) + .expect("failed to find crate type in dependency format list"); let crates = &codegen_results.crate_info.used_crates_static; for &(cnum, _) in crates { diff --git a/src/librustc_codegen_ssa/back/linker.rs b/src/librustc_codegen_ssa/back/linker.rs index c42cd024926..940a9a72bf6 100644 --- a/src/librustc_codegen_ssa/back/linker.rs +++ b/src/librustc_codegen_ssa/back/linker.rs @@ -1092,10 +1092,16 @@ fn exported_symbols(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec { } } - let formats = tcx.sess.dependency_formats.borrow(); - let deps = formats[&crate_type].iter(); + let formats = tcx.dependency_formats(LOCAL_CRATE); + let deps = formats.iter().filter_map(|(t, list)| { + if *t == crate_type { + Some(list) + } else { + None + } + }).next().unwrap(); - for (index, dep_format) in deps.enumerate() { + for (index, dep_format) in deps.iter().enumerate() { let cnum = CrateNum::new(index + 1); // For each dependency that we are linking to statically ... if *dep_format == Linkage::Static { diff --git a/src/librustc_codegen_ssa/back/write.rs b/src/librustc_codegen_ssa/back/write.rs index 1bba479c1fd..1a2c23ae0d4 100644 --- a/src/librustc_codegen_ssa/back/write.rs +++ b/src/librustc_codegen_ssa/back/write.rs @@ -1048,7 +1048,7 @@ fn start_executing_work( }).expect("failed to spawn helper thread"); let mut each_linked_rlib_for_lto = Vec::new(); - drop(link::each_linked_rlib(sess, crate_info, &mut |cnum, path| { + drop(link::each_linked_rlib(crate_info, &mut |cnum, path| { if link::ignored_for_lto(sess, crate_info, cnum) { return } diff --git a/src/librustc_codegen_ssa/base.rs b/src/librustc_codegen_ssa/base.rs index 4acbe0356b4..98d3022a418 100644 --- a/src/librustc_codegen_ssa/base.rs +++ b/src/librustc_codegen_ssa/base.rs @@ -539,7 +539,7 @@ pub fn codegen_crate( // linkage, then it's already got an allocator shim and we'll be using that // one instead. If nothing exists then it's our job to generate the // allocator! - let any_dynamic_crate = tcx.sess.dependency_formats.borrow() + let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE) .iter() .any(|(_, list)| { use rustc::middle::dependency_format::Linkage; @@ -731,6 +731,7 @@ pub fn new(tcx: TyCtxt<'_>) -> CrateInfo { used_crate_source: Default::default(), lang_item_to_crate: Default::default(), missing_lang_items: Default::default(), + dependency_formats: tcx.dependency_formats(LOCAL_CRATE), }; let lang_items = tcx.lang_items(); diff --git a/src/librustc_codegen_ssa/lib.rs b/src/librustc_codegen_ssa/lib.rs index 1708d7235b4..161d3ce61f0 100644 --- a/src/librustc_codegen_ssa/lib.rs +++ b/src/librustc_codegen_ssa/lib.rs @@ -33,6 +33,7 @@ use rustc_data_structures::sync::Lrc; use rustc_data_structures::svh::Svh; use rustc::middle::cstore::{LibSource, CrateSource, NativeLibrary}; +use rustc::middle::dependency_format::Dependencies; use syntax_pos::symbol::Symbol; mod error_codes; @@ -142,6 +143,7 @@ pub struct CrateInfo { pub used_crates_dynamic: Vec<(CrateNum, LibSource)>, pub lang_item_to_crate: FxHashMap, pub missing_lang_items: FxHashMap>, + pub dependency_formats: Lrc, } diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs index e8e8da67334..102ea690f5a 100644 --- a/src/librustc_interface/passes.rs +++ b/src/librustc_interface/passes.rs @@ -1079,10 +1079,6 @@ pub fn start_codegen<'tcx>( tcx.print_debug_stats(); } - time(tcx.sess, "resolving dependency formats", || { - middle::dependency_format::calculate(tcx) - }); - let (metadata, need_metadata_module) = time(tcx.sess, "metadata encoding and writing", || { encode_and_write_metadata(tcx, outputs) }); diff --git a/src/librustc_metadata/cstore_impl.rs b/src/librustc_metadata/cstore_impl.rs index a1e3bbcbf8e..4a81fd3b320 100644 --- a/src/librustc_metadata/cstore_impl.rs +++ b/src/librustc_metadata/cstore_impl.rs @@ -370,6 +370,11 @@ pub fn provide(providers: &mut Providers<'_>) { tcx.arena.alloc(visible_parent_map) }, + dependency_formats: |tcx, cnum| { + assert_eq!(cnum, LOCAL_CRATE); + Lrc::new(crate::dependency_format::calculate(tcx)) + }, + ..*providers }; } diff --git a/src/librustc_metadata/dependency_format.rs b/src/librustc_metadata/dependency_format.rs new file mode 100644 index 00000000000..9de326a67b4 --- /dev/null +++ b/src/librustc_metadata/dependency_format.rs @@ -0,0 +1,371 @@ +//! Resolution of mixing rlibs and dylibs +//! +//! When producing a final artifact, such as a dynamic library, the compiler has +//! a choice between linking an rlib or linking a dylib of all upstream +//! dependencies. The linking phase must guarantee, however, that a library only +//! show up once in the object file. For example, it is illegal for library A to +//! be statically linked to B and C in separate dylibs, and then link B and C +//! into a crate D (because library A appears twice). +//! +//! The job of this module is to calculate what format each upstream crate +//! should be used when linking each output type requested in this session. This +//! generally follows this set of rules: +//! +//! 1. Each library must appear exactly once in the output. +//! 2. Each rlib contains only one library (it's just an object file) +//! 3. Each dylib can contain more than one library (due to static linking), +//! and can also bring in many dynamic dependencies. +//! +//! With these constraints in mind, it's generally a very difficult problem to +//! find a solution that's not "all rlibs" or "all dylibs". I have suspicions +//! that NP-ness may come into the picture here... +//! +//! The current selection algorithm below looks mostly similar to: +//! +//! 1. If static linking is required, then require all upstream dependencies +//! to be available as rlibs. If not, generate an error. +//! 2. If static linking is requested (generating an executable), then +//! attempt to use all upstream dependencies as rlibs. If any are not +//! found, bail out and continue to step 3. +//! 3. Static linking has failed, at least one library must be dynamically +//! linked. Apply a heuristic by greedily maximizing the number of +//! dynamically linked libraries. +//! 4. Each upstream dependency available as a dynamic library is +//! registered. The dependencies all propagate, adding to a map. It is +//! possible for a dylib to add a static library as a dependency, but it +//! is illegal for two dylibs to add the same static library as a +//! dependency. The same dylib can be added twice. Additionally, it is +//! illegal to add a static dependency when it was previously found as a +//! dylib (and vice versa) +//! 5. After all dynamic dependencies have been traversed, re-traverse the +//! remaining dependencies and add them statically (if they haven't been +//! added already). +//! +//! While not perfect, this algorithm should help support use-cases such as leaf +//! dependencies being static while the larger tree of inner dependencies are +//! all dynamic. This isn't currently very well battle tested, so it will likely +//! fall short in some use cases. +//! +//! Currently, there is no way to specify the preference of linkage with a +//! particular library (other than a global dynamic/static switch). +//! Additionally, the algorithm is geared towards finding *any* solution rather +//! than finding a number of solutions (there are normally quite a few). + +use rustc::hir::def_id::CrateNum; +use rustc::middle::cstore::LinkagePreference::{self, RequireStatic, RequireDynamic}; +use rustc::middle::cstore::{self, DepKind}; +use rustc::middle::dependency_format::{DependencyList, Dependencies, Linkage}; +use rustc::session::config; +use rustc::ty::TyCtxt; +use rustc::util::nodemap::FxHashMap; +use rustc_target::spec::PanicStrategy; + +pub fn calculate(tcx: TyCtxt<'_>) -> Dependencies { + tcx.sess.crate_types.borrow().iter().map(|&ty| { + let linkage = calculate_type(tcx, ty); + verify_ok(tcx, &linkage); + (ty, linkage) + }).collect::>() +} + +fn calculate_type(tcx: TyCtxt<'_>, ty: config::CrateType) -> DependencyList { + let sess = &tcx.sess; + + if !sess.opts.output_types.should_codegen() { + return Vec::new(); + } + + let preferred_linkage = match ty { + // cdylibs must have all static dependencies. + config::CrateType::Cdylib => Linkage::Static, + + // Generating a dylib without `-C prefer-dynamic` means that we're going + // to try to eagerly statically link all dependencies. This is normally + // done for end-product dylibs, not intermediate products. + config::CrateType::Dylib if !sess.opts.cg.prefer_dynamic => Linkage::Static, + config::CrateType::Dylib => Linkage::Dynamic, + + // If the global prefer_dynamic switch is turned off, or the final + // executable will be statically linked, prefer static crate linkage. + config::CrateType::Executable if !sess.opts.cg.prefer_dynamic || + sess.crt_static() => Linkage::Static, + config::CrateType::Executable => Linkage::Dynamic, + + // proc-macro crates are mostly cdylibs, but we also need metadata. + config::CrateType::ProcMacro => Linkage::Static, + + // No linkage happens with rlibs, we just needed the metadata (which we + // got long ago), so don't bother with anything. + config::CrateType::Rlib => Linkage::NotLinked, + + // staticlibs must have all static dependencies. + config::CrateType::Staticlib => Linkage::Static, + }; + + if preferred_linkage == Linkage::NotLinked { + // If the crate is not linked, there are no link-time dependencies. + return Vec::new(); + } + + if preferred_linkage == Linkage::Static { + // Attempt static linkage first. For dylibs and executables, we may be + // able to retry below with dynamic linkage. + if let Some(v) = attempt_static(tcx) { + return v; + } + + // Staticlibs, cdylibs, and static executables must have all static + // dependencies. If any are not found, generate some nice pretty errors. + if ty == config::CrateType::Cdylib || ty == config::CrateType::Staticlib || + (ty == config::CrateType::Executable && sess.crt_static() && + !sess.target.target.options.crt_static_allows_dylibs) { + for &cnum in tcx.crates().iter() { + if tcx.dep_kind(cnum).macros_only() { continue } + let src = tcx.used_crate_source(cnum); + if src.rlib.is_some() { continue } + sess.err(&format!("crate `{}` required to be available in rlib format, \ + but was not found in this form", + tcx.crate_name(cnum))); + } + return Vec::new(); + } + } + + let mut formats = FxHashMap::default(); + + // Sweep all crates for found dylibs. Add all dylibs, as well as their + // dependencies, ensuring there are no conflicts. The only valid case for a + // dependency to be relied upon twice is for both cases to rely on a dylib. + for &cnum in tcx.crates().iter() { + if tcx.dep_kind(cnum).macros_only() { continue } + let name = tcx.crate_name(cnum); + let src = tcx.used_crate_source(cnum); + if src.dylib.is_some() { + log::info!("adding dylib: {}", name); + add_library(tcx, cnum, RequireDynamic, &mut formats); + let deps = tcx.dylib_dependency_formats(cnum); + for &(depnum, style) in deps.iter() { + log::info!("adding {:?}: {}", style, tcx.crate_name(depnum)); + add_library(tcx, depnum, style, &mut formats); + } + } + } + + // Collect what we've got so far in the return vector. + let last_crate = tcx.crates().len(); + let mut ret = (1..last_crate+1).map(|cnum| { + match formats.get(&CrateNum::new(cnum)) { + Some(&RequireDynamic) => Linkage::Dynamic, + Some(&RequireStatic) => Linkage::IncludedFromDylib, + None => Linkage::NotLinked, + } + }).collect::>(); + + // Run through the dependency list again, and add any missing libraries as + // static libraries. + // + // If the crate hasn't been included yet and it's not actually required + // (e.g., it's an allocator) then we skip it here as well. + for &cnum in tcx.crates().iter() { + let src = tcx.used_crate_source(cnum); + if src.dylib.is_none() && + !formats.contains_key(&cnum) && + tcx.dep_kind(cnum) == DepKind::Explicit { + assert!(src.rlib.is_some() || src.rmeta.is_some()); + log::info!("adding staticlib: {}", tcx.crate_name(cnum)); + add_library(tcx, cnum, RequireStatic, &mut formats); + ret[cnum.as_usize() - 1] = Linkage::Static; + } + } + + // We've gotten this far because we're emitting some form of a final + // artifact which means that we may need to inject dependencies of some + // form. + // + // Things like allocators and panic runtimes may not have been activated + // quite yet, so do so here. + activate_injected_dep(*sess.injected_panic_runtime.get(), &mut ret, + &|cnum| tcx.is_panic_runtime(cnum)); + + // When dylib B links to dylib A, then when using B we must also link to A. + // It could be the case, however, that the rlib for A is present (hence we + // found metadata), but the dylib for A has since been removed. + // + // For situations like this, we perform one last pass over the dependencies, + // making sure that everything is available in the requested format. + for (cnum, kind) in ret.iter().enumerate() { + let cnum = CrateNum::new(cnum + 1); + let src = tcx.used_crate_source(cnum); + match *kind { + Linkage::NotLinked | + Linkage::IncludedFromDylib => {} + Linkage::Static if src.rlib.is_some() => continue, + Linkage::Dynamic if src.dylib.is_some() => continue, + kind => { + let kind = match kind { + Linkage::Static => "rlib", + _ => "dylib", + }; + sess.err(&format!("crate `{}` required to be available in {} format, \ + but was not found in this form", + tcx.crate_name(cnum), kind)); + } + } + } + + ret +} + +fn add_library( + tcx: TyCtxt<'_>, + cnum: CrateNum, + link: LinkagePreference, + m: &mut FxHashMap, +) { + match m.get(&cnum) { + Some(&link2) => { + // If the linkages differ, then we'd have two copies of the library + // if we continued linking. If the linkages are both static, then we + // would also have two copies of the library (static from two + // different locations). + // + // This error is probably a little obscure, but I imagine that it + // can be refined over time. + if link2 != link || link == RequireStatic { + tcx.sess.struct_err(&format!("cannot satisfy dependencies so `{}` only \ + shows up once", tcx.crate_name(cnum))) + .help("having upstream crates all available in one format \ + will likely make this go away") + .emit(); + } + } + None => { m.insert(cnum, link); } + } +} + +fn attempt_static(tcx: TyCtxt<'_>) -> Option { + let sess = &tcx.sess; + let crates = cstore::used_crates(tcx, RequireStatic); + if !crates.iter().by_ref().all(|&(_, ref p)| p.is_some()) { + return None + } + + // All crates are available in an rlib format, so we're just going to link + // everything in explicitly so long as it's actually required. + let last_crate = tcx.crates().len(); + let mut ret = (1..last_crate+1).map(|cnum| { + if tcx.dep_kind(CrateNum::new(cnum)) == DepKind::Explicit { + Linkage::Static + } else { + Linkage::NotLinked + } + }).collect::>(); + + // Our allocator/panic runtime may not have been linked above if it wasn't + // explicitly linked, which is the case for any injected dependency. Handle + // that here and activate them. + activate_injected_dep(*sess.injected_panic_runtime.get(), &mut ret, + &|cnum| tcx.is_panic_runtime(cnum)); + + Some(ret) +} + +// Given a list of how to link upstream dependencies so far, ensure that an +// injected dependency is activated. This will not do anything if one was +// transitively included already (e.g., via a dylib or explicitly so). +// +// If an injected dependency was not found then we're guaranteed the +// metadata::creader module has injected that dependency (not listed as +// a required dependency) in one of the session's field. If this field is not +// set then this compilation doesn't actually need the dependency and we can +// also skip this step entirely. +fn activate_injected_dep(injected: Option, + list: &mut DependencyList, + replaces_injected: &dyn Fn(CrateNum) -> bool) { + for (i, slot) in list.iter().enumerate() { + let cnum = CrateNum::new(i + 1); + if !replaces_injected(cnum) { + continue + } + if *slot != Linkage::NotLinked { + return + } + } + if let Some(injected) = injected { + let idx = injected.as_usize() - 1; + assert_eq!(list[idx], Linkage::NotLinked); + list[idx] = Linkage::Static; + } +} + +// After the linkage for a crate has been determined we need to verify that +// there's only going to be one allocator in the output. +fn verify_ok(tcx: TyCtxt<'_>, list: &[Linkage]) { + let sess = &tcx.sess; + if list.len() == 0 { + return + } + let mut panic_runtime = None; + for (i, linkage) in list.iter().enumerate() { + if let Linkage::NotLinked = *linkage { + continue + } + let cnum = CrateNum::new(i + 1); + + if tcx.is_panic_runtime(cnum) { + if let Some((prev, _)) = panic_runtime { + let prev_name = tcx.crate_name(prev); + let cur_name = tcx.crate_name(cnum); + sess.err(&format!("cannot link together two \ + panic runtimes: {} and {}", + prev_name, cur_name)); + } + panic_runtime = Some((cnum, tcx.panic_strategy(cnum))); + } + } + + // If we found a panic runtime, then we know by this point that it's the + // only one, but we perform validation here that all the panic strategy + // compilation modes for the whole DAG are valid. + if let Some((cnum, found_strategy)) = panic_runtime { + let desired_strategy = sess.panic_strategy(); + + // First up, validate that our selected panic runtime is indeed exactly + // our same strategy. + if found_strategy != desired_strategy { + sess.err(&format!("the linked panic runtime `{}` is \ + not compiled with this crate's \ + panic strategy `{}`", + tcx.crate_name(cnum), + desired_strategy.desc())); + } + + // Next up, verify that all other crates are compatible with this panic + // strategy. If the dep isn't linked, we ignore it, and if our strategy + // is abort then it's compatible with everything. Otherwise all crates' + // panic strategy must match our own. + for (i, linkage) in list.iter().enumerate() { + if let Linkage::NotLinked = *linkage { + continue + } + if desired_strategy == PanicStrategy::Abort { + continue + } + let cnum = CrateNum::new(i + 1); + let found_strategy = tcx.panic_strategy(cnum); + let is_compiler_builtins = tcx.is_compiler_builtins(cnum); + if is_compiler_builtins || desired_strategy == found_strategy { + continue + } + + sess.err(&format!("the crate `{}` is compiled with the \ + panic strategy `{}` which is \ + incompatible with this crate's \ + strategy of `{}`", + tcx.crate_name(cnum), + found_strategy.desc(), + desired_strategy.desc())); + } + } +} + diff --git a/src/librustc_metadata/encoder.rs b/src/librustc_metadata/encoder.rs index f430f01542e..be083cf6020 100644 --- a/src/librustc_metadata/encoder.rs +++ b/src/librustc_metadata/encoder.rs @@ -1649,20 +1649,22 @@ fn encode_exported_symbols(&mut self, } fn encode_dylib_dependency_formats(&mut self) -> Lazy<[Option]> { - match self.tcx.sess.dependency_formats.borrow().get(&config::CrateType::Dylib) { - Some(arr) => { - self.lazy(arr.iter().map(|slot| { - match *slot { - Linkage::NotLinked | - Linkage::IncludedFromDylib => None, - - Linkage::Dynamic => Some(LinkagePreference::RequireDynamic), - Linkage::Static => Some(LinkagePreference::RequireStatic), - } - })) + let formats = self.tcx.dependency_formats(LOCAL_CRATE); + for (ty, arr) in formats.iter() { + if *ty != config::CrateType::Dylib { + continue; } - None => Lazy::empty(), + return self.lazy(arr.iter().map(|slot| { + match *slot { + Linkage::NotLinked | + Linkage::IncludedFromDylib => None, + + Linkage::Dynamic => Some(LinkagePreference::RequireDynamic), + Linkage::Static => Some(LinkagePreference::RequireStatic), + } + })); } + Lazy::empty() } fn encode_info_for_foreign_item(&mut self, diff --git a/src/librustc_metadata/lib.rs b/src/librustc_metadata/lib.rs index e6104e629e9..9273b064ba9 100644 --- a/src/librustc_metadata/lib.rs +++ b/src/librustc_metadata/lib.rs @@ -32,6 +32,7 @@ mod native_libs; mod link_args; mod foreign_modules; +mod dependency_format; pub mod creader; pub mod cstore;