mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Keep multiple files per work-product

Browse Source 
In the older version, a `.o` and ` .bc` file were separate
work-products.  This newer version keeps, for each codegen-unit, a set
of files of different kinds. We assume that if any kinds are available
then all the kinds we need are available, since the precise set of
switches will depend on attributes and command-line switches.

Should probably test this: the effect of changing attributes in
particular might not be successfully tracked?
This commit is contained in:
Niko Matsakis 2016-07-25 10:51:14 -04:00
parent ceeb158e0a
commit 2f9fff2191
12 changed files with 202 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/librustc/dep_graph/dep_node.rs
View File

@ -246,7 +246,5 @@ impl<D: Clone + Debug> DepNode<D> {
/// the need to be mapped or unmapped. (This ensures we can serialize
/// them even in the absence of a tcx.)
#[derive(Clone, Debug, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable)]
pub enum WorkProductId {
PartitionObjectFile(String), // see (*TransPartition) below
}
pub struct WorkProductId(pub String);

7
src/librustc/dep_graph/graph.rs
View File

@ -10,6 +10,7 @@
use hir::def_id::DefId;
use rustc_data_structures::fnv::FnvHashMap;
use session::config::OutputType;
use std::cell::{Ref, RefCell};
use std::rc::Rc;
use std::sync::Arc;
@ -157,11 +158,11 @@ impl DepGraph {
/// previous hash. If it matches up, we can reuse the object file.
#[derive(Clone, Debug, RustcEncodable, RustcDecodable)]
pub struct WorkProduct {
/// extra hash used to decide if work-product is still suitable;
/// Extra hash used to decide if work-product is still suitable;
/// note that this is *not* a hash of the work-product itself.
/// See documentation on `WorkProduct` type for an example.
pub input_hash: u64,
/// filename storing this work-product (found in the incr. comp. directory)
pub file_name: String,
/// Saved files associated with this CGU
pub saved_files: Vec<(OutputType, String)>,
}

23
src/librustc/session/config.rs
View File

@ -61,7 +61,7 @@ pub enum DebugInfoLevel {
FullDebugInfo,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)]
pub enum OutputType {
Bitcode,
Assembly,
@ -105,6 +105,17 @@ impl OutputType {
OutputType::DepInfo => "dep-info",
}
}
pub fn extension(&self) -> &'static str {
match *self {
OutputType::Bitcode => "bc",
OutputType::Assembly => "s",
OutputType::LlvmAssembly => "ll",
OutputType::Object => "o",
OutputType::DepInfo => "d",
OutputType::Exe => "",
}
}
}
#[derive(Clone)]
@ -215,15 +226,7 @@ impl OutputFilenames {
flavor: OutputType,
codegen_unit_name: Option<&str>)
-> PathBuf {
let extension = match flavor {
OutputType::Bitcode => "bc",
OutputType::Assembly => "s",
OutputType::LlvmAssembly => "ll",
OutputType::Object => "o",
OutputType::DepInfo => "d",
OutputType::Exe => "",
};
let extension = flavor.extension();
self.temp_path_ext(extension, codegen_unit_name)
}

31
src/librustc_incremental/persist/load.rs
View File

@ -260,11 +260,20 @@ fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
debug!("reconcile_work_products: dep-node for {:?} is dirty", swp);
delete_dirty_work_product(tcx, swp);
} else {
let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap();
if path.exists() {
let all_files_exist =
swp.work_product
.saved_files
.iter()
.all(|&(_, ref file_name)| {
let path = in_incr_comp_dir(tcx.sess, &file_name).unwrap();
path.exists()
});
if all_files_exist {
debug!("reconcile_work_products: all files for {:?} exist", swp);
tcx.dep_graph.insert_previous_work_product(&swp.id, swp.work_product);
} else {
debug!("reconcile_work_products: file for {:?} does not exist", swp);
debug!("reconcile_work_products: some file for {:?} does not exist", swp);
delete_dirty_work_product(tcx, swp);
}
}
}
@ -273,13 +282,15 @@ fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
fn delete_dirty_work_product(tcx: TyCtxt,
swp: SerializedWorkProduct) {
debug!("delete_dirty_work_product({:?})", swp);
let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap();
match fs::remove_file(&path) {
Ok(()) => { }
Err(err) => {
tcx.sess.warn(
&format!("file-system error deleting outdated file `{}`: {}",
path.display(), err));
for &(_, ref file_name) in &swp.work_product.saved_files {
let path = in_incr_comp_dir(tcx.sess, file_name).unwrap();
match fs::remove_file(&path) {
Ok(()) => { }
Err(err) => {
tcx.sess.warn(
&format!("file-system error deleting outdated file `{}`: {}",
path.display(), err));
}
}
}
}

68
src/librustc_incremental/persist/work_product.rs
View File

@ -13,47 +13,51 @@
use persist::util::*;
use rustc::dep_graph::{WorkProduct, WorkProductId};
use rustc::session::Session;
use rustc::session::config::OutputType;
use rustc::util::fs::link_or_copy;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
pub fn save_trans_partition(sess: &Session,
partition_name: &str,
cgu_name: &str,
partition_hash: u64,
path_to_obj_file: &Path) {
debug!("save_trans_partition({:?},{},{})",
partition_name,
files: &[(OutputType, PathBuf)]) {
debug!("save_trans_partition({:?},{},{:?})",
cgu_name,
partition_hash,
path_to_obj_file.display());
files);
if sess.opts.incremental.is_none() {
return;
}
let id = Arc::new(WorkProductId::PartitionObjectFile(partition_name.to_string()));
let file_name = format!("cgu-{}", partition_name);
let path_in_incr_dir = in_incr_comp_dir(sess, &file_name).unwrap();
let work_product_id = Arc::new(WorkProductId(cgu_name.to_string()));
// try to delete the file if it already exists
//
// FIXME(#34955) we can be smarter here -- if we are re-using, no need to do anything
if path_in_incr_dir.exists() {
let _ = fs::remove_file(&path_in_incr_dir);
}
let saved_files: Option<Vec<_>> =
files.iter()
.map(|&(kind, ref path)| {
let file_name = format!("cgu-{}.{}", cgu_name, kind.extension());
let path_in_incr_dir = in_incr_comp_dir(sess, &file_name).unwrap();
match link_or_copy(path, &path_in_incr_dir) {
Ok(_) => Some((kind, file_name)),
Err(err) => {
sess.warn(&format!("error copying object file `{}` \
to incremental directory as `{}`: {}",
path.display(),
path_in_incr_dir.display(),
err));
None
}
}
})
.collect();
let saved_files = match saved_files {
Some(v) => v,
None => return,
};
match link_or_copy(path_to_obj_file, &path_in_incr_dir) {
Ok(_) => {
let work_product = WorkProduct {
input_hash: partition_hash,
file_name: file_name,
};
sess.dep_graph.insert_work_product(&id, work_product);
}
Err(err) => {
sess.warn(&format!("error copying object file `{}` \
to incremental directory as `{}`: {}",
path_to_obj_file.display(),
path_in_incr_dir.display(),
err));
}
}
let work_product = WorkProduct {
input_hash: partition_hash,
saved_files: saved_files,
};
sess.dep_graph.insert_work_product(&work_product_id, work_product);
}

56
src/librustc_trans/back/write.rs
View File

@ -337,6 +337,8 @@ struct CodegenContext<'a> {
remark: Passes,
// Worker thread number
worker: usize,
// Directory where incremental data is stored (if any)
incremental: Option<PathBuf>,
}
impl<'a> CodegenContext<'a> {
@ -347,6 +349,7 @@ impl<'a> CodegenContext<'a> {
plugin_passes: sess.plugin_llvm_passes.borrow().clone(),
remark: sess.opts.cg.remark.clone(),
worker: 0,
incremental: sess.opts.incremental.clone(),
}
}
}
@ -612,7 +615,7 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext,
if copy_bc_to_obj {
debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
if let Err(e) = fs::copy(&bc_out, &obj_out) {
if let Err(e) = link_or_copy(&bc_out, &obj_out) {
cgcx.handler.err(&format!("failed to copy bitcode to object file: {}", e));
}
}
@ -754,9 +757,19 @@ pub fn run_passes(sess: &Session,
// If in incr. comp. mode, preserve the `.o` files for potential re-use
for mtrans in trans.modules.iter() {
let path_to_obj = crate_output.temp_path(OutputType::Object, Some(&mtrans.name));
debug!("wrote module {:?} to {:?}", mtrans.name, path_to_obj);
save_trans_partition(sess, &mtrans.name, mtrans.symbol_name_hash, &path_to_obj);
let mut files = vec![];
if modules_config.emit_obj {
let path = crate_output.temp_path(OutputType::Object, Some(&mtrans.name));
files.push((OutputType::Object, path));
}
if modules_config.emit_bc {
let path = crate_output.temp_path(OutputType::Bitcode, Some(&mtrans.name));
files.push((OutputType::Bitcode, path));
}
save_trans_partition(sess, &mtrans.name, mtrans.symbol_name_hash, &files);
}
// All codegen is finished.
@ -941,20 +954,24 @@ fn execute_work_item(cgcx: &CodegenContext,
work_item.config,
work_item.output_names);
}
ModuleSource::Preexisting(ref buf) => {
let obj_out = work_item.output_names.temp_path(OutputType::Object,
Some(&work_item.mtrans.name));
debug!("copying pre-existing module `{}` from {} to {}",
work_item.mtrans.name,
buf.display(),
obj_out.display());
match link_or_copy(buf, &obj_out) {
Ok(()) => { }
Err(err) => {
cgcx.handler.err(&format!("unable to copy {} to {}: {}",
buf.display(),
obj_out.display(),
err));
ModuleSource::Preexisting(wp) => {
let incremental = cgcx.incremental.as_ref().unwrap();
let name = &work_item.mtrans.name;
for (kind, saved_file) in wp.saved_files {
let obj_out = work_item.output_names.temp_path(kind, Some(name));
let source_file = incremental.join(&saved_file);
debug!("copying pre-existing module `{}` from {:?} to {}",
work_item.mtrans.name,
source_file,
obj_out.display());
match link_or_copy(&source_file, &obj_out) {
Ok(()) => { }
Err(err) => {
cgcx.handler.err(&format!("unable to copy {} to {}: {}",
source_file.display(),
obj_out.display(),
err));
}
}
}
}
@ -994,6 +1011,8 @@ fn run_work_multithreaded(sess: &Session,
let mut tx = Some(tx);
futures.push(rx);
let incremental = sess.opts.incremental.clone();
thread::Builder::new().name(format!("codegen-{}", i)).spawn(move || {
let diag_handler = Handler::with_emitter(true, false, box diag_emitter);
@ -1005,6 +1024,7 @@ fn run_work_multithreaded(sess: &Session,
plugin_passes: plugin_passes,
remark: remark,
worker: i,
incremental: incremental,
};
loop {

8
src/librustc_trans/base.rs
View File

@ -43,10 +43,9 @@ use rustc::ty::subst::{self, Substs};
use rustc::traits;
use rustc::ty::{self, Ty, TyCtxt, TypeFoldable};
use rustc::ty::adjustment::CustomCoerceUnsized;
use rustc::dep_graph::DepNode;
use rustc::dep_graph::{DepNode, WorkProduct};
use rustc::hir::map as hir_map;
use rustc::util::common::time;
use rustc_incremental::in_incr_comp_dir;
use rustc::mir::mir_map::MirMap;
use rustc_data_structures::graph::OUTGOING;
use session::config::{self, NoDebugInfo, FullDebugInfo};
@ -103,7 +102,6 @@ use std::cell::{Cell, RefCell};
use std::collections::HashMap;
use std::ptr;
use std::rc::Rc;
use std::path::PathBuf;
use std::str;
use std::{i8, i16, i32, i64};
use syntax_pos::{Span, DUMMY_SP};
@ -2721,7 +2719,7 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
fn trans_reuse_previous_work_products(tcx: TyCtxt,
codegen_units: &[CodegenUnit],
symbol_map: &SymbolMap)
-> Vec<Option<PathBuf>> {
-> Vec<Option<WorkProduct>> {
debug!("trans_reuse_previous_work_products()");
codegen_units
.iter()
@ -2735,7 +2733,7 @@ fn trans_reuse_previous_work_products(tcx: TyCtxt,
if let Some(work_product) = tcx.dep_graph.previous_work_product(&id) {
if work_product.input_hash == hash {
debug!("trans_reuse_previous_work_products: reusing {:?}", work_product);
return Some(in_incr_comp_dir(tcx.sess, &work_product.file_name).unwrap());
return Some(work_product);
} else {
debug!("trans_reuse_previous_work_products: \
not reusing {:?} because hash changed to {:?}",

15
src/librustc_trans/context.rs
View File

@ -10,7 +10,7 @@
use llvm;
use llvm::{ContextRef, ModuleRef, ValueRef, BuilderRef};
use rustc::dep_graph::{DepNode, DepTrackingMap, DepTrackingMapConfig};
use rustc::dep_graph::{DepNode, DepTrackingMap, DepTrackingMapConfig, WorkProduct};
use middle::cstore::LinkMeta;
use rustc::hir::def::ExportMap;
use rustc::hir::def_id::DefId;
@ -40,7 +40,6 @@ use util::nodemap::{NodeMap, NodeSet, DefIdMap, FnvHashMap, FnvHashSet};
use std::ffi::{CStr, CString};
use std::cell::{Cell, RefCell};
use std::path::PathBuf;
use std::marker::PhantomData;
use std::ptr;
use std::rc::Rc;
@ -96,7 +95,7 @@ pub struct SharedCrateContext<'a, 'tcx: 'a> {
pub struct LocalCrateContext<'tcx> {
llmod: ModuleRef,
llcx: ContextRef,
previous_work_product: Option<PathBuf>,
previous_work_product: Option<WorkProduct>,
tn: TypeNames, // FIXME: This seems to be largely unused.
codegen_unit: CodegenUnit<'tcx>,
needs_unwind_cleanup_cache: RefCell<FnvHashMap<Ty<'tcx>, bool>>,
@ -202,13 +201,13 @@ pub struct CrateContextList<'a, 'tcx: 'a> {
impl<'a, 'tcx: 'a> CrateContextList<'a, 'tcx> {
pub fn new(shared_ccx: &'a SharedCrateContext<'a, 'tcx>,
codegen_units: Vec<CodegenUnit<'tcx>>,
previous_work_products: Vec<Option<PathBuf>>,
previous_work_products: Vec<Option<WorkProduct>>,
symbol_map: Rc<SymbolMap<'tcx>>)
-> CrateContextList<'a, 'tcx> {
CrateContextList {
shared: shared_ccx,
local_ccxs: codegen_units.into_iter().zip(previous_work_products).map(|(cgu, path)| {
LocalCrateContext::new(shared_ccx, cgu, path, symbol_map.clone())
local_ccxs: codegen_units.into_iter().zip(previous_work_products).map(|(cgu, wp)| {
LocalCrateContext::new(shared_ccx, cgu, wp, symbol_map.clone())
}).collect()
}
}
@ -541,7 +540,7 @@ impl<'b, 'tcx> SharedCrateContext<'b, 'tcx> {
impl<'tcx> LocalCrateContext<'tcx> {
fn new<'a>(shared: &SharedCrateContext<'a, 'tcx>,
codegen_unit: CodegenUnit<'tcx>,
previous_work_product: Option<PathBuf>,
previous_work_product: Option<WorkProduct>,
symbol_map: Rc<SymbolMap<'tcx>>)
-> LocalCrateContext<'tcx> {
unsafe {
@ -727,7 +726,7 @@ impl<'b, 'tcx> CrateContext<'b, 'tcx> {
self.local().llcx
}
pub fn previous_work_product(&self) -> Option<&PathBuf> {
pub fn previous_work_product(&self) -> Option<&WorkProduct> {
self.local().previous_work_product.as_ref()
}

12
src/librustc_trans/lib.rs
View File

@ -37,7 +37,7 @@
#![feature(unicode)]
#![feature(question_mark)]
use std::path::PathBuf;
use rustc::dep_graph::WorkProduct;
extern crate arena;
extern crate flate;
@ -135,6 +135,11 @@ mod value;
#[derive(Clone)]
pub struct ModuleTranslation {
/// The name of the module. When the crate may be saved between
/// compilations, incremental compilation requires that name be
/// unique amongst **all** crates. Therefore, it should contain
/// something unique to this crate (e.g., a module path) as well
/// as the crate name and disambiguator.
pub name: String,
pub symbol_name_hash: u64,
pub source: ModuleSource,
@ -142,7 +147,10 @@ pub struct ModuleTranslation {
#[derive(Clone)]
pub enum ModuleSource {
Preexisting(PathBuf),
/// Copy the `.o` files or whatever from the incr. comp. directory.
Preexisting(WorkProduct),
/// Rebuild from this LLVM module.
Translated(ModuleLlvm),
}

7
src/librustc_trans/partitioning.rs
View File

@ -143,7 +143,12 @@ pub enum PartitioningStrategy {
}
pub struct CodegenUnit<'tcx> {
/// A name for this CGU. Incremental compilation requires that
/// name be unique amongst **all** crates. Therefore, it should
/// contain something unique to this crate (e.g., a module path)
/// as well as the crate name and disambiguator.
name: InternedString,
items: FnvHashMap<TransItem<'tcx>, llvm::Linkage>,
}
@ -174,7 +179,7 @@ impl<'tcx> CodegenUnit<'tcx> {
}
pub fn work_product_id(&self) -> Arc<WorkProductId> {
Arc::new(WorkProductId::PartitionObjectFile(self.name().to_string()))
Arc::new(WorkProductId(self.name().to_string()))
}
pub fn work_product_dep_node(&self) -> DepNode<DefId> {

25
src/test/incremental/rlib_cross_crate/auxiliary/a.rs Normal file
View File

@ -0,0 +1,25 @@
// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// no-prefer-dynamic
#![crate_type="rlib"]
#[cfg(rpass1)]
pub type X = u32;
#[cfg(rpass2)]
pub type X = i32;
// this version doesn't actually change anything:
#[cfg(rpass3)]
pub type X = i32;
pub type Y = char;

38
src/test/incremental/rlib_cross_crate/b.rs Normal file
View File

@ -0,0 +1,38 @@
// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// Same test as `type_alias_cross_crate`, but with
// `no-prefer-dynamic`, ensuring that we test what happens when we
// build rlibs (before we were only testing dylibs, which meant we
// didn't realize we had to preserve a `bc` file as well).
// aux-build:a.rs
// revisions:rpass1 rpass2 rpass3
// no-prefer-dynamic
#![feature(rustc_attrs)]
extern crate a;
#[rustc_dirty(label="TypeckItemBody", cfg="rpass2")]
#[rustc_clean(label="TypeckItemBody", cfg="rpass3")]
pub fn use_X() -> u32 {
let x: a::X = 22;
x as u32
}
#[rustc_clean(label="TypeckItemBody", cfg="rpass2")]
#[rustc_clean(label="TypeckItemBody", cfg="rpass3")]
pub fn use_Y() {
let x: a::Y = 'c';
}
pub fn main() { }