mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Code to save/load the work-products map from disk

Browse Source 
Work products are deleted if any of their inputs are dirty.
This commit is contained in:
Niko Matsakis 2016-07-21 12:44:59 -04:00
parent ffc13b2f80
commit cca4804251
7 changed files with 192 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/librustc_driver/driver.rs
View File

@ -88,7 +88,7 @@ pub fn compile_input(sess: &Session,
// We need nested scopes here, because the intermediate results can keep
// large chunks of memory alive and we want to free them as soon as
// possible to keep the peak memory usage low
let (outputs, trans) = {
let (outputs, trans, id) = {
let krate = match phase_1_parse_input(sess, cfg, input) {
Ok(krate) => krate,
Err(mut parse_error) => {
@ -212,11 +212,11 @@ pub fn compile_input(sess: &Session,
// Discard interned strings as they are no longer required.
token::clear_ident_interner();
Ok((outputs, trans))
Ok((outputs, trans, id.clone()))
})??
};
let phase5_result = phase_5_run_llvm_passes(sess, &trans, &outputs);
let phase5_result = phase_5_run_llvm_passes(sess, &id, &trans, &outputs);
controller_entry_point!(after_llvm,
sess,
@ -1020,6 +1020,7 @@ pub fn phase_4_translate_to_llvm<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
/// Run LLVM itself, producing a bitcode file, assembly file or object file
/// as a side effect.
pub fn phase_5_run_llvm_passes(sess: &Session,
crate_name: &str,
trans: &trans::CrateTranslation,
outputs: &OutputFilenames) -> CompileResult {
if sess.opts.cg.no_integrated_as {
@ -1041,6 +1042,10 @@ pub fn phase_5_run_llvm_passes(sess: &Session,
|| write::run_passes(sess, trans, &sess.opts.output_types, outputs));
}
time(sess.time_passes(),
"serialize work products",
move || rustc_incremental::save_work_products(sess, crate_name));
if sess.err_count() > 0 {
Err(sess.err_count())
} else {

2
src/librustc_incremental/lib.rs
View File

@ -19,6 +19,7 @@
html_root_url = "https://doc.rust-lang.org/nightly/")]
#![cfg_attr(not(stage0), deny(warnings))]
#![feature(question_mark)]
#![feature(rustc_private)]
#![feature(staged_api)]
@ -40,3 +41,4 @@ pub use assert_dep_graph::assert_dep_graph;
pub use calculate_svh::SvhCalculate;
pub use persist::load_dep_graph;
pub use persist::save_dep_graph;
pub use persist::save_work_products;

12
src/librustc_incremental/persist/data.rs
View File

@ -10,8 +10,9 @@
//! The data that we will serialize and deserialize.
use rustc::dep_graph::DepNode;
use rustc::dep_graph::{DepNode, WorkProduct, WorkProductId};
use rustc::hir::def_id::DefIndex;
use std::sync::Arc;
use super::directory::DefPathIndex;
@ -55,6 +56,15 @@ pub struct SerializedHash {
pub hash: u64,
}
#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct SerializedWorkProduct {
/// node that produced the work-product
pub id: Arc<WorkProductId>,
/// work-product data itself
pub work_product: WorkProduct,
}
/// Data for use when downstream crates get recompiled.
#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct SerializedMetadataHashes {

123
src/librustc_incremental/persist/load.rs
View File

@ -14,12 +14,13 @@ use rbml::Error;
use rbml::opaque::Decoder;
use rustc::dep_graph::DepNode;
use rustc::hir::def_id::DefId;
use rustc::session::Session;
use rustc::ty::TyCtxt;
use rustc_data_structures::fnv::FnvHashSet;
use rustc_serialize::Decodable as RustcDecodable;
use std::io::Read;
use std::fs::File;
use std::path::Path;
use std::fs::{self, File};
use std::path::{Path};
use super::data::*;
use super::directory::*;
@ -38,18 +39,40 @@ type CleanEdges = Vec<(DepNode<DefId>, DepNode<DefId>)>;
/// actually it doesn't matter all that much.) See `README.md` for
/// more general overview.
pub fn load_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {
let _ignore = tcx.dep_graph.in_ignore();
if tcx.sess.opts.incremental.is_none() {
return;
}
if let Some(dep_graph) = dep_graph_path(tcx) {
// FIXME(#32754) lock file?
load_dep_graph_if_exists(tcx, &dep_graph);
dirty_clean::check_dirty_clean_annotations(tcx);
let _ignore = tcx.dep_graph.in_ignore();
load_dep_graph_if_exists(tcx);
dirty_clean::check_dirty_clean_annotations(tcx);
}
fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {
let dep_graph_path = dep_graph_path(tcx).unwrap();
let dep_graph_data = match load_data(tcx.sess, &dep_graph_path) {
Some(p) => p,
None => return // no file
};
let work_products_path = tcx_work_products_path(tcx).unwrap();
let work_products_data = match load_data(tcx.sess, &work_products_path) {
Some(p) => p,
None => return // no file
};
match decode_dep_graph(tcx, &dep_graph_data, &work_products_data) {
Ok(()) => return,
Err(err) => bug!("decoding error in dep-graph from `{}` and `{}`: {}",
dep_graph_path.display(),
work_products_path.display(),
err),
}
}
pub fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, path: &Path) {
fn load_data(sess: &Session, path: &Path) -> Option<Vec<u8>> {
if !path.exists() {
return;
return None;
}
let mut data = vec![];
@ -57,31 +80,32 @@ pub fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, path: &Pa
File::open(path)
.and_then(|mut file| file.read_to_end(&mut data))
{
Ok(_) => { }
Ok(_) => {
Some(data)
}
Err(err) => {
tcx.sess.err(
sess.err(
&format!("could not load dep-graph from `{}`: {}",
path.display(), err));
return;
None
}
}
match decode_dep_graph(tcx, &data) {
Ok(dirty) => dirty,
Err(err) => {
bug!("decoding error in dep-graph from `{}`: {}", path.display(), err);
}
}
}
/// Decode the dep graph and load the edges/nodes that are still clean
/// into `tcx.dep_graph`. On success, returns a hashset containing all
/// the paths of work-products from clean nodes (any work-products not
/// in this set can be deleted).
pub fn decode_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
data: &[u8])
dep_graph_data: &[u8],
work_products_data: &[u8])
-> Result<(), Error>
{
// Deserialize the directory and dep-graph.
let mut decoder = Decoder::new(data, 0);
let directory = try!(DefIdDirectory::decode(&mut decoder));
let serialized_dep_graph = try!(SerializedDepGraph::decode(&mut decoder));
let mut dep_graph_decoder = Decoder::new(dep_graph_data, 0);
let directory = try!(DefIdDirectory::decode(&mut dep_graph_decoder));
let serialized_dep_graph = try!(SerializedDepGraph::decode(&mut dep_graph_decoder));
debug!("decode_dep_graph: directory = {:#?}", directory);
debug!("decode_dep_graph: serialized_dep_graph = {:#?}", serialized_dep_graph);
@ -121,12 +145,18 @@ pub fn decode_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
// Add nodes and edges that are not dirty into our main graph.
let dep_graph = tcx.dep_graph.clone();
for (source, target) in clean_edges.into_iter().chain(clean_nodes) {
let _task = dep_graph.in_task(target.clone());
dep_graph.read(source.clone());
debug!("decode_dep_graph: clean edge: {:?} -> {:?}", source, target);
let _task = dep_graph.in_task(target);
dep_graph.read(source);
}
// Add in work-products that are still clean, and delete those that are
// dirty.
let mut work_product_decoder = Decoder::new(work_products_data, 0);
let work_products = try!(<Vec<SerializedWorkProduct>>::decode(&mut work_product_decoder));
reconcile_work_products(tcx, work_products, &dirty_nodes);
Ok(())
}
@ -141,9 +171,9 @@ fn initial_dirty_nodes<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
match hash.node.map_def(|&i| retraced.def_id(i)) {
Some(dep_node) => {
let current_hash = hcx.hash(&dep_node).unwrap();
debug!("initial_dirty_nodes: hash of {:?} is {:?}, was {:?}",
dep_node, current_hash, hash.hash);
if current_hash != hash.hash {
debug!("initial_dirty_nodes: {:?} is dirty as hash is {:?}, was {:?}",
dep_node, current_hash, hash.hash);
dirty_nodes.insert(dep_node);
}
}
@ -177,6 +207,8 @@ fn compute_clean_edges(serialized_edges: &[(SerializedEdge)],
clean_edges.push((source, target))
} else {
// source removed, target must be dirty
debug!("compute_clean_edges: {:?} dirty because {:?} no longer exists",
target, serialized_source);
dirty_nodes.insert(target);
}
} else {
@ -213,3 +245,40 @@ fn compute_clean_edges(serialized_edges: &[(SerializedEdge)],
clean_edges
}
/// Go through the list of work-products produced in the previous run.
/// Delete any whose nodes have been found to be dirty or which are
/// otherwise no longer applicable.
fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
work_products: Vec<SerializedWorkProduct>,
dirty_nodes: &DirtyNodes) {
debug!("reconcile_work_products({:?})", work_products);
for swp in work_products {
let dep_node = DepNode::WorkProduct(swp.id.clone());
if dirty_nodes.contains(&dep_node) {
debug!("reconcile_work_products: dep-node for {:?} is dirty", swp);
delete_dirty_work_product(tcx, swp);
} else {
let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap();
if path.exists() {
tcx.dep_graph.insert_previous_work_product(&swp.id, swp.work_product);
} else {
debug!("reconcile_work_products: file for {:?} does not exist", swp);
}
}
}
}
fn delete_dirty_work_product(tcx: TyCtxt,
swp: SerializedWorkProduct) {
debug!("delete_dirty_work_product({:?})", swp);
let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap();
match fs::remove_file(&path) {
Ok(()) => { }
Err(err) => {
tcx.sess.warn(
&format!("file-system error deleting outdated file `{}`: {}",
path.display(), err));
}
}
}

1
src/librustc_incremental/persist/mod.rs
View File

@ -22,3 +22,4 @@ mod util;
pub use self::load::load_dep_graph;
pub use self::save::save_dep_graph;
pub use self::save::save_work_products;

51
src/librustc_incremental/persist/save.rs
View File

@ -11,6 +11,7 @@
use rbml::opaque::Encoder;
use rustc::dep_graph::DepNode;
use rustc::middle::cstore::LOCAL_CRATE;
use rustc::session::Session;
use rustc::ty::TyCtxt;
use rustc_serialize::{Encodable as RustcEncodable};
use std::hash::{Hasher, SipHasher};
@ -24,19 +25,26 @@ use super::hash::*;
use super::util::*;
pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {
debug!("save_dep_graph()");
let _ignore = tcx.dep_graph.in_ignore();
let sess = tcx.sess;
let mut hcx = HashContext::new(tcx);
save_in(&mut hcx, dep_graph_path(tcx), encode_dep_graph);
save_in(&mut hcx, metadata_hash_path(tcx, LOCAL_CRATE), encode_metadata_hashes);
save_in(sess, dep_graph_path(tcx), |e| encode_dep_graph(&mut hcx, e));
save_in(sess, metadata_hash_path(tcx, LOCAL_CRATE), |e| encode_metadata_hashes(&mut hcx, e));
}
fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>,
opt_path_buf: Option<PathBuf>,
encode: F)
where F: FnOnce(&mut HashContext<'a, 'tcx>, &mut Encoder) -> io::Result<()>
{
let tcx = hcx.tcx;
pub fn save_work_products(sess: &Session, local_crate_name: &str) {
debug!("save_work_products()");
let _ignore = sess.dep_graph.in_ignore();
let path = sess_work_products_path(sess, local_crate_name);
save_in(sess, path, |e| encode_work_products(sess, e));
}
fn save_in<F>(sess: &Session,
opt_path_buf: Option<PathBuf>,
encode: F)
where F: FnOnce(&mut Encoder) -> io::Result<()>
{
let path_buf = match opt_path_buf {
Some(p) => p,
None => return
@ -49,7 +57,7 @@ fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>,
match fs::remove_file(&path_buf) {
Ok(()) => { }
Err(err) => {
tcx.sess.err(
sess.err(
&format!("unable to delete old dep-graph at `{}`: {}",
path_buf.display(), err));
return;
@ -59,10 +67,10 @@ fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>,
// generate the data in a memory buffer
let mut wr = Cursor::new(Vec::new());
match encode(hcx, &mut Encoder::new(&mut wr)) {
match encode(&mut Encoder::new(&mut wr)) {
Ok(()) => { }
Err(err) => {
tcx.sess.err(
sess.err(
&format!("could not encode dep-graph to `{}`: {}",
path_buf.display(), err));
return;
@ -77,7 +85,7 @@ fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>,
{
Ok(_) => { }
Err(err) => {
tcx.sess.err(
sess.err(
&format!("failed to write dep-graph to `{}`: {}",
path_buf.display(), err));
return;
@ -192,3 +200,22 @@ pub fn encode_metadata_hashes<'a, 'tcx>(hcx: &mut HashContext<'a, 'tcx>,
Ok(())
}
pub fn encode_work_products(sess: &Session,
encoder: &mut Encoder)
-> io::Result<()>
{
let work_products: Vec<_> =
sess.dep_graph.work_products()
.iter()
.map(|(id, work_product)| {
SerializedWorkProduct {
id: id.clone(),
work_product: work_product.clone(),
}
})
.collect();
work_products.encode(encoder)
}

46
src/librustc_incremental/persist/util.rs
View File

@ -9,6 +9,7 @@
// except according to those terms.
use rustc::middle::cstore::LOCAL_CRATE;
use rustc::session::Session;
use rustc::ty::TyCtxt;
use std::fs;
@ -17,33 +18,56 @@ use std::path::{Path, PathBuf};
use syntax::ast;
pub fn dep_graph_path(tcx: TyCtxt) -> Option<PathBuf> {
path(tcx, LOCAL_CRATE, "local")
tcx_path(tcx, LOCAL_CRATE, "local")
}
pub fn metadata_hash_path(tcx: TyCtxt, cnum: ast::CrateNum) -> Option<PathBuf> {
path(tcx, cnum, "metadata")
tcx_path(tcx, cnum, "metadata")
}
fn path(tcx: TyCtxt, cnum: ast::CrateNum, suffix: &str) -> Option<PathBuf> {
pub fn tcx_work_products_path(tcx: TyCtxt) -> Option<PathBuf> {
let crate_name = tcx.crate_name(LOCAL_CRATE);
sess_work_products_path(tcx.sess, &crate_name)
}
pub fn sess_work_products_path(sess: &Session,
local_crate_name: &str)
-> Option<PathBuf> {
let crate_disambiguator = sess.local_crate_disambiguator();
path(sess, local_crate_name, &crate_disambiguator, "work-products")
}
pub fn in_incr_comp_dir(sess: &Session, file_name: &str) -> Option<PathBuf> {
sess.opts.incremental.as_ref().map(|incr_dir| incr_dir.join(file_name))
}
fn tcx_path(tcx: TyCtxt,
cnum: ast::CrateNum,
middle: &str)
-> Option<PathBuf> {
path(tcx.sess, &tcx.crate_name(cnum), &tcx.crate_disambiguator(cnum), middle)
}
fn path(sess: &Session,
crate_name: &str,
crate_disambiguator: &str,
middle: &str)
-> Option<PathBuf> {
// For now, just save/load dep-graph from
// directory/dep_graph.rbml
tcx.sess.opts.incremental.as_ref().and_then(|incr_dir| {
sess.opts.incremental.as_ref().and_then(|incr_dir| {
match create_dir_racy(&incr_dir) {
Ok(()) => {}
Err(err) => {
tcx.sess.err(
sess.err(
&format!("could not create the directory `{}`: {}",
incr_dir.display(), err));
return None;
}
}
let crate_name = tcx.crate_name(cnum);
let crate_disambiguator = tcx.crate_disambiguator(cnum);
let file_name = format!("{}-{}.{}.bin",
crate_name,
crate_disambiguator,
suffix);
let file_name = format!("{}-{}.{}.bin", crate_name, crate_disambiguator, middle);
Some(incr_dir.join(file_name))
})
}