mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

incr.comp.: Implement query diagnostic persistence.

Browse Source
This commit is contained in:
Michael Woerister 2017-10-19 14:32:39 +02:00
parent 686d2a7f14
commit f55425dfcd
15 changed files with 362 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/librustc/dep_graph/graph.rs
View File

@ -8,6 +8,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use errors::DiagnosticBuilder;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher,
StableHashingContextProvider};
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
@ -568,6 +569,24 @@ impl DepGraph {
"DepGraph::try_mark_green() - Duplicate fingerprint \
insertion for {:?}", dep_node);
// ... emitting any stored diagnostic ...
{
let diagnostics = tcx.on_disk_query_result_cache
.load_diagnostics(prev_dep_node_index);
if diagnostics.len() > 0 {
let handle = tcx.sess.diagnostic();
// Promote the previous diagnostics to the current session.
tcx.on_disk_query_result_cache
.store_diagnostics(dep_node_index, diagnostics.clone());
for diagnostic in diagnostics {
DiagnosticBuilder::new_diagnostic(handle, diagnostic).emit();
}
}
}
// ... and finally storing a "Green" entry in the color map.
let old_color = data.colors
.borrow_mut()

2
src/librustc/dep_graph/mod.rs
View File

@ -26,4 +26,4 @@ pub use self::prev::PreviousDepGraph;
pub use self::query::DepGraphQuery;
pub use self::safe::AssertDepGraphSafe;
pub use self::safe::DepGraphSafe;
pub use self::serialized::SerializedDepGraph;
pub use self::serialized::{SerializedDepGraph, SerializedDepNodeIndex};

1
src/librustc/lib.rs
View File

@ -46,6 +46,7 @@
#![feature(const_fn)]
#![feature(core_intrinsics)]
#![feature(i128_type)]
#![feature(inclusive_range_syntax)]
#![cfg_attr(windows, feature(libc))]
#![feature(never_type)]
#![feature(nonzero)]

16
src/librustc/ty/context.rs
View File

@ -853,6 +853,11 @@ pub struct GlobalCtxt<'tcx> {
pub dep_graph: DepGraph,
/// This provides access to the incr. comp. on-disk cache for query results.
/// Do not access this directly. It is only meant to be used by
/// `DepGraph::try_mark_green()` and the query infrastructure in `ty::maps`.
pub(crate) on_disk_query_result_cache: maps::OnDiskCache<'tcx>,
/// Common types, pre-interned for your convenience.
pub types: CommonTypes<'tcx>,
@ -1054,6 +1059,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
resolutions: ty::Resolutions,
named_region_map: resolve_lifetime::NamedRegionMap,
hir: hir_map::Map<'tcx>,
on_disk_query_result_cache: maps::OnDiskCache<'tcx>,
crate_name: &str,
tx: mpsc::Sender<Box<Any + Send>>,
output_filenames: &OutputFilenames,
@ -1137,6 +1143,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
global_arenas: arenas,
global_interners: interners,
dep_graph: dep_graph.clone(),
on_disk_query_result_cache,
types: common_types,
named_region_map: NamedRegionMap {
defs,
@ -1298,6 +1305,15 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
self.in_scope_traits_map(def_index);
}
}
pub fn serialize_query_result_cache<E>(self,
encoder: &mut E)
-> Result<(), E::Error>
where E: ::rustc_serialize::Encoder
{
self.on_disk_query_result_cache.serialize(encoder)
}
}
impl<'a, 'tcx> TyCtxt<'a, 'tcx, 'tcx> {

3
src/librustc/ty/maps/mod.rs
View File

@ -70,6 +70,9 @@ mod config;
pub use self::config::QueryConfig;
use self::config::QueryDescription;
mod on_disk_cache;
pub use self::on_disk_cache::OnDiskCache;
// Each of these maps also corresponds to a method on a
// `Provider` trait for requesting a value of that type,
// and a method on `Maps` itself for doing that in a

204
src/librustc/ty/maps/on_disk_cache.rs Normal file
View File

@ -0,0 +1,204 @@
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::indexed_vec::Idx;
use errors::Diagnostic;
use rustc_serialize::{Decodable, Decoder, Encodable, Encoder, opaque,
SpecializedDecoder};
use session::Session;
use std::borrow::Cow;
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::mem;
use syntax::codemap::{CodeMap, StableFilemapId};
use syntax_pos::{BytePos, Span, NO_EXPANSION, DUMMY_SP};
pub struct OnDiskCache<'sess> {
prev_diagnostics: FxHashMap<SerializedDepNodeIndex, Vec<Diagnostic>>,
_prev_filemap_starts: BTreeMap<BytePos, StableFilemapId>,
codemap: &'sess CodeMap,
current_diagnostics: RefCell<FxHashMap<DepNodeIndex, Vec<Diagnostic>>>,
}
#[derive(RustcEncodable, RustcDecodable)]
struct Header {
prev_filemap_starts: BTreeMap<BytePos, StableFilemapId>,
}
#[derive(RustcEncodable, RustcDecodable)]
struct Body {
diagnostics: Vec<(SerializedDepNodeIndex, Vec<Diagnostic>)>,
}
impl<'sess> OnDiskCache<'sess> {
pub fn new_empty(codemap: &'sess CodeMap) -> OnDiskCache<'sess> {
OnDiskCache {
prev_diagnostics: FxHashMap(),
_prev_filemap_starts: BTreeMap::new(),
codemap,
current_diagnostics: RefCell::new(FxHashMap()),
}
}
pub fn new(sess: &'sess Session, data: &[u8]) -> OnDiskCache<'sess> {
debug_assert!(sess.opts.incremental.is_some());
let mut decoder = opaque::Decoder::new(&data[..], 0);
let header = Header::decode(&mut decoder).unwrap();
let prev_diagnostics: FxHashMap<_, _> = {
let mut decoder = CacheDecoder {
opaque: decoder,
codemap: sess.codemap(),
prev_filemap_starts: &header.prev_filemap_starts,
};
let body = Body::decode(&mut decoder).unwrap();
body.diagnostics.into_iter().collect()
};
OnDiskCache {
prev_diagnostics,
_prev_filemap_starts: header.prev_filemap_starts,
codemap: sess.codemap(),
current_diagnostics: RefCell::new(FxHashMap()),
}
}
pub fn serialize<'a, 'tcx, E>(&self,
encoder: &mut E)
-> Result<(), E::Error>
where E: Encoder
{
let prev_filemap_starts: BTreeMap<_, _> = self
.codemap
.files()
.iter()
.map(|fm| (fm.start_pos, StableFilemapId::new(fm)))
.collect();
Header { prev_filemap_starts }.encode(encoder)?;
let diagnostics: Vec<(SerializedDepNodeIndex, Vec<Diagnostic>)> =
self.current_diagnostics
.borrow()
.iter()
.map(|(k, v)| (SerializedDepNodeIndex::new(k.index()), v.clone()))
.collect();
Body { diagnostics }.encode(encoder)?;
Ok(())
}
pub fn load_diagnostics(&self,
dep_node_index: SerializedDepNodeIndex)
-> Vec<Diagnostic> {
self.prev_diagnostics.get(&dep_node_index).cloned().unwrap_or(vec![])
}
pub fn store_diagnostics(&self,
dep_node_index: DepNodeIndex,
diagnostics: Vec<Diagnostic>) {
let mut current_diagnostics = self.current_diagnostics.borrow_mut();
let prev = current_diagnostics.insert(dep_node_index, diagnostics);
debug_assert!(prev.is_none());
}
pub fn store_diagnostics_for_anon_node(&self,
dep_node_index: DepNodeIndex,
mut diagnostics: Vec<Diagnostic>) {
let mut current_diagnostics = self.current_diagnostics.borrow_mut();
let x = current_diagnostics.entry(dep_node_index).or_insert_with(|| {
mem::replace(&mut diagnostics, Vec::new())
});
x.extend(diagnostics.into_iter());
}
}
impl<'a> SpecializedDecoder<Span> for CacheDecoder<'a> {
fn specialized_decode(&mut self) -> Result<Span, Self::Error> {
let lo = BytePos::decode(self)?;
let hi = BytePos::decode(self)?;
if let Some((prev_filemap_start, filemap_id)) = self.find_filemap_prev_bytepos(lo) {
if let Some(current_filemap) = self.codemap.filemap_by_stable_id(filemap_id) {
let lo = (lo + current_filemap.start_pos) - prev_filemap_start;
let hi = (hi + current_filemap.start_pos) - prev_filemap_start;
return Ok(Span::new(lo, hi, NO_EXPANSION));
}
}
Ok(DUMMY_SP)
}
}
struct CacheDecoder<'a> {
opaque: opaque::Decoder<'a>,
codemap: &'a CodeMap,
prev_filemap_starts: &'a BTreeMap<BytePos, StableFilemapId>,
}
impl<'a> CacheDecoder<'a> {
fn find_filemap_prev_bytepos(&self,
prev_bytepos: BytePos)
-> Option<(BytePos, StableFilemapId)> {
for (start, id) in self.prev_filemap_starts.range(BytePos(0) ... prev_bytepos).rev() {
return Some((*start, *id))
}
None
}
}
macro_rules! decoder_methods {
($($name:ident -> $ty:ty;)*) => {
$(fn $name(&mut self) -> Result<$ty, Self::Error> {
self.opaque.$name()
})*
}
}
impl<'sess> Decoder for CacheDecoder<'sess> {
type Error = String;
decoder_methods! {
read_nil -> ();
read_u128 -> u128;
read_u64 -> u64;
read_u32 -> u32;
read_u16 -> u16;
read_u8 -> u8;
read_usize -> usize;
read_i128 -> i128;
read_i64 -> i64;
read_i32 -> i32;
read_i16 -> i16;
read_i8 -> i8;
read_isize -> isize;
read_bool -> bool;
read_f64 -> f64;
read_f32 -> f32;
read_char -> char;
read_str -> Cow<str>;
}
fn error(&mut self, err: &str) -> Self::Error {
self.opaque.error(err)
}
}

53
src/librustc/ty/maps/plumbing.rs
View File

@ -13,14 +13,14 @@
//! provider, manage the caches, and so forth.
use dep_graph::{DepNodeIndex, DepNode, DepKind, DepNodeColor};
use errors::{Diagnostic, DiagnosticBuilder};
use errors::DiagnosticBuilder;
use ty::{TyCtxt};
use ty::maps::Query; // NB: actually generated by the macros in this file
use ty::maps::config::QueryDescription;
use ty::item_path;
use rustc_data_structures::fx::{FxHashMap};
use std::cell::{RefMut, Cell};
use std::cell::RefMut;
use std::marker::PhantomData;
use std::mem;
use syntax_pos::Span;
@ -33,34 +33,19 @@ pub(super) struct QueryMap<D: QueryDescription> {
pub(super) struct QueryValue<T> {
pub(super) value: T,
pub(super) index: DepNodeIndex,
pub(super) diagnostics: Option<Box<QueryDiagnostics>>,
}
impl<T> QueryValue<T> {
pub(super) fn new(value: T,
dep_node_index: DepNodeIndex,
diagnostics: Vec<Diagnostic>)
dep_node_index: DepNodeIndex)
-> QueryValue<T> {
QueryValue {
value,
index: dep_node_index,
diagnostics: if diagnostics.len() == 0 {
None
} else {
Some(Box::new(QueryDiagnostics {
diagnostics,
emitted_diagnostics: Cell::new(true),
}))
},
}
}
}
pub(super) struct QueryDiagnostics {
pub(super) diagnostics: Vec<Diagnostic>,
pub(super) emitted_diagnostics: Cell<bool>,
}
impl<M: QueryDescription> QueryMap<M> {
pub(super) fn new() -> QueryMap<M> {
QueryMap {
@ -284,16 +269,6 @@ macro_rules! define_maps {
);
if let Some(value) = tcx.maps.$name.borrow().map.get(&key) {
if let Some(ref d) = value.diagnostics {
if !d.emitted_diagnostics.get() {
d.emitted_diagnostics.set(true);
let handle = tcx.sess.diagnostic();
for diagnostic in d.diagnostics.iter() {
DiagnosticBuilder::new_diagnostic(handle, diagnostic.clone())
.emit();
}
}
}
profq_msg!(tcx, ProfileQueriesMsg::CacheHit);
tcx.dep_graph.read_index(value.index);
return Ok((&value.value).clone());
@ -331,7 +306,11 @@ macro_rules! define_maps {
let ((result, dep_node_index), diagnostics) = res;
tcx.dep_graph.read_index(dep_node_index);
let value = QueryValue::new(result, dep_node_index, diagnostics);
tcx.on_disk_query_result_cache
.store_diagnostics_for_anon_node(dep_node_index, diagnostics);
let value = QueryValue::new(result, dep_node_index);
return Ok((&tcx.maps
.$name
@ -398,8 +377,11 @@ macro_rules! define_maps {
{
debug_assert!(tcx.dep_graph.is_green(dep_node_index));
// We don't do any caching yet, so recompute
let (result, diagnostics) = tcx.cycle_check(span, Query::$name(key), || {
// We don't do any caching yet, so recompute.
// The diagnostics for this query have already been promoted to
// the current session during try_mark_green(), so we can ignore
// them here.
let (result, _) = tcx.cycle_check(span, Query::$name(key), || {
tcx.sess.diagnostic().track_diagnostics(|| {
// The dep-graph for this computation is already in place
tcx.dep_graph.with_ignore(|| {
@ -412,7 +394,7 @@ macro_rules! define_maps {
tcx.dep_graph.mark_loaded_from_cache(dep_node_index, true);
}
let value = QueryValue::new(result, dep_node_index, diagnostics);
let value = QueryValue::new(result, dep_node_index);
Ok((&tcx.maps
.$name
@ -447,7 +429,12 @@ macro_rules! define_maps {
tcx.dep_graph.mark_loaded_from_cache(dep_node_index, false);
}
let value = QueryValue::new(result, dep_node_index, diagnostics);
if dep_node.kind != ::dep_graph::DepKind::Null {
tcx.on_disk_query_result_cache
.store_diagnostics(dep_node_index, diagnostics);
}
let value = QueryValue::new(result, dep_node_index);
Ok(((&tcx.maps
.$name

5
src/librustc_driver/driver.rs
View File

@ -941,6 +941,10 @@ pub fn phase_3_run_analysis_passes<'tcx, F, R>(sess: &'tcx Session,
let time_passes = sess.time_passes();
let query_result_on_disk_cache = time(time_passes,
"load query result cache",
|| rustc_incremental::load_query_result_cache(sess));
let named_region_map = time(time_passes,
"lifetime resolution",
|| middle::resolve_lifetime::krate(sess, cstore, &hir_map))?;
@ -1049,6 +1053,7 @@ pub fn phase_3_run_analysis_passes<'tcx, F, R>(sess: &'tcx Session,
resolutions,
named_region_map,
hir_map,
query_result_on_disk_cache,
name,
tx,
output_filenames,

6
src/librustc_incremental/lib.rs
View File

@ -17,6 +17,9 @@
#![feature(rand)]
#![feature(conservative_impl_trait)]
#![feature(i128_type)]
#![feature(inclusive_range_syntax)]
#![feature(specialization)]
extern crate graphviz;
#[macro_use] extern crate rustc;
@ -31,8 +34,9 @@ mod assert_dep_graph;
mod persist;
pub use assert_dep_graph::assert_dep_graph;
pub use persist::load_dep_graph;
pub use persist::dep_graph_tcx_init;
pub use persist::load_dep_graph;
pub use persist::load_query_result_cache;
pub use persist::save_dep_graph;
pub use persist::save_trans_partition;
pub use persist::save_work_products;

5
src/librustc_incremental/persist/fs.rs
View File

@ -131,6 +131,7 @@ const LOCK_FILE_EXT: &'static str = ".lock";
const DEP_GRAPH_FILENAME: &'static str = "dep-graph.bin";
const WORK_PRODUCTS_FILENAME: &'static str = "work-products.bin";
const METADATA_HASHES_FILENAME: &'static str = "metadata.bin";
const QUERY_CACHE_FILENAME: &'static str = "query-cache.bin";
// We encode integers using the following base, so they are shorter than decimal
// or hexadecimal numbers (we want short file and directory names). Since these
@ -150,6 +151,10 @@ pub fn metadata_hash_export_path(sess: &Session) -> PathBuf {
in_incr_comp_dir_sess(sess, METADATA_HASHES_FILENAME)
}
pub fn query_cache_path(sess: &Session) -> PathBuf {
in_incr_comp_dir_sess(sess, QUERY_CACHE_FILENAME)
}
pub fn lock_file_path(session_dir: &Path) -> PathBuf {
let crate_dir = session_dir.parent().unwrap();

13
src/librustc_incremental/persist/load.rs
View File

@ -15,6 +15,7 @@ use rustc::hir::svh::Svh;
use rustc::ich::Fingerprint;
use rustc::session::Session;
use rustc::ty::TyCtxt;
use rustc::ty::maps::OnDiskCache;
use rustc::util::nodemap::DefIdMap;
use rustc_serialize::Decodable as RustcDecodable;
use rustc_serialize::opaque::Decoder;
@ -195,3 +196,15 @@ pub fn load_dep_graph(sess: &Session) -> PreviousDepGraph {
empty
}
}
pub fn load_query_result_cache<'sess>(sess: &'sess Session) -> OnDiskCache<'sess> {
if sess.opts.incremental.is_none() {
return OnDiskCache::new_empty(sess.codemap());
}
if let Some(bytes) = load_data(sess, &query_cache_path(sess)) {
OnDiskCache::new(sess, &bytes[..])
} else {
OnDiskCache::new_empty(sess.codemap())
}
}

3
src/librustc_incremental/persist/mod.rs
View File

@ -23,8 +23,9 @@ mod file_format;
pub use self::fs::prepare_session_directory;
pub use self::fs::finalize_session_directory;
pub use self::fs::in_incr_comp_dir;
pub use self::load::load_dep_graph;
pub use self::load::dep_graph_tcx_init;
pub use self::load::load_dep_graph;
pub use self::load::load_query_result_cache;
pub use self::save::save_dep_graph;
pub use self::save::save_work_products;
pub use self::work_product::save_trans_partition;

12
src/librustc_incremental/persist/save.rs
View File

@ -63,6 +63,12 @@ pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
e));
}
time(sess.time_passes(), "persist query result cache", || {
save_in(sess,
query_cache_path(sess),
|e| encode_query_cache(tcx, e));
});
time(sess.time_passes(), "persist dep-graph", || {
save_in(sess,
dep_graph_path(sess),
@ -298,3 +304,9 @@ fn encode_work_products(dep_graph: &DepGraph,
work_products.encode(encoder)
}
fn encode_query_cache(tcx: TyCtxt,
encoder: &mut Encoder)
-> io::Result<()> {
tcx.serialize_query_result_cache(encoder)
}

37
src/libsyntax/codemap.rs
View File

@ -17,11 +17,15 @@
//! within the CodeMap, which upon request can be converted to line and column
//! information, source code snippets, etc.
pub use syntax_pos::*;
pub use syntax_pos::hygiene::{ExpnFormat, ExpnInfo, NameAndSpan};
pub use self::ExpnFormat::*;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::stable_hasher::StableHasher;
use std::cell::{RefCell, Ref};
use std::hash::Hash;
use std::path::{Path, PathBuf};
use std::rc::Rc;
@ -98,6 +102,24 @@ impl FileLoader for RealFileLoader {
}
}
// This is a FileMap identifier that is used to correlate FileMaps between
// subsequent compilation sessions (which is something we need to do during
// incremental compilation).
#[derive(Copy, Clone, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable)]
pub struct StableFilemapId(u128);
impl StableFilemapId {
pub fn new(filemap: &FileMap) -> StableFilemapId {
let mut hasher = StableHasher::new();
filemap.name.hash(&mut hasher);
filemap.name_was_remapped.hash(&mut hasher);
filemap.unmapped_path.hash(&mut hasher);
StableFilemapId(hasher.finish())
}
}
// _____________________________________________________________________________
// CodeMap
//
@ -108,6 +130,7 @@ pub struct CodeMap {
// This is used to apply the file path remapping as specified via
// -Zremap-path-prefix to all FileMaps allocated within this CodeMap.
path_mapping: FilePathMapping,
stable_id_to_filemap: RefCell<FxHashMap<StableFilemapId, Rc<FileMap>>>,
}
impl CodeMap {
@ -116,6 +139,7 @@ impl CodeMap {
files: RefCell::new(Vec::new()),
file_loader: Box::new(RealFileLoader),
path_mapping,
stable_id_to_filemap: RefCell::new(FxHashMap()),
}
}
@ -126,6 +150,7 @@ impl CodeMap {
files: RefCell::new(Vec::new()),
file_loader,
path_mapping,
stable_id_to_filemap: RefCell::new(FxHashMap()),
}
}
@ -146,6 +171,10 @@ impl CodeMap {
self.files.borrow()
}
pub fn filemap_by_stable_id(&self, stable_id: StableFilemapId) -> Option<Rc<FileMap>> {
self.stable_id_to_filemap.borrow().get(&stable_id).map(|fm| fm.clone())
}
fn next_start_pos(&self) -> usize {
let files = self.files.borrow();
match files.last() {
@ -180,6 +209,10 @@ impl CodeMap {
files.push(filemap.clone());
self.stable_id_to_filemap
.borrow_mut()
.insert(StableFilemapId::new(&filemap), filemap.clone());
filemap
}
@ -241,6 +274,10 @@ impl CodeMap {
files.push(filemap.clone());
self.stable_id_to_filemap
.borrow_mut()
.insert(StableFilemapId::new(&filemap), filemap.clone());
filemap
}

19
src/test/incremental/warnings-reemitted.rs Normal file
View File

@ -0,0 +1,19 @@
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// revisions: cfail1 cfail2 cfail3
// compile-flags: -Coverflow-checks=on
// must-compile-successfully
#![allow(warnings)]
fn main() {
255u8 + 1; //~ WARNING this expression will panic at run-time
}