mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Create QuerySideEffects and use it for diagnostics

Browse Source
This commit is contained in:
Aaron Hill 2021-07-23 16:40:26 -05:00
parent 5782f01a51
commit e6a5231238
No known key found for this signature in database
GPG Key ID: B4087E510E98B164
5 changed files with 102 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
compiler/rustc_query_impl/src/on_disk_cache.rs
View File

@ -1,9 +1,7 @@
use crate::QueryCtxt;
use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, OnceCell};
use rustc_data_structures::thin_vec::ThinVec;
use rustc_data_structures::unhash::UnhashMap;
use rustc_errors::Diagnostic;
use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, LOCAL_CRATE};
use rustc_hir::definitions::DefPathHash;
use rustc_index::vec::{Idx, IndexVec};
@ -13,7 +11,7 @@
use rustc_middle::ty::codec::{RefDecodable, TyDecoder, TyEncoder};
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_query_system::dep_graph::DepContext;
use rustc_query_system::query::QueryContext;
use rustc_query_system::query::{QueryContext, QuerySideEffects};
use rustc_serialize::{
opaque::{self, FileEncodeResult, FileEncoder, IntEncodedWithFixedSize},
Decodable, Decoder, Encodable, Encoder,
@ -41,14 +39,14 @@
/// Provides an interface to incremental compilation data cached from the
/// previous compilation session. This data will eventually include the results
/// of a few selected queries (like `typeck` and `mir_optimized`) and
/// any diagnostics that have been emitted during a query.
/// any side effects that have been emitted during a query.
pub struct OnDiskCache<'sess> {
// The complete cache data in serialized form.
serialized_data: Vec<u8>,
// Collects all `Diagnostic`s emitted during the current compilation
// Collects all `QuerySideEffects` created during the current compilation
// session.
current_diagnostics: Lock<FxHashMap<DepNodeIndex, Vec<Diagnostic>>>,
current_side_effects: Lock<FxHashMap<DepNodeIndex, QuerySideEffects>>,
cnum_map: OnceCell<UnhashMap<StableCrateId, CrateNum>>,
@ -62,9 +60,9 @@ pub struct OnDiskCache<'sess> {
// `serialized_data`.
query_result_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
// A map from dep-node to the position of any associated diagnostics in
// A map from dep-node to the position of any associated `QuerySideEffects` in
// `serialized_data`.
prev_diagnostics_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
prev_side_effects_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
alloc_decoding_state: AllocDecodingState,
@ -113,8 +111,8 @@ pub struct OnDiskCache<'sess> {
#[derive(Encodable, Decodable)]
struct Footer {
file_index_to_stable_id: FxHashMap<SourceFileIndex, EncodedSourceFileId>,
query_result_index: EncodedQueryResultIndex,
diagnostics_index: EncodedQueryResultIndex,
query_result_index: EncodedDepNodeIndex,
side_effects_index: EncodedDepNodeIndex,
// The location of all allocations.
interpret_alloc_index: Vec<u32>,
// See `OnDiskCache.syntax_contexts`
@ -125,9 +123,7 @@ struct Footer {
foreign_expn_data: UnhashMap<ExpnHash, u32>,
}
pub type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
type EncodedDiagnosticsIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
type EncodedDiagnostics = Vec<Diagnostic>;
pub type EncodedDepNodeIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Encodable, Decodable)]
struct SourceFileIndex(u32);
@ -213,9 +209,9 @@ fn new(sess: &'sess Session, data: Vec<u8>, start_pos: usize) -> Self {
file_index_to_file: Default::default(),
cnum_map: OnceCell::new(),
source_map: sess.source_map(),
current_diagnostics: Default::default(),
current_side_effects: Default::default(),
query_result_index: footer.query_result_index.into_iter().collect(),
prev_diagnostics_index: footer.diagnostics_index.into_iter().collect(),
prev_side_effects_index: footer.side_effects_index.into_iter().collect(),
alloc_decoding_state: AllocDecodingState::new(footer.interpret_alloc_index),
syntax_contexts: footer.syntax_contexts,
expn_data: footer.expn_data,
@ -234,9 +230,9 @@ fn new_empty(source_map: &'sess SourceMap) -> Self {
file_index_to_file: Default::default(),
cnum_map: OnceCell::new(),
source_map,
current_diagnostics: Default::default(),
current_side_effects: Default::default(),
query_result_index: Default::default(),
prev_diagnostics_index: Default::default(),
prev_side_effects_index: Default::default(),
alloc_decoding_state: AllocDecodingState::new(Vec::new()),
syntax_contexts: FxHashMap::default(),
expn_data: UnhashMap::default(),
@ -301,7 +297,7 @@ fn serialize(&self, tcx: TyCtxt<'sess>, encoder: &mut FileEncoder) -> FileEncode
};
// Encode query results.
let mut query_result_index = EncodedQueryResultIndex::new();
let mut query_result_index = EncodedDepNodeIndex::new();
tcx.sess.time("encode_query_results", || -> FileEncodeResult {
let enc = &mut encoder;
@ -309,18 +305,16 @@ fn serialize(&self, tcx: TyCtxt<'sess>, encoder: &mut FileEncoder) -> FileEncode
QueryCtxt::from_tcx(tcx).encode_query_results(enc, qri)
})?;
// Encode diagnostics.
let diagnostics_index: EncodedDiagnosticsIndex = self
.current_diagnostics
// Encode side effects.
let side_effects_index: EncodedDepNodeIndex = self
.current_side_effects
.borrow()
.iter()
.map(
|(dep_node_index, diagnostics)| -> Result<_, <FileEncoder as Encoder>::Error> {
|(dep_node_index, side_effects)| -> Result<_, <FileEncoder as Encoder>::Error> {
let pos = AbsoluteBytePos::new(encoder.position());
// Let's make sure we get the expected type here.
let diagnostics: &EncodedDiagnostics = diagnostics;
let dep_node_index = SerializedDepNodeIndex::new(dep_node_index.index());
encoder.encode_tagged(dep_node_index, diagnostics)?;
encoder.encode_tagged(dep_node_index, side_effects)?;
Ok((dep_node_index, pos))
},
@ -386,7 +380,7 @@ fn serialize(&self, tcx: TyCtxt<'sess>, encoder: &mut FileEncoder) -> FileEncode
&Footer {
file_index_to_stable_id,
query_result_index,
diagnostics_index,
side_effects_index,
interpret_alloc_index,
syntax_contexts,
expn_data,
@ -488,30 +482,26 @@ pub fn as_dyn(&self) -> &dyn rustc_middle::ty::OnDiskCache<'sess> {
self as _
}
/// Loads a diagnostic emitted during the previous compilation session.
pub fn load_diagnostics(
/// Loads a `QuerySideEffects` created during the previous compilation session.
pub fn load_side_effects(
&self,
tcx: TyCtxt<'_>,
dep_node_index: SerializedDepNodeIndex,
) -> Vec<Diagnostic> {
let diagnostics: Option<EncodedDiagnostics> =
self.load_indexed(tcx, dep_node_index, &self.prev_diagnostics_index, "diagnostics");
) -> QuerySideEffects {
let side_effects: Option<QuerySideEffects> =
self.load_indexed(tcx, dep_node_index, &self.prev_side_effects_index, "side_effects");
diagnostics.unwrap_or_default()
side_effects.unwrap_or_default()
}
/// Stores a diagnostic emitted during the current compilation session.
/// Anything stored like this will be available via `load_diagnostics` in
/// Stores a `QuerySideEffects` emitted during the current compilation session.
/// Anything stored like this will be available via `load_side_effects` in
/// the next compilation session.
#[inline(never)]
#[cold]
pub fn store_diagnostics(
&self,
dep_node_index: DepNodeIndex,
diagnostics: ThinVec<Diagnostic>,
) {
let mut current_diagnostics = self.current_diagnostics.borrow_mut();
let prev = current_diagnostics.insert(dep_node_index, diagnostics.into());
pub fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
let mut current_side_effects = self.current_side_effects.borrow_mut();
let prev = current_side_effects.insert(dep_node_index, side_effects);
debug_assert!(prev.is_none());
}
@ -539,22 +529,21 @@ pub fn try_load_query_result<'tcx, T>(
self.load_indexed(tcx, dep_node_index, &self.query_result_index, "query result")
}
/// Stores a diagnostic emitted during computation of an anonymous query.
/// Stores side effect emitted during computation of an anonymous query.
/// Since many anonymous queries can share the same `DepNode`, we aggregate
/// them -- as opposed to regular queries where we assume that there is a
/// 1:1 relationship between query-key and `DepNode`.
#[inline(never)]
#[cold]
pub fn store_diagnostics_for_anon_node(
pub fn store_side_effects_for_anon_node(
&self,
dep_node_index: DepNodeIndex,
diagnostics: ThinVec<Diagnostic>,
side_effects: QuerySideEffects,
) {
let mut current_diagnostics = self.current_diagnostics.borrow_mut();
let mut current_side_effects = self.current_side_effects.borrow_mut();
let x = current_diagnostics.entry(dep_node_index).or_default();
x.extend(Into::<Vec<_>>::into(diagnostics));
let x = current_side_effects.entry(dep_node_index).or_default();
x.append(side_effects);
}
fn load_indexed<'tcx, T>(
@ -1155,7 +1144,7 @@ fn encode(&self, e: &mut CacheEncoder<'a, 'tcx, FileEncoder>) -> FileEncodeResul
pub fn encode_query_results<'a, 'tcx, CTX, Q>(
tcx: CTX,
encoder: &mut CacheEncoder<'a, 'tcx, FileEncoder>,
query_result_index: &mut EncodedQueryResultIndex,
query_result_index: &mut EncodedDepNodeIndex,
) -> FileEncodeResult
where
CTX: QueryContext + 'tcx,

20
compiler/rustc_query_impl/src/plumbing.rs
View File

@ -7,7 +7,9 @@
use rustc_middle::ty::tls::{self, ImplicitCtxt};
use rustc_middle::ty::{self, TyCtxt};
use rustc_query_system::dep_graph::HasDepContext;
use rustc_query_system::query::{QueryContext, QueryDescription, QueryJobId, QueryMap};
use rustc_query_system::query::{
QueryContext, QueryDescription, QueryJobId, QueryMap, QuerySideEffects,
};
use rustc_data_structures::sync::Lock;
use rustc_data_structures::thin_vec::ThinVec;
@ -83,27 +85,27 @@ fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool {
}
// Interactions with on_disk_cache
fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic> {
fn load_side_effects(&self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects {
self.queries
.on_disk_cache
.as_ref()
.map(|c| c.load_diagnostics(**self, prev_dep_node_index))
.map(|c| c.load_side_effects(**self, prev_dep_node_index))
.unwrap_or_default()
}
fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>) {
fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
if let Some(c) = self.queries.on_disk_cache.as_ref() {
c.store_diagnostics(dep_node_index, diagnostics)
c.store_side_effects(dep_node_index, side_effects)
}
}
fn store_diagnostics_for_anon_node(
fn store_side_effects_for_anon_node(
&self,
dep_node_index: DepNodeIndex,
diagnostics: ThinVec<Diagnostic>,
side_effects: QuerySideEffects,
) {
if let Some(c) = self.queries.on_disk_cache.as_ref() {
c.store_diagnostics_for_anon_node(dep_node_index, diagnostics)
c.store_side_effects_for_anon_node(dep_node_index, side_effects)
}
}
@ -163,7 +165,7 @@ pub unsafe fn deadlock(self, registry: &rustc_rayon_core::Registry) {
pub(super) fn encode_query_results(
self,
encoder: &mut on_disk_cache::CacheEncoder<'a, 'tcx, opaque::FileEncoder>,
query_result_index: &mut on_disk_cache::EncodedQueryResultIndex,
query_result_index: &mut on_disk_cache::EncodedDepNodeIndex,
) -> opaque::FileEncodeResult {
macro_rules! encode_queries {
($($query:ident,)*) => {

63
compiler/rustc_query_system/src/dep_graph/graph.rs
View File

@ -5,23 +5,20 @@
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::steal::Steal;
use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering};
use rustc_data_structures::unlikely;
use rustc_errors::Diagnostic;
use rustc_index::vec::IndexVec;
use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
use parking_lot::{Condvar, Mutex};
use parking_lot::Mutex;
use smallvec::{smallvec, SmallVec};
use std::collections::hash_map::Entry;
use std::hash::Hash;
use std::marker::PhantomData;
use std::mem;
use std::sync::atomic::Ordering::Relaxed;
use super::query::DepGraphQuery;
use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex};
use super::{DepContext, DepKind, DepNode, HasDepContext, WorkProductId};
use crate::query::QueryContext;
use crate::query::{QueryContext, QuerySideEffects};
#[cfg(debug_assertions)]
use {super::debug::EdgeFilter, std::env};
@ -87,11 +84,7 @@ struct DepGraphData<K: DepKind> {
colors: DepNodeColorMap,
/// A set of loaded diagnostics that is in the progress of being emitted.
emitting_diagnostics: Mutex<FxHashSet<DepNodeIndex>>,
/// Used to wait for diagnostics to be emitted.
emitting_diagnostics_cond_var: Condvar,
processed_side_effects: Mutex<FxHashSet<DepNodeIndex>>,
/// When we load, there may be `.o` files, cached MIR, or other such
/// things available to us. If we find that they are not dirty, we
@ -144,8 +137,7 @@ pub fn new(
previous_work_products: prev_work_products,
dep_node_debug: Default::default(),
current,
emitting_diagnostics: Default::default(),
emitting_diagnostics_cond_var: Condvar::new(),
processed_side_effects: Default::default(),
previous: prev_graph,
colors: DepNodeColorMap::new(prev_graph_node_count),
})),
@ -691,7 +683,7 @@ fn try_mark_previous_green<Ctxt: QueryContext<DepKind = K>>(
// FIXME: Store the fact that a node has diagnostics in a bit in the dep graph somewhere
// Maybe store a list on disk and encode this fact in the DepNodeState
let diagnostics = tcx.load_diagnostics(prev_dep_node_index);
let side_effects = tcx.load_side_effects(prev_dep_node_index);
#[cfg(not(parallel_compiler))]
debug_assert!(
@ -701,8 +693,8 @@ fn try_mark_previous_green<Ctxt: QueryContext<DepKind = K>>(
dep_node
);
if unlikely!(!diagnostics.is_empty()) {
self.emit_diagnostics(tcx, data, dep_node_index, prev_dep_node_index, diagnostics);
if unlikely!(!side_effects.is_empty()) {
self.emit_side_effects(tcx, data, dep_node_index, side_effects);
}
// ... and finally storing a "Green" entry in the color map.
@ -717,54 +709,27 @@ fn try_mark_previous_green<Ctxt: QueryContext<DepKind = K>>(
/// This may be called concurrently on multiple threads for the same dep node.
#[cold]
#[inline(never)]
fn emit_diagnostics<Ctxt: QueryContext<DepKind = K>>(
fn emit_side_effects<Ctxt: QueryContext<DepKind = K>>(
&self,
tcx: Ctxt,
data: &DepGraphData<K>,
dep_node_index: DepNodeIndex,
prev_dep_node_index: SerializedDepNodeIndex,
diagnostics: Vec<Diagnostic>,
side_effects: QuerySideEffects,
) {
let mut emitting = data.emitting_diagnostics.lock();
let mut processed = data.processed_side_effects.lock();
if data.colors.get(prev_dep_node_index) == Some(DepNodeColor::Green(dep_node_index)) {
// The node is already green so diagnostics must have been emitted already
return;
}
if emitting.insert(dep_node_index) {
if processed.insert(dep_node_index) {
// We were the first to insert the node in the set so this thread
// must emit the diagnostics and signal other potentially waiting
// threads after.
mem::drop(emitting);
// must process side effects
// Promote the previous diagnostics to the current session.
tcx.store_diagnostics(dep_node_index, diagnostics.clone().into());
tcx.store_side_effects(dep_node_index, side_effects.clone());
let handle = tcx.dep_context().sess().diagnostic();
for diagnostic in diagnostics {
for diagnostic in side_effects.diagnostics {
handle.emit_diagnostic(&diagnostic);
}
// Mark the node as green now that diagnostics are emitted
data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index));
// Remove the node from the set
data.emitting_diagnostics.lock().remove(&dep_node_index);
// Wake up waiters
data.emitting_diagnostics_cond_var.notify_all();
} else {
// We must wait for the other thread to finish emitting the diagnostic
loop {
data.emitting_diagnostics_cond_var.wait(&mut emitting);
if data.colors.get(prev_dep_node_index) == Some(DepNodeColor::Green(dep_node_index))
{
break;
}
}
}
}

35
compiler/rustc_query_system/src/query/mod.rs
View File

@ -62,6 +62,31 @@ pub fn default_span(&self, span: Span) -> Span {
}
}
/// Tracks 'side effects' for a particular query.
/// This struct is saved to disk along with the query result,
/// and loaded from disk if we mark the query as green.
/// This allows us to 'replay' changes to global state
/// that would otherwise only occur if we actually
/// executed the query method.
#[derive(Debug, Clone, Default, Encodable, Decodable)]
pub struct QuerySideEffects {
/// Stores any diagnostics emitted during query execution.
/// These diagnostics will be re-emitted if we mark
/// the query as green.
pub diagnostics: ThinVec<Diagnostic>,
}
impl QuerySideEffects {
pub fn is_empty(&self) -> bool {
let QuerySideEffects { diagnostics } = self;
diagnostics.is_empty()
}
pub fn append(&mut self, other: QuerySideEffects) {
let QuerySideEffects { diagnostics } = self;
diagnostics.extend(other.diagnostics);
}
}
pub trait QueryContext: HasDepContext {
/// Get the query information from the TLS context.
fn current_query_job(&self) -> Option<QueryJobId<Self::DepKind>>;
@ -74,17 +99,17 @@ pub trait QueryContext: HasDepContext {
/// Try to force a dep node to execute and see if it's green.
fn try_force_from_dep_node(&self, dep_node: &DepNode<Self::DepKind>) -> bool;
/// Load diagnostics associated to the node in the previous session.
fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic>;
/// Load side effects associated to the node in the previous session.
fn load_side_effects(&self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects;
/// Register diagnostics for the given node, for use in next session.
fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>);
fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects);
/// Register diagnostics for the given node, for use in next session.
fn store_diagnostics_for_anon_node(
fn store_side_effects_for_anon_node(
&self,
dep_node_index: DepNodeIndex,
diagnostics: ThinVec<Diagnostic>,
side_effects: QuerySideEffects,
);
/// Executes a job by changing the `ImplicitCtxt` to point to the

14
compiler/rustc_query_system/src/query/plumbing.rs
View File

@ -9,7 +9,7 @@
use crate::query::job::{
report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId,
};
use crate::query::{QueryContext, QueryMap, QueryStackFrame};
use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame};
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fx::{FxHashMap, FxHasher};
@ -479,8 +479,10 @@ fn try_execute_query<CTX, C>(
dep_graph.read_index(dep_node_index);
if unlikely!(!diagnostics.is_empty()) {
tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics);
let side_effects = QuerySideEffects { diagnostics };
if unlikely!(!side_effects.is_empty()) {
tcx.store_side_effects_for_anon_node(dep_node_index, side_effects);
}
return job.complete(result, dep_node_index);
@ -677,8 +679,10 @@ fn force_query_with_job<C, CTX>(
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
if unlikely!(!diagnostics.is_empty()) && dep_node.kind != DepKind::NULL {
tcx.store_diagnostics(dep_node_index, diagnostics);
let side_effects = QuerySideEffects { diagnostics };
if unlikely!(!side_effects.is_empty()) && dep_node.kind != DepKind::NULL {
tcx.store_side_effects(dep_node_index, side_effects);
}
let result = job.complete(result, dep_node_index);