mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move dep graph methods to DepGraphData to avoid branches and unwraps

Browse Source
This commit is contained in:
John Kåre Alsaker 2023-02-24 00:48:50 +01:00
parent 6a179026de
commit 60ed37c2e1
3 changed files with 222 additions and 162 deletions
compiler/rustc_query_system/src
dep_graph
graph.rsmod.rs
query
plumbing.rs

285
compiler/rustc_query_system/src/dep_graph/graph.rs

@ -70,7 +70,7 @@ impl DepNodeColor {
} }
} }
struct DepGraphData<K: DepKind> { pub struct DepGraphData<K: DepKind> {
/// The new encoding of the dependency graph, optimized for red/green /// The new encoding of the dependency graph, optimized for red/green
/// tracking. The `current` field is the dependency graph of only the /// tracking. The `current` field is the dependency graph of only the
/// current compilation session: We don't merge the previous dep-graph into /// current compilation session: We don't merge the previous dep-graph into
@ -168,6 +168,11 @@ impl<K: DepKind> DepGraph<K> {
DepGraph { data: None, virtual_dep_node_index: Lrc::new(AtomicU32::new(0)) } DepGraph { data: None, virtual_dep_node_index: Lrc::new(AtomicU32::new(0)) }
} }
#[inline]
pub fn data(&self) -> Option<&DepGraphData<K>> {
self.data.as_deref()
}
/// Returns `true` if we are actually building the full dep-graph, and `false` otherwise. /// Returns `true` if we are actually building the full dep-graph, and `false` otherwise.
#[inline] #[inline]
pub fn is_fully_enabled(&self) -> bool { pub fn is_fully_enabled(&self) -> bool {
@ -245,13 +250,45 @@ impl<K: DepKind> DepGraph<K> {
/// in the query infrastructure, and is not currently needed by the /// in the query infrastructure, and is not currently needed by the
/// decoding of any query results. Should the need arise in the future, /// decoding of any query results. Should the need arise in the future,
/// we should consider extending the query system with this functionality. /// we should consider extending the query system with this functionality.
pub fn with_query_deserialization<OP, R>(&self, op: OP) -> R pub fn with_query_deserialization<OP, R>(op: OP) -> R
where where
OP: FnOnce() -> R, OP: FnOnce() -> R,
{ {
K::with_deps(TaskDepsRef::Forbid, op) K::with_deps(TaskDepsRef::Forbid, op)
} }
#[inline(always)]
pub fn with_task<Ctxt: HasDepContext<DepKind = K>, A: Debug, R>(
&self,
key: DepNode<K>,
cx: Ctxt,
arg: A,
task: fn(Ctxt, A) -> R,
hash_result: Option<fn(&mut StableHashingContext<'_>, &R) -> Fingerprint>,
) -> (R, DepNodeIndex) {
match self.data() {
Some(data) => data.with_task(key, cx, arg, task, hash_result),
None => (task(cx, arg), self.next_virtual_depnode_index()),
}
}
pub fn with_anon_task<Tcx: DepContext<DepKind = K>, OP, R>(
&self,
cx: Tcx,
dep_kind: K,
op: OP,
) -> (R, DepNodeIndex)
where
OP: FnOnce() -> R,
{
match self.data() {
Some(data) => data.with_anon_task(cx, dep_kind, op),
None => (op(), self.next_virtual_depnode_index()),
}
}
}
impl<K: DepKind> DepGraphData<K> {
/// Starts a new dep-graph task. Dep-graph tasks are specified /// Starts a new dep-graph task. Dep-graph tasks are specified
/// using a free function (`task`) and **not** a closure -- this /// using a free function (`task`) and **not** a closure -- this
/// is intentional because we want to exercise tight control over /// is intentional because we want to exercise tight control over
@ -288,29 +325,6 @@ impl<K: DepKind> DepGraph<K> {
task: fn(Ctxt, A) -> R, task: fn(Ctxt, A) -> R,
hash_result: Option<fn(&mut StableHashingContext<'_>, &R) -> Fingerprint>, hash_result: Option<fn(&mut StableHashingContext<'_>, &R) -> Fingerprint>,
) -> (R, DepNodeIndex) { ) -> (R, DepNodeIndex) {
if self.is_fully_enabled() {
self.with_task_impl(key, cx, arg, task, hash_result)
} else {
// Incremental compilation is turned off. We just execute the task
// without tracking. We still provide a dep-node index that uniquely
// identifies the task so that we have a cheap way of referring to
// the query for self-profiling.
(task(cx, arg), self.next_virtual_depnode_index())
}
}
#[inline(always)]
fn with_task_impl<Ctxt: HasDepContext<DepKind = K>, A: Debug, R>(
&self,
key: DepNode<K>,
cx: Ctxt,
arg: A,
task: fn(Ctxt, A) -> R,
hash_result: Option<fn(&mut StableHashingContext<'_>, &R) -> Fingerprint>,
) -> (R, DepNodeIndex) {
// This function is only called when the graph is enabled.
let data = self.data.as_ref().unwrap();
// If the following assertion triggers, it can have two reasons: // If the following assertion triggers, it can have two reasons:
// 1. Something is wrong with DepNode creation, either here or // 1. Something is wrong with DepNode creation, either here or
// in `DepGraph::try_mark_green()`. // in `DepGraph::try_mark_green()`.
@ -351,9 +365,9 @@ impl<K: DepKind> DepGraph<K> {
let print_status = cfg!(debug_assertions) && dcx.sess().opts.unstable_opts.dep_tasks; let print_status = cfg!(debug_assertions) && dcx.sess().opts.unstable_opts.dep_tasks;
// Intern the new `DepNode`. // Intern the new `DepNode`.
let (dep_node_index, prev_and_color) = data.current.intern_node( let (dep_node_index, prev_and_color) = self.current.intern_node(
dcx.profiler(), dcx.profiler(),
&data.previous, &self.previous,
key, key,
edges, edges,
current_fingerprint, current_fingerprint,
@ -364,12 +378,12 @@ impl<K: DepKind> DepGraph<K> {
if let Some((prev_index, color)) = prev_and_color { if let Some((prev_index, color)) = prev_and_color {
debug_assert!( debug_assert!(
data.colors.get(prev_index).is_none(), self.colors.get(prev_index).is_none(),
"DepGraph::with_task() - Duplicate DepNodeColor \ "DepGraph::with_task() - Duplicate DepNodeColor \
insertion for {key:?}" insertion for {key:?}"
); );
data.colors.insert(prev_index, color); self.colors.insert(prev_index, color);
} }
(result, dep_node_index) (result, dep_node_index)
@ -388,57 +402,55 @@ impl<K: DepKind> DepGraph<K> {
{ {
debug_assert!(!cx.is_eval_always(dep_kind)); debug_assert!(!cx.is_eval_always(dep_kind));
if let Some(ref data) = self.data { let task_deps = Lock::new(TaskDeps::default());
let task_deps = Lock::new(TaskDeps::default()); let result = K::with_deps(TaskDepsRef::Allow(&task_deps), op);
let result = K::with_deps(TaskDepsRef::Allow(&task_deps), op); let task_deps = task_deps.into_inner();
let task_deps = task_deps.into_inner(); let task_deps = task_deps.reads;
let task_deps = task_deps.reads;
let dep_node_index = match task_deps.len() { let dep_node_index = match task_deps.len() {
0 => { 0 => {
// Because the dep-node id of anon nodes is computed from the sets of its // Because the dep-node id of anon nodes is computed from the sets of its
// dependencies we already know what the ID of this dependency-less node is // dependencies we already know what the ID of this dependency-less node is
// going to be (i.e. equal to the precomputed // going to be (i.e. equal to the precomputed
// `SINGLETON_DEPENDENCYLESS_ANON_NODE`). As a consequence we can skip creating // `SINGLETON_DEPENDENCYLESS_ANON_NODE`). As a consequence we can skip creating
// a `StableHasher` and sending the node through interning. // a `StableHasher` and sending the node through interning.
DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE
} }
1 => { 1 => {
// When there is only one dependency, don't bother creating a node. // When there is only one dependency, don't bother creating a node.
task_deps[0] task_deps[0]
} }
_ => { _ => {
// The dep node indices are hashed here instead of hashing the dep nodes of the // The dep node indices are hashed here instead of hashing the dep nodes of the
// dependencies. These indices may refer to different nodes per session, but this isn't // dependencies. These indices may refer to different nodes per session, but this isn't
// a problem here because we that ensure the final dep node hash is per session only by // a problem here because we that ensure the final dep node hash is per session only by
// combining it with the per session random number `anon_id_seed`. This hash only need // combining it with the per session random number `anon_id_seed`. This hash only need
// to map the dependencies to a single value on a per session basis. // to map the dependencies to a single value on a per session basis.
let mut hasher = StableHasher::new(); let mut hasher = StableHasher::new();
task_deps.hash(&mut hasher); task_deps.hash(&mut hasher);
let target_dep_node = DepNode { let target_dep_node = DepNode {
kind: dep_kind, kind: dep_kind,
// Fingerprint::combine() is faster than sending Fingerprint // Fingerprint::combine() is faster than sending Fingerprint
// through the StableHasher (at least as long as StableHasher // through the StableHasher (at least as long as StableHasher
// is so slow). // is so slow).
hash: data.current.anon_id_seed.combine(hasher.finish()).into(), hash: self.current.anon_id_seed.combine(hasher.finish()).into(),
}; };
data.current.intern_new_node( self.current.intern_new_node(
cx.profiler(), cx.profiler(),
target_dep_node, target_dep_node,
task_deps, task_deps,
Fingerprint::ZERO, Fingerprint::ZERO,
) )
} }
}; };
(result, dep_node_index) (result, dep_node_index)
} else {
(op(), self.next_virtual_depnode_index())
}
} }
}
impl<K: DepKind> DepGraph<K> {
#[inline] #[inline]
pub fn read_index(&self, dep_node_index: DepNodeIndex) { pub fn read_index(&self, dep_node_index: DepNodeIndex) {
if let Some(ref data) = self.data { if let Some(ref data) = self.data {
@ -521,7 +533,7 @@ impl<K: DepKind> DepGraph<K> {
// For sanity, we still check that the loaded stable hash and the new one match. // For sanity, we still check that the loaded stable hash and the new one match.
if let Some(dep_node_index) = self.dep_node_index_of_opt(&node) { if let Some(dep_node_index) = self.dep_node_index_of_opt(&node) {
let _current_fingerprint = let _current_fingerprint =
crate::query::incremental_verify_ich(cx, result, &node, hash_result); crate::query::incremental_verify_ich(cx, data, result, &node, hash_result);
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
if hash_result.is_some() { if hash_result.is_some() {
@ -585,24 +597,59 @@ impl<K: DepKind> DepGraph<K> {
#[inline] #[inline]
pub fn dep_node_index_of_opt(&self, dep_node: &DepNode<K>) -> Option<DepNodeIndex> { pub fn dep_node_index_of_opt(&self, dep_node: &DepNode<K>) -> Option<DepNodeIndex> {
let data = self.data.as_ref().unwrap(); self.data.as_ref().unwrap().dep_node_index_of_opt(dep_node)
let current = &data.current; }
}
if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) { impl<K: DepKind> DepGraphData<K> {
current.prev_index_to_index.lock()[prev_index] #[inline]
pub fn dep_node_index_of_opt(&self, dep_node: &DepNode<K>) -> Option<DepNodeIndex> {
if let Some(prev_index) = self.previous.node_to_index_opt(dep_node) {
self.current.prev_index_to_index.lock()[prev_index]
} else { } else {
current.new_node_to_index.get_shard_by_value(dep_node).lock().get(dep_node).copied() self.current
.new_node_to_index
.get_shard_by_value(dep_node)
.lock()
.get(dep_node)
.copied()
} }
} }
#[inline] #[inline]
pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool { pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
self.data.is_some() && self.dep_node_index_of_opt(dep_node).is_some() self.dep_node_index_of_opt(dep_node).is_some()
}
fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> {
if let Some(prev_index) = self.previous.node_to_index_opt(dep_node) {
self.colors.get(prev_index)
} else {
// This is a node that did not exist in the previous compilation session.
None
}
}
/// Returns true if the given node has been marked as green during the
/// current compilation session. Used in various assertions
pub fn is_green(&self, dep_node: &DepNode<K>) -> bool {
self.node_color(dep_node).map_or(false, |c| c.is_green())
} }
#[inline] #[inline]
pub fn prev_fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> { pub fn prev_fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
self.data.as_ref().unwrap().previous.fingerprint_of(dep_node) self.previous.fingerprint_of(dep_node)
}
pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode<K>) {
self.debug_loaded_from_disk.lock().insert(dep_node);
}
}
impl<K: DepKind> DepGraph<K> {
#[inline]
pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
self.data.is_some() && self.dep_node_index_of_opt(dep_node).is_some()
} }
/// Checks whether a previous work product exists for `v` and, if /// Checks whether a previous work product exists for `v` and, if
@ -617,10 +664,6 @@ impl<K: DepKind> DepGraph<K> {
&self.data.as_ref().unwrap().previous_work_products &self.data.as_ref().unwrap().previous_work_products
} }
pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode<K>) {
self.data.as_ref().unwrap().debug_loaded_from_disk.lock().insert(dep_node);
}
pub fn debug_was_loaded_from_disk(&self, dep_node: DepNode<K>) -> bool { pub fn debug_was_loaded_from_disk(&self, dep_node: DepNode<K>) -> bool {
self.data.as_ref().unwrap().debug_loaded_from_disk.lock().contains(&dep_node) self.data.as_ref().unwrap().debug_loaded_from_disk.lock().contains(&dep_node)
} }
@ -645,17 +688,22 @@ impl<K: DepKind> DepGraph<K> {
fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> { fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> {
if let Some(ref data) = self.data { if let Some(ref data) = self.data {
if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) { return data.node_color(dep_node);
return data.colors.get(prev_index);
} else {
// This is a node that did not exist in the previous compilation session.
return None;
}
} }
None None
} }
pub fn try_mark_green<Qcx: QueryContext<DepKind = K>>(
&self,
qcx: Qcx,
dep_node: &DepNode<K>,
) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> {
self.data().and_then(|data| data.try_mark_green(qcx, dep_node))
}
}
impl<K: DepKind> DepGraphData<K> {
/// Try to mark a node index for the node dep_node. /// Try to mark a node index for the node dep_node.
/// ///
/// A node will have an index, when it's already been marked green, or when we can mark it /// A node will have an index, when it's already been marked green, or when we can mark it
@ -668,26 +716,23 @@ impl<K: DepKind> DepGraph<K> {
) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> { ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> {
debug_assert!(!qcx.dep_context().is_eval_always(dep_node.kind)); debug_assert!(!qcx.dep_context().is_eval_always(dep_node.kind));
// Return None if the dep graph is disabled
let data = self.data.as_ref()?;
// Return None if the dep node didn't exist in the previous session // Return None if the dep node didn't exist in the previous session
let prev_index = data.previous.node_to_index_opt(dep_node)?; let prev_index = self.previous.node_to_index_opt(dep_node)?;
match data.colors.get(prev_index) { match self.colors.get(prev_index) {
Some(DepNodeColor::Green(dep_node_index)) => return Some((prev_index, dep_node_index)), Some(DepNodeColor::Green(dep_node_index)) => return Some((prev_index, dep_node_index)),
Some(DepNodeColor::Red) => return None, Some(DepNodeColor::Red) => return None,
None => {} None => {}
} }
let backtrace = backtrace_printer(qcx.dep_context().sess(), data, prev_index); let backtrace = backtrace_printer(qcx.dep_context().sess(), self, prev_index);
// This DepNode and the corresponding query invocation existed // This DepNode and the corresponding query invocation existed
// in the previous compilation session too, so we can try to // in the previous compilation session too, so we can try to
// mark it as green by recursively marking all of its // mark it as green by recursively marking all of its
// dependencies green. // dependencies green.
let ret = self let ret = self
.try_mark_previous_green(qcx, data, prev_index, &dep_node) .try_mark_previous_green(qcx, prev_index, &dep_node)
.map(|dep_node_index| (prev_index, dep_node_index)); .map(|dep_node_index| (prev_index, dep_node_index));
// We succeeded, no backtrace. // We succeeded, no backtrace.
@ -695,16 +740,15 @@ impl<K: DepKind> DepGraph<K> {
return ret; return ret;
} }
#[instrument(skip(self, qcx, data, parent_dep_node_index), level = "debug")] #[instrument(skip(self, qcx, parent_dep_node_index), level = "debug")]
fn try_mark_parent_green<Qcx: QueryContext<DepKind = K>>( fn try_mark_parent_green<Qcx: QueryContext<DepKind = K>>(
&self, &self,
qcx: Qcx, qcx: Qcx,
data: &DepGraphData<K>,
parent_dep_node_index: SerializedDepNodeIndex, parent_dep_node_index: SerializedDepNodeIndex,
dep_node: &DepNode<K>, dep_node: &DepNode<K>,
) -> Option<()> { ) -> Option<()> {
let dep_dep_node_color = data.colors.get(parent_dep_node_index); let dep_dep_node_color = self.colors.get(parent_dep_node_index);
let dep_dep_node = &data.previous.index_to_node(parent_dep_node_index); let dep_dep_node = &self.previous.index_to_node(parent_dep_node_index);
match dep_dep_node_color { match dep_dep_node_color {
Some(DepNodeColor::Green(_)) => { Some(DepNodeColor::Green(_)) => {
@ -733,8 +777,7 @@ impl<K: DepKind> DepGraph<K> {
dep_dep_node, dep_dep_node.hash, dep_dep_node, dep_dep_node.hash,
); );
let node_index = let node_index = self.try_mark_previous_green(qcx, parent_dep_node_index, dep_dep_node);
self.try_mark_previous_green(qcx, data, parent_dep_node_index, dep_dep_node);
if node_index.is_some() { if node_index.is_some() {
debug!("managed to MARK dependency {dep_dep_node:?} as green",); debug!("managed to MARK dependency {dep_dep_node:?} as green",);
@ -750,7 +793,7 @@ impl<K: DepKind> DepGraph<K> {
return None; return None;
} }
let dep_dep_node_color = data.colors.get(parent_dep_node_index); let dep_dep_node_color = self.colors.get(parent_dep_node_index);
match dep_dep_node_color { match dep_dep_node_color {
Some(DepNodeColor::Green(_)) => { Some(DepNodeColor::Green(_)) => {
@ -783,30 +826,29 @@ impl<K: DepKind> DepGraph<K> {
} }
/// Try to mark a dep-node which existed in the previous compilation session as green. /// Try to mark a dep-node which existed in the previous compilation session as green.
#[instrument(skip(self, qcx, data, prev_dep_node_index), level = "debug")] #[instrument(skip(self, qcx, prev_dep_node_index), level = "debug")]
fn try_mark_previous_green<Qcx: QueryContext<DepKind = K>>( fn try_mark_previous_green<Qcx: QueryContext<DepKind = K>>(
&self, &self,
qcx: Qcx, qcx: Qcx,
data: &DepGraphData<K>,
prev_dep_node_index: SerializedDepNodeIndex, prev_dep_node_index: SerializedDepNodeIndex,
dep_node: &DepNode<K>, dep_node: &DepNode<K>,
) -> Option<DepNodeIndex> { ) -> Option<DepNodeIndex> {
#[cfg(not(parallel_compiler))] #[cfg(not(parallel_compiler))]
{ {
debug_assert!(!self.dep_node_exists(dep_node)); debug_assert!(!self.dep_node_exists(dep_node));
debug_assert!(data.colors.get(prev_dep_node_index).is_none()); debug_assert!(self.colors.get(prev_dep_node_index).is_none());
} }
// We never try to mark eval_always nodes as green // We never try to mark eval_always nodes as green
debug_assert!(!qcx.dep_context().is_eval_always(dep_node.kind)); debug_assert!(!qcx.dep_context().is_eval_always(dep_node.kind));
debug_assert_eq!(data.previous.index_to_node(prev_dep_node_index), *dep_node); debug_assert_eq!(self.previous.index_to_node(prev_dep_node_index), *dep_node);
let prev_deps = data.previous.edge_targets_from(prev_dep_node_index); let prev_deps = self.previous.edge_targets_from(prev_dep_node_index);
for &dep_dep_node_index in prev_deps { for &dep_dep_node_index in prev_deps {
let backtrace = backtrace_printer(qcx.dep_context().sess(), data, dep_dep_node_index); let backtrace = backtrace_printer(qcx.dep_context().sess(), self, dep_dep_node_index);
let success = self.try_mark_parent_green(qcx, data, dep_dep_node_index, dep_node); let success = self.try_mark_parent_green(qcx, dep_dep_node_index, dep_node);
backtrace.disable(); backtrace.disable();
success?; success?;
} }
@ -819,9 +861,9 @@ impl<K: DepKind> DepGraph<K> {
// We allocating an entry for the node in the current dependency graph and // We allocating an entry for the node in the current dependency graph and
// adding all the appropriate edges imported from the previous graph // adding all the appropriate edges imported from the previous graph
let dep_node_index = data.current.promote_node_and_deps_to_current( let dep_node_index = self.current.promote_node_and_deps_to_current(
qcx.dep_context().profiler(), qcx.dep_context().profiler(),
&data.previous, &self.previous,
prev_dep_node_index, prev_dep_node_index,
); );
@ -833,20 +875,20 @@ impl<K: DepKind> DepGraph<K> {
#[cfg(not(parallel_compiler))] #[cfg(not(parallel_compiler))]
debug_assert!( debug_assert!(
data.colors.get(prev_dep_node_index).is_none(), self.colors.get(prev_dep_node_index).is_none(),
"DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \ "DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \
insertion for {dep_node:?}" insertion for {dep_node:?}"
); );
if !side_effects.is_empty() { if !side_effects.is_empty() {
self.with_query_deserialization(|| { DepGraph::<K>::with_query_deserialization(|| {
self.emit_side_effects(qcx, data, dep_node_index, side_effects) self.emit_side_effects(qcx, dep_node_index, side_effects)
}); });
} }
// ... and finally storing a "Green" entry in the color map. // ... and finally storing a "Green" entry in the color map.
// Multiple threads can all write the same color here // Multiple threads can all write the same color here
data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index)); self.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index));
debug!("successfully marked {dep_node:?} as green"); debug!("successfully marked {dep_node:?} as green");
Some(dep_node_index) Some(dep_node_index)
@ -859,11 +901,10 @@ impl<K: DepKind> DepGraph<K> {
fn emit_side_effects<Qcx: QueryContext<DepKind = K>>( fn emit_side_effects<Qcx: QueryContext<DepKind = K>>(
&self, &self,
qcx: Qcx, qcx: Qcx,
data: &DepGraphData<K>,
dep_node_index: DepNodeIndex, dep_node_index: DepNodeIndex,
side_effects: QuerySideEffects, side_effects: QuerySideEffects,
) { ) {
let mut processed = data.processed_side_effects.lock(); let mut processed = self.processed_side_effects.lock();
if processed.insert(dep_node_index) { if processed.insert(dep_node_index) {
// We were the first to insert the node in the set so this thread // We were the first to insert the node in the set so this thread
@ -879,7 +920,9 @@ impl<K: DepKind> DepGraph<K> {
} }
} }
} }
}
impl<K: DepKind> DepGraph<K> {
/// Returns true if the given node has been marked as red during the /// Returns true if the given node has been marked as red during the
/// current compilation session. Used in various assertions /// current compilation session. Used in various assertions
pub fn is_red(&self, dep_node: &DepNode<K>) -> bool { pub fn is_red(&self, dep_node: &DepNode<K>) -> bool {

3
compiler/rustc_query_system/src/dep_graph/mod.rs

@ -6,7 +6,8 @@ mod serialized;
pub use dep_node::{DepKindStruct, DepNode, DepNodeParams, WorkProductId}; pub use dep_node::{DepKindStruct, DepNode, DepNodeParams, WorkProductId};
pub use graph::{ pub use graph::{
hash_result, DepGraph, DepNodeColor, DepNodeIndex, TaskDeps, TaskDepsRef, WorkProduct, hash_result, DepGraph, DepGraphData, DepNodeColor, DepNodeIndex, TaskDeps, TaskDepsRef,
WorkProduct,
}; };
pub use query::DepGraphQuery; pub use query::DepGraphQuery;
pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex}; pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex};

96
compiler/rustc_query_system/src/query/plumbing.rs

@ -2,8 +2,10 @@
//! generate the actual methods on tcx which find and execute the provider, //! generate the actual methods on tcx which find and execute the provider,
//! manage the caches, and so forth. //! manage the caches, and so forth.
use crate::dep_graph::HasDepContext; use crate::dep_graph::{
use crate::dep_graph::{DepContext, DepKind, DepNode, DepNodeIndex, DepNodeParams}; DepContext, DepGraph, DepKind, DepNode, DepNodeIndex, DepNodeParams, TaskDepsRef,
};
use crate::dep_graph::{DepGraphData, HasDepContext};
use crate::ich::StableHashingContext; use crate::ich::StableHashingContext;
use crate::query::caches::QueryCache; use crate::query::caches::QueryCache;
use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo}; use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
@ -411,32 +413,35 @@ where
Qcx: QueryContext, Qcx: QueryContext,
{ {
let dep_graph = qcx.dep_context().dep_graph(); let dep_graph = qcx.dep_context().dep_graph();
let dep_graph_data = match dep_graph.data() {
// Fast path for when incr. comp. is off.
None => {
// Fingerprint the key, just to assert that it doesn't
// have anything we don't consider hashable
if cfg!(debug_assertions) {
let _ = key.to_fingerprint(*qcx.dep_context());
}
// Fast path for when incr. comp. is off. let prof_timer = qcx.dep_context().profiler().query_provider();
if !dep_graph.is_fully_enabled() { let result =
// Fingerprint the key, just to assert that it doesn't qcx.start_query(job_id, query.depth_limit(), None, || query.compute(qcx, key));
// have anything we don't consider hashable let dep_node_index = dep_graph.next_virtual_depnode_index();
if cfg!(debug_assertions) { prof_timer.finish_with_query_invocation_id(dep_node_index.into());
let _ = key.to_fingerprint(*qcx.dep_context());
}
let prof_timer = qcx.dep_context().profiler().query_provider(); // Similarly, fingerprint the result to assert that
let result = qcx.start_query(job_id, query.depth_limit(), None, || query.compute(qcx, key)); // it doesn't have anything not considered hashable.
let dep_node_index = dep_graph.next_virtual_depnode_index(); if cfg!(debug_assertions)
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
// Similarly, fingerprint the result to assert that
// it doesn't have anything not considered hashable.
if cfg!(debug_assertions)
&& let Some(hash_result) = query.hash_result() && let Some(hash_result) = query.hash_result()
{ {
qcx.dep_context().with_stable_hashing_context(|mut hcx| { qcx.dep_context().with_stable_hashing_context(|mut hcx| {
hash_result(&mut hcx, &result); hash_result(&mut hcx, &result);
}); });
} }
return (result, dep_node_index); return (result, dep_node_index);
} }
Some(data) => data,
};
if !query.anon() && !query.eval_always() { if !query.anon() && !query.eval_always() {
// `to_dep_node` is expensive for some `DepKind`s. // `to_dep_node` is expensive for some `DepKind`s.
@ -446,7 +451,7 @@ where
// The diagnostics for this query will be promoted to the current session during // The diagnostics for this query will be promoted to the current session during
// `try_mark_green()`, so we can ignore them here. // `try_mark_green()`, so we can ignore them here.
if let Some(ret) = qcx.start_query(job_id, false, None, || { if let Some(ret) = qcx.start_query(job_id, false, None, || {
try_load_from_disk_and_cache_in_memory(query, qcx, &key, &dep_node) try_load_from_disk_and_cache_in_memory(query, dep_graph_data, qcx, &key, &dep_node)
}) { }) {
return ret; return ret;
} }
@ -458,7 +463,7 @@ where
let (result, dep_node_index) = let (result, dep_node_index) =
qcx.start_query(job_id, query.depth_limit(), Some(&diagnostics), || { qcx.start_query(job_id, query.depth_limit(), Some(&diagnostics), || {
if query.anon() { if query.anon() {
return dep_graph.with_anon_task(*qcx.dep_context(), query.dep_kind(), || { return dep_graph_data.with_anon_task(*qcx.dep_context(), query.dep_kind(), || {
query.compute(qcx, key) query.compute(qcx, key)
}); });
} }
@ -467,7 +472,7 @@ where
let dep_node = let dep_node =
dep_node_opt.unwrap_or_else(|| query.construct_dep_node(*qcx.dep_context(), &key)); dep_node_opt.unwrap_or_else(|| query.construct_dep_node(*qcx.dep_context(), &key));
dep_graph.with_task( dep_graph_data.with_task(
dep_node, dep_node,
(qcx, query), (qcx, query),
key, key,
@ -495,6 +500,7 @@ where
#[inline(always)] #[inline(always)]
fn try_load_from_disk_and_cache_in_memory<Q, Qcx>( fn try_load_from_disk_and_cache_in_memory<Q, Qcx>(
query: Q, query: Q,
dep_graph_data: &DepGraphData<Qcx::DepKind>,
qcx: Qcx, qcx: Qcx,
key: &Q::Key, key: &Q::Key,
dep_node: &DepNode<Qcx::DepKind>, dep_node: &DepNode<Qcx::DepKind>,
@ -506,10 +512,9 @@ where
// Note this function can be called concurrently from the same query // Note this function can be called concurrently from the same query
// We must ensure that this is handled correctly. // We must ensure that this is handled correctly.
let dep_graph = qcx.dep_context().dep_graph(); let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, &dep_node)?;
let (prev_dep_node_index, dep_node_index) = dep_graph.try_mark_green(qcx, &dep_node)?;
debug_assert!(dep_graph.is_green(dep_node)); debug_assert!(dep_graph_data.is_green(dep_node));
// First we try to load the result from the on-disk cache. // First we try to load the result from the on-disk cache.
// Some things are never cached on disk. // Some things are never cached on disk.
@ -519,8 +524,9 @@ where
// The call to `with_query_deserialization` enforces that no new `DepNodes` // The call to `with_query_deserialization` enforces that no new `DepNodes`
// are created during deserialization. See the docs of that method for more // are created during deserialization. See the docs of that method for more
// details. // details.
let result = let result = DepGraph::<Qcx::DepKind>::with_query_deserialization(|| {
dep_graph.with_query_deserialization(|| try_load_from_disk(qcx, prev_dep_node_index)); try_load_from_disk(qcx, prev_dep_node_index)
});
prof_timer.finish_with_query_invocation_id(dep_node_index.into()); prof_timer.finish_with_query_invocation_id(dep_node_index.into());
@ -528,14 +534,11 @@ where
if std::intrinsics::unlikely( if std::intrinsics::unlikely(
qcx.dep_context().sess().opts.unstable_opts.query_dep_graph, qcx.dep_context().sess().opts.unstable_opts.query_dep_graph,
) { ) {
dep_graph.mark_debug_loaded_from_disk(*dep_node) dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
} }
let prev_fingerprint = qcx let prev_fingerprint =
.dep_context() dep_graph_data.prev_fingerprint_of(dep_node).unwrap_or(Fingerprint::ZERO);
.dep_graph()
.prev_fingerprint_of(dep_node)
.unwrap_or(Fingerprint::ZERO);
// If `-Zincremental-verify-ich` is specified, re-hash results from // If `-Zincremental-verify-ich` is specified, re-hash results from
// the cache and make sure that they have the expected fingerprint. // the cache and make sure that they have the expected fingerprint.
// //
@ -547,7 +550,13 @@ where
if std::intrinsics::unlikely( if std::intrinsics::unlikely(
try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich, try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich,
) { ) {
incremental_verify_ich(*qcx.dep_context(), &result, dep_node, query.hash_result()); incremental_verify_ich(
*qcx.dep_context(),
dep_graph_data,
&result,
dep_node,
query.hash_result(),
);
} }
return Some((result, dep_node_index)); return Some((result, dep_node_index));
@ -566,7 +575,7 @@ where
let prof_timer = qcx.dep_context().profiler().query_provider(); let prof_timer = qcx.dep_context().profiler().query_provider();
// The dep-graph for this computation is already in-place. // The dep-graph for this computation is already in-place.
let result = dep_graph.with_ignore(|| query.compute(qcx, *key)); let result = Qcx::DepKind::with_deps(TaskDepsRef::Ignore, || query.compute(qcx, *key));
prof_timer.finish_with_query_invocation_id(dep_node_index.into()); prof_timer.finish_with_query_invocation_id(dep_node_index.into());
@ -579,15 +588,22 @@ where
// //
// See issue #82920 for an example of a miscompilation that would get turned into // See issue #82920 for an example of a miscompilation that would get turned into
// an ICE by this check // an ICE by this check
incremental_verify_ich(*qcx.dep_context(), &result, dep_node, query.hash_result()); incremental_verify_ich(
*qcx.dep_context(),
dep_graph_data,
&result,
dep_node,
query.hash_result(),
);
Some((result, dep_node_index)) Some((result, dep_node_index))
} }
#[inline] #[inline]
#[instrument(skip(tcx, result, hash_result), level = "debug")] #[instrument(skip(tcx, dep_graph_data, result, hash_result), level = "debug")]
pub(crate) fn incremental_verify_ich<Tcx, V: Debug>( pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
tcx: Tcx, tcx: Tcx,
dep_graph_data: &DepGraphData<Tcx::DepKind>,
result: &V, result: &V,
dep_node: &DepNode<Tcx::DepKind>, dep_node: &DepNode<Tcx::DepKind>,
hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>, hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>,
@ -596,7 +612,7 @@ where
Tcx: DepContext, Tcx: DepContext,
{ {
assert!( assert!(
tcx.dep_graph().is_green(dep_node), dep_graph_data.is_green(dep_node),
"fingerprint for green query instance not loaded from cache: {dep_node:?}", "fingerprint for green query instance not loaded from cache: {dep_node:?}",
); );
@ -604,7 +620,7 @@ where
tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result)) tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result))
}); });
let old_hash = tcx.dep_graph().prev_fingerprint_of(dep_node); let old_hash = dep_graph_data.prev_fingerprint_of(dep_node);
if Some(new_hash) != old_hash { if Some(new_hash) != old_hash {
incremental_verify_ich_failed( incremental_verify_ich_failed(