mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #69808 - cjgillot:vtbl, r=pnkfelix

Browse Source 
Avoid duplicating code for each query

There are at the moment roughly 170 queries in librustc.
The way `ty::query` is structured, a lot of code is duplicated for each query.
I suspect this to be responsible for a part of librustc'c compile time.

The first part of this PR reduces the amount of code generic on the query,
replacing it by code generic on the key-value types. I can split it out if needed.

In a second part, the non-inlined methods in the `QueryAccessors` and `QueryDescription` traits
are made into a virtual dispatch table. This allows to reduce even more the number of generated
functions.

This allows to save 1.5s on check build, and 10% on the size of the librustc.rlib.
(Attributed roughly half and half).
My computer is not good enough to measure properly compiling time.
I have no idea of the effect on performance. A perf run may be required.

cc #65031
This commit is contained in:
bors 2020-05-01 21:08:23 +00:00
commit dba944a6b7
10 changed files with 240 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/librustc_macros/src/query.rs
View File

@ -365,7 +365,7 @@ fn add_query_description_impl(
#[allow(unused_variables, unused_braces)]
fn cache_on_disk(
#tcx: TyCtxt<'tcx>,
#key: Self::Key,
#key: &Self::Key,
#value: Option<&Self::Value>
) -> bool {
#expr
@ -441,7 +441,7 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream {
.unwrap_or(false));
let key = <#arg as DepNodeParams<TyCtxt<'_>>>::recover($tcx, $dep_node).unwrap();
if queries::#name::cache_on_disk($tcx, key, None) {
if queries::#name::cache_on_disk($tcx, &key, None) {
let _ = $tcx.#name(key);
}
}

25
src/librustc_middle/dep_graph/dep_node.rs
View File

@ -183,31 +183,10 @@ macro_rules! define_dep_nodes {
// tuple args
$({
erase!($tuple_arg_ty);
let hash = DepNodeParams::to_fingerprint(&arg, _tcx);
let dep_node = DepNode {
kind: DepKind::$variant,
hash
};
#[cfg(debug_assertions)]
{
if !dep_node.kind.can_reconstruct_query_key() &&
(_tcx.sess.opts.debugging_opts.incremental_info ||
_tcx.sess.opts.debugging_opts.query_dep_graph)
{
_tcx.dep_graph.register_dep_node_debug_str(dep_node, || {
arg.to_debug_str(_tcx)
});
}
}
return dep_node;
return DepNode::construct(_tcx, DepKind::$variant, &arg)
})*
DepNode {
kind: DepKind::$variant,
hash: Fingerprint::ZERO,
}
return DepNode::construct(_tcx, DepKind::$variant, &())
}
)*
}

4
src/librustc_middle/dep_graph/mod.rs
View File

@ -98,6 +98,10 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
fn debug_dep_tasks(&self) -> bool {
self.sess.opts.debugging_opts.dep_tasks
}
fn debug_dep_node(&self) -> bool {
self.sess.opts.debugging_opts.incremental_info
|| self.sess.opts.debugging_opts.query_dep_graph
}
fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool {
// FIXME: This match is just a workaround for incremental bugs and should

2
src/librustc_middle/ty/query/mod.rs
View File

@ -1,4 +1,4 @@
use crate::dep_graph::{self, DepConstructor, DepNode, DepNodeParams};
use crate::dep_graph::{self, DepNode, DepNodeParams};
use crate::hir::exports::Export;
use crate::hir::map;
use crate::infer::canonical::{self, Canonical};

2
src/librustc_middle/ty/query/on_disk_cache.rs
View File

@ -1009,7 +1009,7 @@ where
state.iter_results(|results| {
for (key, value, dep_node) in results {
if Q::cache_on_disk(tcx, key.clone(), Some(&value)) {
if Q::cache_on_disk(tcx, &key, Some(&value)) {
let dep_node = SerializedDepNodeIndex::new(dep_node.index());
// Record position of the cache entry.

6
src/librustc_middle/ty/query/plumbing.rs
View File

@ -348,12 +348,6 @@ macro_rules! define_queries_inner {
&tcx.queries.$name
}
#[allow(unused)]
#[inline(always)]
fn to_dep_node(tcx: TyCtxt<$tcx>, key: &Self::Key) -> DepNode {
DepConstructor::$node(tcx, *key)
}
#[inline]
fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
let provider = tcx.queries.providers.get(key.query_crate())

24
src/librustc_query_system/dep_graph/dep_node.rs
View File

@ -64,6 +64,24 @@ impl<K: DepKind> DepNode<K> {
debug_assert!(!kind.has_params());
DepNode { kind, hash: Fingerprint::ZERO }
}
pub fn construct<Ctxt, Key>(tcx: Ctxt, kind: K, arg: &Key) -> DepNode<K>
where
Ctxt: crate::query::QueryContext<DepKind = K>,
Key: DepNodeParams<Ctxt>,
{
let hash = arg.to_fingerprint(tcx);
let dep_node = DepNode { kind, hash };
#[cfg(debug_assertions)]
{
if !kind.can_reconstruct_query_key() && tcx.debug_dep_node() {
tcx.dep_graph().register_dep_node_debug_str(dep_node, || arg.to_debug_str(tcx));
}
}
return dep_node;
}
}
impl<K: DepKind> fmt::Debug for DepNode<K> {
@ -120,6 +138,12 @@ where
}
}
impl<Ctxt: DepContext> DepNodeParams<Ctxt> for () {
fn to_fingerprint(&self, _: Ctxt) -> Fingerprint {
Fingerprint::ZERO
}
}
/// A "work product" corresponds to a `.o` (or other) file that we
/// save in between runs. These IDs do not have a `DefId` but rather
/// some independent path or string that persists between runs without

1
src/librustc_query_system/dep_graph/mod.rs
View File

@ -28,6 +28,7 @@ pub trait DepContext: Copy {
fn create_stable_hashing_context(&self) -> Self::StableHashingContext;
fn debug_dep_tasks(&self) -> bool;
fn debug_dep_node(&self) -> bool;
/// Try to force a dep node to execute and see if it's green.
fn try_force_from_dep_node(&self, dep_node: &DepNode<Self::DepKind>) -> bool;

79
src/librustc_query_system/query/config.rs
View File

@ -24,6 +24,53 @@ pub trait QueryConfig<CTX> {
type Stored: Clone;
}
pub(crate) struct QueryVtable<CTX: QueryContext, K, V> {
pub anon: bool,
pub dep_kind: CTX::DepKind,
pub eval_always: bool,
// Don't use this method to compute query results, instead use the methods on TyCtxt
pub compute: fn(CTX, K) -> V,
pub hash_result: fn(&mut CTX::StableHashingContext, &V) -> Option<Fingerprint>,
pub handle_cycle_error: fn(CTX, CycleError<CTX::Query>) -> V,
pub cache_on_disk: fn(CTX, &K, Option<&V>) -> bool,
pub try_load_from_disk: fn(CTX, SerializedDepNodeIndex) -> Option<V>,
}
impl<CTX: QueryContext, K, V> QueryVtable<CTX, K, V> {
pub(crate) fn to_dep_node(&self, tcx: CTX, key: &K) -> DepNode<CTX::DepKind>
where
K: crate::dep_graph::DepNodeParams<CTX>,
{
DepNode::construct(tcx, self.dep_kind, key)
}
pub(crate) fn compute(&self, tcx: CTX, key: K) -> V {
(self.compute)(tcx, key)
}
pub(crate) fn hash_result(
&self,
hcx: &mut CTX::StableHashingContext,
value: &V,
) -> Option<Fingerprint> {
(self.hash_result)(hcx, value)
}
pub(crate) fn handle_cycle_error(&self, tcx: CTX, error: CycleError<CTX::Query>) -> V {
(self.handle_cycle_error)(tcx, error)
}
pub(crate) fn cache_on_disk(&self, tcx: CTX, key: &K, value: Option<&V>) -> bool {
(self.cache_on_disk)(tcx, key, value)
}
pub(crate) fn try_load_from_disk(&self, tcx: CTX, index: SerializedDepNodeIndex) -> Option<V> {
(self.try_load_from_disk)(tcx, index)
}
}
pub trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> {
const ANON: bool;
const EVAL_ALWAYS: bool;
@ -34,7 +81,12 @@ pub trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> {
// Don't use this method to access query results, instead use the methods on TyCtxt
fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX, Self::Cache>;
fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind>;
fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind>
where
Self::Key: crate::dep_graph::DepNodeParams<CTX>,
{
DepNode::construct(tcx, Self::DEP_KIND, key)
}
// Don't use this method to compute query results, instead use the methods on TyCtxt
fn compute(tcx: CTX, key: Self::Key) -> Self::Value;
@ -51,7 +103,7 @@ pub trait QueryDescription<CTX: QueryContext>: QueryAccessors<CTX> {
fn describe(tcx: CTX, key: Self::Key) -> Cow<'static, str>;
#[inline]
fn cache_on_disk(_: CTX, _: Self::Key, _: Option<&Self::Value>) -> bool {
fn cache_on_disk(_: CTX, _: &Self::Key, _: Option<&Self::Value>) -> bool {
false
}
@ -60,6 +112,27 @@ pub trait QueryDescription<CTX: QueryContext>: QueryAccessors<CTX> {
}
}
pub(crate) trait QueryVtableExt<CTX: QueryContext, K, V> {
const VTABLE: QueryVtable<CTX, K, V>;
}
impl<CTX, Q> QueryVtableExt<CTX, Q::Key, Q::Value> for Q
where
CTX: QueryContext,
Q: QueryDescription<CTX>,
{
const VTABLE: QueryVtable<CTX, Q::Key, Q::Value> = QueryVtable {
anon: Q::ANON,
dep_kind: Q::DEP_KIND,
eval_always: Q::EVAL_ALWAYS,
compute: Q::compute,
hash_result: Q::hash_result,
handle_cycle_error: Q::handle_cycle_error,
cache_on_disk: Q::cache_on_disk,
try_load_from_disk: Q::try_load_from_disk,
};
}
impl<CTX: QueryContext, M> QueryDescription<CTX> for M
where
M: QueryAccessors<CTX, Key = DefId>,
@ -73,7 +146,7 @@ where
}
}
default fn cache_on_disk(_: CTX, _: Self::Key, _: Option<&Self::Value>) -> bool {
default fn cache_on_disk(_: CTX, _: &Self::Key, _: Option<&Self::Value>) -> bool {
false
}

196
src/librustc_query_system/query/plumbing.rs
View File

@ -5,7 +5,7 @@
use crate::dep_graph::{DepKind, DepNode};
use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
use crate::query::caches::QueryCache;
use crate::query::config::QueryDescription;
use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt};
use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId};
use crate::query::QueryContext;
@ -29,7 +29,7 @@ use std::ptr;
#[cfg(debug_assertions)]
use std::sync::atomic::{AtomicUsize, Ordering};
pub struct QueryStateShard<CTX: QueryContext, K, C> {
pub(super) struct QueryStateShard<CTX: QueryContext, K, C> {
pub(super) cache: C,
active: FxHashMap<K, QueryResult<CTX>>,
@ -80,6 +80,7 @@ enum QueryResult<CTX: QueryContext> {
}
impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> {
#[inline(always)]
pub fn iter_results<R>(
&self,
f: impl for<'a> FnOnce(
@ -89,6 +90,7 @@ impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> {
self.cache.iter(&self.shards, |shard| &mut shard.cache, f)
}
#[inline(always)]
pub fn all_inactive(&self) -> bool {
let shards = self.shards.lock_shards();
shards.iter().all(|shard| shard.active.is_empty())
@ -168,14 +170,15 @@ where
/// This function is inlined because that results in a noticeable speed-up
/// for some compile-time benchmarks.
#[inline(always)]
fn try_start<'a, 'b, Q>(
fn try_start<'a, 'b>(
tcx: CTX,
state: &'b QueryState<CTX, C>,
span: Span,
key: &C::Key,
mut lookup: QueryLookup<'a, CTX, C::Key, C::Sharded>,
query: &QueryVtable<CTX, C::Key, C::Value>,
) -> TryGetJob<'b, CTX, C>
where
Q: QueryDescription<CTX, Key = C::Key, Stored = C::Stored, Value = C::Value, Cache = C>,
CTX: QueryContext,
{
let lock = &mut *lookup.lock;
@ -194,7 +197,7 @@ where
};
// Create the id of the job we're waiting for
let id = QueryJobId::new(job.id, lookup.shard, Q::DEP_KIND);
let id = QueryJobId::new(job.id, lookup.shard, query.dep_kind);
(job.latch(id), _query_blocked_prof_timer)
}
@ -209,15 +212,14 @@ where
lock.jobs = id;
let id = QueryShardJobId(NonZeroU32::new(id).unwrap());
let global_id = QueryJobId::new(id, lookup.shard, Q::DEP_KIND);
let global_id = QueryJobId::new(id, lookup.shard, query.dep_kind);
let job = tcx.current_query_job();
let job = QueryJob::new(id, span, job);
entry.insert(QueryResult::Started(job));
let owner =
JobOwner { state: Q::query_state(tcx), id: global_id, key: (*key).clone() };
let owner = JobOwner { state, id: global_id, key: (*key).clone() };
return TryGetJob::NotYetStarted(owner);
}
};
@ -227,8 +229,8 @@ where
// so we just return the error.
#[cfg(not(parallel_compiler))]
return TryGetJob::Cycle(cold_path(|| {
let value = Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span));
Q::query_state(tcx).cache.store_nocache(value)
let value = query.handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span));
state.cache.store_nocache(value)
}));
// With parallel queries we might just have to wait on some other
@ -238,14 +240,14 @@ where
let result = latch.wait_on(tcx, span);
if let Err(cycle) = result {
let value = Q::handle_cycle_error(tcx, cycle);
let value = Q::query_state(tcx).cache.store_nocache(value);
let value = query.handle_cycle_error(tcx, cycle);
let value = state.cache.store_nocache(value);
return TryGetJob::Cycle(value);
}
let cached = try_get_cached(
tcx,
Q::query_state(tcx),
state,
(*key).clone(),
|value, index| (value.clone(), index),
|_, _| panic!("value must be in cache after waiting"),
@ -382,17 +384,21 @@ where
}
#[inline(always)]
fn try_execute_query<Q, CTX>(
fn try_execute_query<CTX, C>(
tcx: CTX,
state: &QueryState<CTX, C>,
span: Span,
key: Q::Key,
lookup: QueryLookup<'_, CTX, Q::Key, <Q::Cache as QueryCache>::Sharded>,
) -> Q::Stored
key: C::Key,
lookup: QueryLookup<'_, CTX, C::Key, C::Sharded>,
query: &QueryVtable<CTX, C::Key, C::Value>,
) -> C::Stored
where
Q: QueryDescription<CTX>,
C: QueryCache,
C::Key: Eq + Clone + Debug + crate::dep_graph::DepNodeParams<CTX>,
C::Stored: Clone,
CTX: QueryContext,
{
let job = match JobOwner::try_start::<Q>(tcx, span, &key, lookup) {
let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) {
TryGetJob::NotYetStarted(job) => job,
TryGetJob::Cycle(result) => return result,
#[cfg(parallel_compiler)]
@ -406,15 +412,15 @@ where
// expensive for some `DepKind`s.
if !tcx.dep_graph().is_fully_enabled() {
let null_dep_node = DepNode::new_no_params(DepKind::NULL);
return force_query_with_job::<Q, _>(tcx, key, job, null_dep_node).0;
return force_query_with_job(tcx, key, job, null_dep_node, query).0;
}
if Q::ANON {
if query.anon {
let prof_timer = tcx.profiler().query_provider();
let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
tcx.start_query(job.id, diagnostics, |tcx| {
tcx.dep_graph().with_anon_task(Q::DEP_KIND, || Q::compute(tcx, key))
tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key))
})
});
@ -429,9 +435,9 @@ where
return job.complete(tcx, result, dep_node_index);
}
let dep_node = Q::to_dep_node(tcx, &key);
let dep_node = query.to_dep_node(tcx, &key);
if !Q::EVAL_ALWAYS {
if !query.eval_always {
// The diagnostics for this query will be
// promoted to the current session during
// `try_mark_green()`, so we can ignore them here.
@ -439,12 +445,13 @@ where
let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node);
marked.map(|(prev_dep_node_index, dep_node_index)| {
(
load_from_disk_and_cache_in_memory::<Q, _>(
load_from_disk_and_cache_in_memory(
tcx,
key.clone(),
prev_dep_node_index,
dep_node_index,
&dep_node,
query,
),
dep_node_index,
)
@ -455,21 +462,21 @@ where
}
}
let (result, dep_node_index) = force_query_with_job::<Q, _>(tcx, key, job, dep_node);
let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, query);
tcx.dep_graph().read_index(dep_node_index);
result
}
fn load_from_disk_and_cache_in_memory<Q, CTX>(
fn load_from_disk_and_cache_in_memory<CTX, K, V>(
tcx: CTX,
key: Q::Key,
key: K,
prev_dep_node_index: SerializedDepNodeIndex,
dep_node_index: DepNodeIndex,
dep_node: &DepNode<CTX::DepKind>,
) -> Q::Value
query: &QueryVtable<CTX, K, V>,
) -> V
where
CTX: QueryContext,
Q: QueryDescription<CTX>,
{
// Note this function can be called concurrently from the same query
// We must ensure that this is handled correctly.
@ -477,9 +484,9 @@ where
debug_assert!(tcx.dep_graph().is_green(dep_node));
// First we try to load the result from the on-disk cache.
let result = if Q::cache_on_disk(tcx, key.clone(), None) {
let result = if query.cache_on_disk(tcx, &key, None) {
let prof_timer = tcx.profiler().incr_cache_loading();
let result = Q::try_load_from_disk(tcx, prev_dep_node_index);
let result = query.try_load_from_disk(tcx, prev_dep_node_index);
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
// We always expect to find a cached result for things that
@ -503,7 +510,7 @@ where
let prof_timer = tcx.profiler().query_provider();
// The dep-graph for this computation is already in-place.
let result = tcx.dep_graph().with_ignore(|| Q::compute(tcx, key));
let result = tcx.dep_graph().with_ignore(|| query.compute(tcx, key));
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
@ -513,7 +520,7 @@ where
// If `-Zincremental-verify-ich` is specified, re-hash results from
// the cache and make sure that they have the expected fingerprint.
if unlikely!(tcx.incremental_verify_ich()) {
incremental_verify_ich::<Q, _>(tcx, &result, dep_node, dep_node_index);
incremental_verify_ich(tcx, &result, dep_node, dep_node_index, query);
}
result
@ -521,14 +528,14 @@ where
#[inline(never)]
#[cold]
fn incremental_verify_ich<Q, CTX>(
fn incremental_verify_ich<CTX, K, V>(
tcx: CTX,
result: &Q::Value,
result: &V,
dep_node: &DepNode<CTX::DepKind>,
dep_node_index: DepNodeIndex,
query: &QueryVtable<CTX, K, V>,
) where
CTX: QueryContext,
Q: QueryDescription<CTX>,
{
assert!(
Some(tcx.dep_graph().fingerprint_of(dep_node_index))
@ -540,7 +547,7 @@ fn incremental_verify_ich<Q, CTX>(
debug!("BEGIN verify_ich({:?})", dep_node);
let mut hcx = tcx.create_stable_hashing_context();
let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
let new_hash = query.hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
debug!("END verify_ich({:?})", dep_node);
let old_hash = tcx.dep_graph().fingerprint_of(dep_node_index);
@ -549,14 +556,17 @@ fn incremental_verify_ich<Q, CTX>(
}
#[inline(always)]
fn force_query_with_job<Q, CTX>(
fn force_query_with_job<C, CTX>(
tcx: CTX,
key: Q::Key,
job: JobOwner<'_, CTX, Q::Cache>,
key: C::Key,
job: JobOwner<'_, CTX, C>,
dep_node: DepNode<CTX::DepKind>,
) -> (Q::Stored, DepNodeIndex)
query: &QueryVtable<CTX, C::Key, C::Value>,
) -> (C::Stored, DepNodeIndex)
where
Q: QueryDescription<CTX>,
C: QueryCache,
C::Key: Eq + Clone + Debug,
C::Stored: Clone,
CTX: QueryContext,
{
// If the following assertion triggers, it can have two reasons:
@ -577,16 +587,16 @@ where
let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
tcx.start_query(job.id, diagnostics, |tcx| {
if Q::EVAL_ALWAYS {
if query.eval_always {
tcx.dep_graph().with_eval_always_task(
dep_node,
tcx,
key,
Q::compute,
Q::hash_result,
query.compute,
query.hash_result,
)
} else {
tcx.dep_graph().with_task(dep_node, tcx, key, Q::compute, Q::hash_result)
tcx.dep_graph().with_task(dep_node, tcx, key, query.compute, query.hash_result)
}
})
});
@ -605,22 +615,28 @@ where
}
#[inline(never)]
pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored
fn get_query_impl<CTX, C>(
tcx: CTX,
state: &QueryState<CTX, C>,
span: Span,
key: C::Key,
query: &QueryVtable<CTX, C::Key, C::Value>,
) -> C::Stored
where
Q: QueryDescription<CTX>,
CTX: QueryContext,
C: QueryCache,
C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
C::Stored: Clone,
{
debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
try_get_cached(
tcx,
Q::query_state(tcx),
state,
key,
|value, index| {
tcx.dep_graph().read_index(index);
value.clone()
},
|key, lookup| try_execute_query::<Q, _>(tcx, span, key, lookup),
|key, lookup| try_execute_query(tcx, state, span, key, lookup, query),
)
}
@ -631,20 +647,26 @@ where
/// side-effects -- e.g., in order to report errors for erroneous programs.
///
/// Note: The optimization is only available during incr. comp.
pub fn ensure_query<Q, CTX>(tcx: CTX, key: Q::Key)
where
Q: QueryDescription<CTX>,
#[inline(never)]
fn ensure_query_impl<CTX, C>(
tcx: CTX,
state: &QueryState<CTX, C>,
key: C::Key,
query: &QueryVtable<CTX, C::Key, C::Value>,
) where
C: QueryCache,
C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
CTX: QueryContext,
{
if Q::EVAL_ALWAYS {
let _ = get_query::<Q, _>(tcx, DUMMY_SP, key);
if query.eval_always {
let _ = get_query_impl(tcx, state, DUMMY_SP, key, query);
return;
}
// Ensuring an anonymous query makes no sense
assert!(!Q::ANON);
assert!(!query.anon);
let dep_node = Q::to_dep_node(tcx, &key);
let dep_node = query.to_dep_node(tcx, &key);
match tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node) {
None => {
@ -654,7 +676,7 @@ where
// DepNodeIndex. We must invoke the query itself. The performance cost
// this introduces should be negligible as we'll immediately hit the
// in-memory cache, or another query down the line will.
let _ = get_query::<Q, _>(tcx, DUMMY_SP, key);
let _ = get_query_impl(tcx, state, DUMMY_SP, key, query);
}
Some((_, dep_node_index)) => {
tcx.profiler().query_cache_hit(dep_node_index.into());
@ -662,9 +684,17 @@ where
}
}
pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode<CTX::DepKind>)
where
Q: QueryDescription<CTX>,
#[inline(never)]
fn force_query_impl<CTX, C>(
tcx: CTX,
state: &QueryState<CTX, C>,
key: C::Key,
span: Span,
dep_node: DepNode<CTX::DepKind>,
query: &QueryVtable<CTX, C::Key, C::Value>,
) where
C: QueryCache,
C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
CTX: QueryContext,
{
// We may be concurrently trying both execute and force a query.
@ -672,19 +702,51 @@ where
try_get_cached(
tcx,
Q::query_state(tcx),
state,
key,
|_, _| {
// Cache hit, do nothing
},
|key, lookup| {
let job = match JobOwner::try_start::<Q>(tcx, span, &key, lookup) {
let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) {
TryGetJob::NotYetStarted(job) => job,
TryGetJob::Cycle(_) => return,
#[cfg(parallel_compiler)]
TryGetJob::JobCompleted(_) => return,
};
force_query_with_job::<Q, _>(tcx, key, job, dep_node);
force_query_with_job(tcx, key, job, dep_node, query);
},
);
}
#[inline(always)]
pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored
where
Q: QueryDescription<CTX>,
Q::Key: crate::dep_graph::DepNodeParams<CTX>,
CTX: QueryContext,
{
debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
get_query_impl(tcx, Q::query_state(tcx), span, key, &Q::VTABLE)
}
#[inline(always)]
pub fn ensure_query<Q, CTX>(tcx: CTX, key: Q::Key)
where
Q: QueryDescription<CTX>,
Q::Key: crate::dep_graph::DepNodeParams<CTX>,
CTX: QueryContext,
{
ensure_query_impl(tcx, Q::query_state(tcx), key, &Q::VTABLE)
}
#[inline(always)]
pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode<CTX::DepKind>)
where
Q: QueryDescription<CTX>,
Q::Key: crate::dep_graph::DepNodeParams<CTX>,
CTX: QueryContext,
{
force_query_impl(tcx, Q::query_state(tcx), key, span, dep_node, &Q::VTABLE)
}