From 3fc8ed68e99034ad5410cef47e8cd94828ef8946 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 6 Feb 2021 14:52:04 +0100 Subject: [PATCH] Check query cache before calling into the query engine. --- .../rustc_middle/src/ty/query/plumbing.rs | 24 +++++++-- .../rustc_query_system/src/query/plumbing.rs | 52 +++++++++++++------ 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs index 9a011846fd6..0961d4d0091 100644 --- a/compiler/rustc_middle/src/ty/query/plumbing.rs +++ b/compiler/rustc_middle/src/ty/query/plumbing.rs @@ -422,7 +422,15 @@ impl TyCtxtEnsure<$tcx> { $($(#[$attr])* #[inline(always)] pub fn $name(self, key: query_helper_param_ty!($($K)*)) { - get_query::, _>(self.tcx, DUMMY_SP, key.into_query_param(), QueryMode::Ensure); + let key = key.into_query_param(); + let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, |_| {}); + + let lookup = match cached { + Ok(()) => return, + Err(lookup) => lookup, + }; + + get_query::, _>(self.tcx, DUMMY_SP, key, lookup, QueryMode::Ensure); })* } @@ -465,7 +473,7 @@ pub fn at(self, span: Span) -> TyCtxtAt<$tcx> { #[must_use] pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx> { - self.at(DUMMY_SP).$name(key.into_query_param()) + self.at(DUMMY_SP).$name(key) })* /// All self-profiling events generated by the query engine use @@ -503,7 +511,17 @@ impl TyCtxtAt<$tcx> { #[inline(always)] pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx> { - get_query::, _>(self.tcx, self.span, key.into_query_param(), QueryMode::Get).unwrap() + let key = key.into_query_param(); + let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, |value| { + value.clone() + }); + + let lookup = match cached { + Ok(value) => return value, + Err(lookup) => lookup, + }; + + get_query::, _>(self.tcx, self.span, key, lookup, QueryMode::Get).unwrap() })* } diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index c2e89e131b3..2610ce83e4d 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -263,7 +263,18 @@ fn try_start<'b, CTX>( return TryGetJob::Cycle(value); } - let cached = try_get_cached(tcx, cache, key, |value, index| (value.clone(), index)) + let cached = cache + .cache + .lookup(cache, &key, |value, index| { + if unlikely!(tcx.profiler().enabled()) { + tcx.profiler().query_cache_hit(index.into()); + } + #[cfg(debug_assertions)] + { + cache.cache_hits.fetch_add(1, Ordering::Relaxed); + } + (value.clone(), index) + }) .unwrap_or_else(|_| panic!("value must be in cache after waiting")); if let Some(prof_timer) = _query_blocked_prof_timer.take() { @@ -374,7 +385,7 @@ enum TryGetJob<'tcx, D, Q, C> /// It returns the shard index and a lock guard to the shard, /// which will be used if the query is not in the cache and we need /// to compute it. -fn try_get_cached<'a, CTX, C, R, OnHit>( +pub fn try_get_cached<'a, CTX, C, R, OnHit>( tcx: CTX, cache: &'a QueryCacheStore, key: &C::Key, @@ -384,7 +395,7 @@ fn try_get_cached<'a, CTX, C, R, OnHit>( where C: QueryCache, CTX: QueryContext, - OnHit: FnOnce(&C::Stored, DepNodeIndex) -> R, + OnHit: FnOnce(&C::Stored) -> R, { cache.cache.lookup(cache, &key, |value, index| { if unlikely!(tcx.profiler().enabled()) { @@ -394,7 +405,8 @@ fn try_get_cached<'a, CTX, C, R, OnHit>( { cache.cache_hits.fetch_add(1, Ordering::Relaxed); } - on_hit(value, index) + tcx.dep_graph().read_index(index); + on_hit(value) }) } @@ -632,6 +644,7 @@ fn get_query_impl( cache: &QueryCacheStore, span: Span, key: C::Key, + lookup: QueryLookup, query: &QueryVtable, ) -> C::Stored where @@ -639,14 +652,7 @@ fn get_query_impl( C: QueryCache, C::Key: crate::dep_graph::DepNodeParams, { - let cached = try_get_cached(tcx, cache, &key, |value, index| { - tcx.dep_graph().read_index(index); - value.clone() - }); - match cached { - Ok(value) => value, - Err(lookup) => try_execute_query(tcx, state, cache, span, key, lookup, query), - } + try_execute_query(tcx, state, cache, span, key, lookup, query) } /// Ensure that either this query has all green inputs or been executed. @@ -705,9 +711,14 @@ fn force_query_impl( { // We may be concurrently trying both execute and force a query. // Ensure that only one of them runs the query. - - let cached = try_get_cached(tcx, cache, &key, |_, _| { - // Cache hit, do nothing + let cached = cache.cache.lookup(cache, &key, |_, index| { + if unlikely!(tcx.profiler().enabled()) { + tcx.profiler().query_cache_hit(index.into()); + } + #[cfg(debug_assertions)] + { + cache.cache_hits.fetch_add(1, Ordering::Relaxed); + } }); let lookup = match cached { @@ -731,7 +742,13 @@ pub enum QueryMode { Ensure, } -pub fn get_query(tcx: CTX, span: Span, key: Q::Key, mode: QueryMode) -> Option +pub fn get_query( + tcx: CTX, + span: Span, + key: Q::Key, + lookup: QueryLookup, + mode: QueryMode, +) -> Option where Q: QueryDescription, Q::Key: crate::dep_graph::DepNodeParams, @@ -745,7 +762,8 @@ pub fn get_query(tcx: CTX, span: Span, key: Q::Key, mode: QueryMode) -> } debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span); - let value = get_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), span, key, query); + let value = + get_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), span, key, lookup, query); Some(value) }