Factor query arena allocation out from query caches

compiler/rustc_interface/src/passes.rs

@@ -676,9 +676,7 @@ pub fn create_global_ctxt<'tcx>(
         callback(sess, &mut local_providers, &mut extern_providers);
     }
 
-    let queries = queries.get_or_init(|| {
+    let queries = queries.get_or_init(|| TcxQueries::new(query_result_on_disk_cache));
-        TcxQueries::new(local_providers, extern_providers, query_result_on_disk_cache)
-    });
 
     sess.time("setup_global_ctxt", || {
         gcx_cell.get_or_init(move || {
@@ -690,6 +688,8 @@ pub fn create_global_ctxt<'tcx>(
                 untracked,
                 dep_graph,
                 queries.on_disk_cache.as_ref().map(OnDiskCache::as_dyn),
+                local_providers,
+                extern_providers,
                 queries.as_dyn(),
                 rustc_query_impl::query_callbacks(arena),
             )

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -114,7 +114,7 @@ macro_rules! provide_one {
         fn $name<'tcx>(
             $tcx: TyCtxt<'tcx>,
             def_id_arg: ty::query::query_keys::$name<'tcx>,
-        ) -> ty::query::query_values::$name<'tcx> {
+        ) -> ty::query::query_provided::$name<'tcx> {
             let _prof_timer =
                 $tcx.prof.generic_activity(concat!("metadata_decode_entry_", stringify!($name)));
 

compiler/rustc_middle/src/ty/context.rs

@@ -18,6 +18,8 @@
 use crate::thir::Thir;
 use crate::traits;
 use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData};
+use crate::ty::query::ExternProviders;
+use crate::ty::query::Providers;
 use crate::ty::query::{self, TyCtxtAt};
 use crate::ty::{
     self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, DefIdTree, FloatTy, FloatVar,
@@ -479,7 +481,7 @@ pub struct GlobalCtxt<'tcx> {
     pub on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
 
     pub queries: &'tcx dyn query::QueryEngine<'tcx>,
-    pub query_caches: query::QueryCaches<'tcx>,
+    pub query_system: query::QuerySystem<'tcx>,
     pub(crate) query_kinds: &'tcx [DepKindStruct<'tcx>],
 
     // Internal caches for metadata decoding. No need to track deps on this.
@@ -639,6 +641,8 @@ pub fn create_global_ctxt(
         untracked: Untracked,
         dep_graph: DepGraph,
         on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
+        local_providers: Providers,
+        extern_providers: ExternProviders,
         queries: &'tcx dyn query::QueryEngine<'tcx>,
         query_kinds: &'tcx [DepKindStruct<'tcx>],
     ) -> GlobalCtxt<'tcx> {
@@ -664,7 +668,7 @@ pub fn create_global_ctxt(
             untracked,
             on_disk_cache,
             queries,
-            query_caches: query::QueryCaches::default(),
+            query_system: query::QuerySystem::new(local_providers, extern_providers),
             query_kinds,
             ty_rcache: Default::default(),
             pred_rcache: Default::default(),

compiler/rustc_middle/src/ty/query.rs

@@ -1,3 +1,5 @@
+#![allow(unused_parens)]
+
 use crate::dep_graph;
 use crate::infer::canonical::{self, Canonical};
 use crate::lint::LintExpectation;
@@ -34,6 +36,7 @@
 use crate::ty::util::AlwaysRequiresDrop;
 use crate::ty::GeneratorDiagnosticData;
 use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, UnusedGenericParams};
+use rustc_arena::TypedArena;
 use rustc_ast as ast;
 use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_attr as attr;
@@ -41,6 +44,7 @@
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::Lrc;
+use rustc_data_structures::sync::WorkerLocal;
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
@@ -66,6 +70,24 @@
 pub(crate) use rustc_query_system::query::QueryJobId;
 use rustc_query_system::query::*;
 
+pub struct QuerySystem<'tcx> {
+    pub local_providers: Box<Providers>,
+    pub extern_providers: Box<ExternProviders>,
+    pub arenas: QueryArenas<'tcx>,
+    pub caches: QueryCaches<'tcx>,
+}
+
+impl<'tcx> QuerySystem<'tcx> {
+    pub fn new(local_providers: Providers, extern_providers: ExternProviders) -> Self {
+        QuerySystem {
+            local_providers: Box::new(local_providers),
+            extern_providers: Box::new(extern_providers),
+            arenas: Default::default(),
+            caches: Default::default(),
+        }
+    }
+}
+
 #[derive(Copy, Clone)]
 pub struct TyCtxtAt<'tcx> {
     pub tcx: TyCtxt<'tcx>,
@@ -112,10 +134,10 @@ macro_rules! query_helper_param_ty {
 }
 
 macro_rules! query_if_arena {
-    ([] $arena:ty, $no_arena:ty) => {
+    ([] $arena:tt $no_arena:tt) => {
         $no_arena
     };
-    ([(arena_cache) $($rest:tt)*] $arena:ty, $no_arena:ty) => {
+    ([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => {
         $arena
     };
     ([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
@@ -131,7 +153,7 @@ macro_rules! separate_provide_extern_decl {
         for<'tcx> fn(
             TyCtxt<'tcx>,
             query_keys::$name<'tcx>,
-        ) -> query_values::$name<'tcx>
+        ) -> query_provided::$name<'tcx>
     };
     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
         separate_provide_extern_decl!([$($modifiers)*][$($args)*])
@@ -183,30 +205,62 @@ pub mod query_keys {
 
             $(pub type $name<'tcx> = $($K)*;)*
         }
-        #[allow(nonstandard_style, unused_lifetimes, unused_parens)]
+        #[allow(nonstandard_style, unused_lifetimes)]
         pub mod query_values {
             use super::*;
 
-            $(pub type $name<'tcx> = query_if_arena!([$($modifiers)*] <$V as Deref>::Target, $V);)*
+            $(pub type $name<'tcx> = $V;)*
         }
-        #[allow(nonstandard_style, unused_lifetimes, unused_parens)]
+        #[allow(nonstandard_style, unused_lifetimes)]
+        pub mod query_provided {
+            use super::*;
+
+            $(
+                pub type $name<'tcx> = query_if_arena!([$($modifiers)*] (<$V as Deref>::Target) ($V));
+            )*
+        }
+        #[allow(nonstandard_style, unused_lifetimes)]
+        pub mod query_provided_to_value {
+            use super::*;
+
+            $(
+                #[inline]
+                pub fn $name<'tcx>(
+                    _tcx: TyCtxt<'tcx>,
+                    value: query_provided::$name<'tcx>,
+                ) -> query_values::$name<'tcx> {
+                    query_if_arena!([$($modifiers)*]
+                        (&*_tcx.query_system.arenas.$name.alloc(value))
+                        (value)
+                    )
+                }
+            )*
+        }
+        #[allow(nonstandard_style, unused_lifetimes)]
         pub mod query_storage {
             use super::*;
 
             $(
-                pub type $name<'tcx> = query_if_arena!([$($modifiers)*]
+                pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache;
-                    <<$($K)* as Key>::CacheSelector
-                        as CacheSelector<'tcx, <$V as Deref>::Target>>::ArenaCache,
-                    <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache
-                );
             )*
         }
 
-        #[allow(nonstandard_style, unused_lifetimes)]
+        pub struct QueryArenas<'tcx> {
-        pub mod query_stored {
+            $($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
-            use super::*;
+                (WorkerLocal<TypedArena<<$V as Deref>::Target>>)
+                ()
+            ),)*
+        }
 
-            $(pub type $name<'tcx> = $V;)*
+        impl Default for QueryArenas<'_> {
+            fn default() -> Self {
+                Self {
+                    $($name: query_if_arena!([$($modifiers)*]
+                        (WorkerLocal::new(|_| Default::default()))
+                        ()
+                    ),)*
+                }
+            }
         }
 
         #[derive(Default)]
@@ -221,7 +275,7 @@ pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
                 let key = key.into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
-                match try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key) {
+                match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
                     Some(_) => return,
                     None => self.tcx.queries.$name(self.tcx, DUMMY_SP, key, QueryMode::Ensure),
                 };
@@ -246,7 +300,7 @@ pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
                 let key = key.into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
-                match try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key) {
+                match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
                     Some(value) => value,
                     None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(),
                 }
@@ -257,7 +311,7 @@ pub struct Providers {
             $(pub $name: for<'tcx> fn(
                 TyCtxt<'tcx>,
                 query_keys::$name<'tcx>,
-            ) -> query_values::$name<'tcx>,)*
+            ) -> query_provided::$name<'tcx>,)*
         }
 
         pub struct ExternProviders {
@@ -334,12 +388,13 @@ macro_rules! define_feedable {
         $(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
             $(#[$attr])*
             #[inline(always)]
-            pub fn $name(self, value: query_values::$name<'tcx>) -> $V {
+            pub fn $name(self, value: query_provided::$name<'tcx>) -> $V {
                 let key = self.key().into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
                 let tcx = self.tcx;
-                let cache = &tcx.query_caches.$name;
+                let value = query_provided_to_value::$name(tcx, value);
+                let cache = &tcx.query_system.caches.$name;
 
                 match try_get_cached(tcx, cache, &key) {
                     Some(old) => {
@@ -357,7 +412,8 @@ pub fn $name(self, value: query_values::$name<'tcx>) -> $V {
                             &value,
                             hash_result!([$($modifiers)*]),
                         );
-                        cache.complete(key, value, dep_node_index)
+                        cache.complete(key, value, dep_node_index);
+                        value
                     }
                 }
             }

compiler/rustc_query_impl/src/lib.rs

@@ -21,8 +21,10 @@
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::{self, DepKindStruct};
 use rustc_middle::query::Key;
-use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
+use rustc_middle::ty::query::QueryEngine;
-use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
+use rustc_middle::ty::query::{
+    query_keys, query_provided, query_provided_to_value, query_storage, query_values,
+};
 use rustc_middle::ty::TyCtxt;
 use rustc_span::Span;
 

compiler/rustc_query_impl/src/plumbing.rs

@@ -278,13 +278,13 @@ macro_rules! hash_result {
 
 macro_rules! get_provider {
     ([][$tcx:expr, $name:ident, $key:expr]) => {{
-        $tcx.queries.local_providers.$name
+        $tcx.query_system.local_providers.$name
     }};
     ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
         if $key.query_crate_is_local() {
-            $tcx.queries.local_providers.$name
+            $tcx.query_system.local_providers.$name
         } else {
-            $tcx.queries.extern_providers.$name
+            $tcx.query_system.extern_providers.$name
         }
     }};
     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
@@ -293,14 +293,14 @@ macro_rules! get_provider {
 }
 
 macro_rules! should_ever_cache_on_disk {
-    ([]) => {{
+    ([]$yes:tt $no:tt) => {{
-        None
+        $no
     }};
-    ([(cache) $($rest:tt)*]) => {{
+    ([(cache) $($rest:tt)*]$yes:tt $no:tt) => {{
-        Some($crate::plumbing::try_load_from_disk::<Self::Value>)
+        $yes
     }};
-    ([$other:tt $($modifiers:tt)*]) => {
+    ([$other:tt $($modifiers:tt)*]$yes:tt $no:tt) => {
-        should_ever_cache_on_disk!([$($modifiers)*])
+        should_ever_cache_on_disk!([$($modifiers)*]$yes $no)
     };
 }
 
@@ -472,7 +472,6 @@ mod queries {
         $(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> {
             type Key = query_keys::$name<'tcx>;
             type Value = query_values::$name<'tcx>;
-            type Stored = query_stored::$name<'tcx>;
             const NAME: &'static str = stringify!($name);
 
             #[inline]
@@ -493,24 +492,40 @@ fn query_state<'a>(tcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, crate::dep
             fn query_cache<'a>(tcx: QueryCtxt<'tcx>) -> &'a Self::Cache
                 where 'tcx:'a
             {
-                &tcx.query_caches.$name
+                &tcx.query_system.caches.$name
             }
 
-            fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Stored {
+            fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
                 tcx.$name(key)
             }
 
             #[inline]
             // key is only sometimes used
             #[allow(unused_variables)]
-            fn compute(qcx: QueryCtxt<'tcx>, key: &Self::Key) -> fn(TyCtxt<'tcx>, Self::Key) -> Self::Value {
+            fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
-                get_provider!([$($modifiers)*][qcx, $name, key])
+                query_provided_to_value::$name(
+                    tcx,
+                    get_provider!([$($modifiers)*][tcx, $name, key])(tcx, key)
+                )
             }
 
             #[inline]
-            fn try_load_from_disk(qcx: QueryCtxt<'tcx>, key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self> {
+            fn try_load_from_disk(_qcx: QueryCtxt<'tcx>, _key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self> {
-                let cache_on_disk = Self::cache_on_disk(qcx.tcx, key);
+                should_ever_cache_on_disk!([$($modifiers)*] {
-                if cache_on_disk { should_ever_cache_on_disk!([$($modifiers)*]) } else { None }
+                    if Self::cache_on_disk(_qcx.tcx, _key) {
+                        Some(|qcx: QueryCtxt<'tcx>, dep_node| {
+                            let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>(
+                                qcx,
+                                dep_node
+                            );
+                            value.map(|value| query_provided_to_value::$name(qcx.tcx, value))
+                        })
+                    } else {
+                        None
+                    }
+                } {
+                    None
+                })
             }
 
             const ANON: bool = is_anon!([$($modifiers)*]);
@@ -633,7 +648,7 @@ pub(super) const fn $name<'tcx>() -> QueryStruct<'tcx> { QueryStruct {
                     $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
                         tcx,
                         stringify!($name),
-                        &tcx.query_caches.$name,
+                        &tcx.query_system.caches.$name,
                         string_cache,
                     )
                 },
@@ -649,18 +664,12 @@ pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct<'
     }
 }
 
-use crate::{ExternProviders, OnDiskCache, Providers};
+use crate::OnDiskCache;
 
 impl<'tcx> Queries<'tcx> {
-    pub fn new(
+    pub fn new(on_disk_cache: Option<OnDiskCache<'tcx>>) -> Self {
-        local_providers: Providers,
-        extern_providers: ExternProviders,
-        on_disk_cache: Option<OnDiskCache<'tcx>>,
-    ) -> Self {
         use crate::query_structs;
         Queries {
-            local_providers: Box::new(local_providers),
-            extern_providers: Box::new(extern_providers),
             query_structs: make_dep_kind_array!(query_structs).to_vec(),
             on_disk_cache,
             jobs: AtomicU64::new(1),
@@ -674,8 +683,6 @@ macro_rules! define_queries_struct {
      input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
         #[derive(Default)]
         pub struct Queries<'tcx> {
-            local_providers: Box<Providers>,
-            extern_providers: Box<ExternProviders>,
             query_structs: Vec<$crate::plumbing::QueryStruct<'tcx>>,
             pub on_disk_cache: Option<OnDiskCache<'tcx>>,
             jobs: AtomicU64,
@@ -725,7 +732,7 @@ fn $name(
                 span: Span,
                 key: <queries::$name<'tcx> as QueryConfig<QueryCtxt<'tcx>>>::Key,
                 mode: QueryMode,
-            ) -> Option<query_stored::$name<'tcx>> {
+            ) -> Option<query_values::$name<'tcx>> {
                 let qcx = QueryCtxt { tcx, queries: self };
                 get_query::<queries::$name<'tcx>, _, rustc_middle::dep_graph::DepKind>(qcx, span, key, mode)
             })*

compiler/rustc_query_system/src/query/caches.rs

@@ -1,12 +1,10 @@
 use crate::dep_graph::DepNodeIndex;
 
-use rustc_arena::TypedArena;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sharded;
 #[cfg(parallel_compiler)]
 use rustc_data_structures::sharded::Sharded;
 use rustc_data_structures::sync::Lock;
-use rustc_data_structures::sync::WorkerLocal;
 use rustc_index::vec::{Idx, IndexVec};
 use std::fmt::Debug;
 use std::hash::Hash;
@@ -16,12 +14,10 @@ pub trait CacheSelector<'tcx, V> {
     type Cache
     where
         V: Copy;
-    type ArenaCache;
 }
 
 pub trait QueryStorage {
-    type Value: Debug;
+    type Value: Copy;
-    type Stored: Copy;
 }
 
 pub trait QueryCache: QueryStorage + Sized {
@@ -31,9 +27,9 @@ pub trait QueryCache: QueryStorage + Sized {
     /// It returns the shard index and a lock guard to the shard,
     /// which will be used if the query is not in the cache and we need
     /// to compute it.
-    fn lookup(&self, key: &Self::Key) -> Option<(Self::Stored, DepNodeIndex)>;
+    fn lookup(&self, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>;
 
-    fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex) -> Self::Stored;
+    fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex);
 
     fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex));
 }
@@ -44,7 +40,6 @@ impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector<'tcx, V> for DefaultCacheSelecto
     type Cache = DefaultCache<K, V>
     where
         V: Copy;
-    type ArenaCache = ArenaCache<'tcx, K, V>;
 }
 
 pub struct DefaultCache<K, V> {
@@ -62,7 +57,6 @@ fn default() -> Self {
 
 impl<K: Eq + Hash, V: Copy + Debug> QueryStorage for DefaultCache<K, V> {
     type Value = V;
-    type Stored = V;
 }
 
 impl<K, V> QueryCache for DefaultCache<K, V>
@@ -85,7 +79,7 @@ fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
     }
 
     #[inline]
-    fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
+    fn complete(&self, key: K, value: V, index: DepNodeIndex) {
         #[cfg(parallel_compiler)]
         let mut lock = self.cache.get_shard_by_value(&key).lock();
         #[cfg(not(parallel_compiler))]
@@ -93,7 +87,6 @@ fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
         // We may be overwriting another value. This is all right, since the dep-graph
         // will check that the fingerprint matches.
         lock.insert(key, (value, index));
-        value
     }
 
     fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
@@ -122,7 +115,6 @@ impl<'tcx, V: 'tcx> CacheSelector<'tcx, V> for SingleCacheSelector {
     type Cache = SingleCache<V>
     where
         V: Copy;
-    type ArenaCache = ArenaCache<'tcx, (), V>;
 }
 
 pub struct SingleCache<V> {
@@ -137,7 +129,6 @@ fn default() -> Self {
 
 impl<V: Copy + Debug> QueryStorage for SingleCache<V> {
     type Value = V;
-    type Stored = V;
 }
 
 impl<V> QueryCache for SingleCache<V>
@@ -152,9 +143,8 @@ fn lookup(&self, _key: &()) -> Option<(V, DepNodeIndex)> {
     }
 
     #[inline]
-    fn complete(&self, _key: (), value: V, index: DepNodeIndex) -> Self::Stored {
+    fn complete(&self, _key: (), value: V, index: DepNodeIndex) {
         *self.cache.lock() = Some((value, index));
-        value
     }
 
     fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
@@ -162,85 +152,12 @@ fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
     }
 }
 
-pub struct ArenaCache<'tcx, K, V> {
-    arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
-    #[cfg(parallel_compiler)]
-    cache: Sharded<FxHashMap<K, &'tcx (V, DepNodeIndex)>>,
-    #[cfg(not(parallel_compiler))]
-    cache: Lock<FxHashMap<K, &'tcx (V, DepNodeIndex)>>,
-}
-
-impl<'tcx, K, V> Default for ArenaCache<'tcx, K, V> {
-    fn default() -> Self {
-        ArenaCache { arena: WorkerLocal::new(|_| TypedArena::default()), cache: Default::default() }
-    }
-}
-
-impl<'tcx, K: Eq + Hash, V: Debug + 'tcx> QueryStorage for ArenaCache<'tcx, K, V> {
-    type Value = V;
-    type Stored = &'tcx V;
-}
-
-impl<'tcx, K, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V>
-where
-    K: Eq + Hash + Clone + Debug,
-    V: Debug,
-{
-    type Key = K;
-
-    #[inline(always)]
-    fn lookup(&self, key: &K) -> Option<(&'tcx V, DepNodeIndex)> {
-        let key_hash = sharded::make_hash(key);
-        #[cfg(parallel_compiler)]
-        let lock = self.cache.get_shard_by_hash(key_hash).lock();
-        #[cfg(not(parallel_compiler))]
-        let lock = self.cache.lock();
-        let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key);
-
-        if let Some((_, value)) = result { Some((&value.0, value.1)) } else { None }
-    }
-
-    #[inline]
-    fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
-        let value = self.arena.alloc((value, index));
-        let value = unsafe { &*(value as *const _) };
-        #[cfg(parallel_compiler)]
-        let mut lock = self.cache.get_shard_by_value(&key).lock();
-        #[cfg(not(parallel_compiler))]
-        let mut lock = self.cache.lock();
-        // We may be overwriting another value. This is all right, since the dep-graph
-        // will check that the fingerprint matches.
-        lock.insert(key, value);
-        &value.0
-    }
-
-    fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
-        #[cfg(parallel_compiler)]
-        {
-            let shards = self.cache.lock_shards();
-            for shard in shards.iter() {
-                for (k, v) in shard.iter() {
-                    f(k, &v.0, v.1);
-                }
-            }
-        }
-        #[cfg(not(parallel_compiler))]
-        {
-            let map = self.cache.lock();
-            for (k, v) in map.iter() {
-                f(k, &v.0, v.1);
-            }
-        }
-    }
-}
-
 pub struct VecCacheSelector<K>(PhantomData<K>);
 
 impl<'tcx, K: Idx, V: 'tcx> CacheSelector<'tcx, V> for VecCacheSelector<K> {
     type Cache = VecCache<K, V>
     where
         V: Copy;
-    type ArenaCache = VecArenaCache<'tcx, K, V>;
 }
 
 pub struct VecCache<K: Idx, V> {
@@ -258,7 +175,6 @@ fn default() -> Self {
 
 impl<K: Eq + Idx, V: Copy + Debug> QueryStorage for VecCache<K, V> {
     type Value = V;
-    type Stored = V;
 }
 
 impl<K, V> QueryCache for VecCache<K, V>
@@ -278,87 +194,12 @@ fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
     }
 
     #[inline]
-    fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
+    fn complete(&self, key: K, value: V, index: DepNodeIndex) {
         #[cfg(parallel_compiler)]
         let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
         #[cfg(not(parallel_compiler))]
         let mut lock = self.cache.lock();
         lock.insert(key, (value, index));
-        value
-    }
-
-    fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
-        #[cfg(parallel_compiler)]
-        {
-            let shards = self.cache.lock_shards();
-            for shard in shards.iter() {
-                for (k, v) in shard.iter_enumerated() {
-                    if let Some(v) = v {
-                        f(&k, &v.0, v.1);
-                    }
-                }
-            }
-        }
-        #[cfg(not(parallel_compiler))]
-        {
-            let map = self.cache.lock();
-            for (k, v) in map.iter_enumerated() {
-                if let Some(v) = v {
-                    f(&k, &v.0, v.1);
-                }
-            }
-        }
-    }
-}
-
-pub struct VecArenaCache<'tcx, K: Idx, V> {
-    arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
-    #[cfg(parallel_compiler)]
-    cache: Sharded<IndexVec<K, Option<&'tcx (V, DepNodeIndex)>>>,
-    #[cfg(not(parallel_compiler))]
-    cache: Lock<IndexVec<K, Option<&'tcx (V, DepNodeIndex)>>>,
-}
-
-impl<'tcx, K: Idx, V> Default for VecArenaCache<'tcx, K, V> {
-    fn default() -> Self {
-        VecArenaCache {
-            arena: WorkerLocal::new(|_| TypedArena::default()),
-            cache: Default::default(),
-        }
-    }
-}
-
-impl<'tcx, K: Eq + Idx, V: Debug + 'tcx> QueryStorage for VecArenaCache<'tcx, K, V> {
-    type Value = V;
-    type Stored = &'tcx V;
-}
-
-impl<'tcx, K, V: 'tcx> QueryCache for VecArenaCache<'tcx, K, V>
-where
-    K: Eq + Idx + Clone + Debug,
-    V: Debug,
-{
-    type Key = K;
-
-    #[inline(always)]
-    fn lookup(&self, key: &K) -> Option<(&'tcx V, DepNodeIndex)> {
-        #[cfg(parallel_compiler)]
-        let lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
-        #[cfg(not(parallel_compiler))]
-        let lock = self.cache.lock();
-        if let Some(Some(value)) = lock.get(*key) { Some((&value.0, value.1)) } else { None }
-    }
-
-    #[inline]
-    fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
-        let value = self.arena.alloc((value, index));
-        let value = unsafe { &*(value as *const _) };
-        #[cfg(parallel_compiler)]
-        let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
-        #[cfg(not(parallel_compiler))]
-        let mut lock = self.cache.lock();
-        lock.insert(key, value);
-        &value.0
     }
 
     fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {

compiler/rustc_query_system/src/query/config.rs

@@ -20,10 +20,9 @@ pub trait QueryConfig<Qcx: QueryContext> {
     const NAME: &'static str;
 
     type Key: DepNodeParams<Qcx::DepContext> + Eq + Hash + Clone + Debug;
-    type Value: Debug;
+    type Value: Debug + Copy;
-    type Stored: Debug + Copy + std::borrow::Borrow<Self::Value>;
 
-    type Cache: QueryCache<Key = Self::Key, Stored = Self::Stored, Value = Self::Value>;
+    type Cache: QueryCache<Key = Self::Key, Value = Self::Value>;
 
     // Don't use this method to access query results, instead use the methods on TyCtxt
     fn query_state<'a>(tcx: Qcx) -> &'a QueryState<Self::Key, Qcx::DepKind>
@@ -38,9 +37,9 @@ fn query_cache<'a>(tcx: Qcx) -> &'a Self::Cache
     fn cache_on_disk(tcx: Qcx::DepContext, key: &Self::Key) -> bool;
 
     // Don't use this method to compute query results, instead use the methods on TyCtxt
-    fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Stored;
+    fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Value;
 
-    fn compute(tcx: Qcx, key: &Self::Key) -> fn(Qcx::DepContext, Self::Key) -> Self::Value;
+    fn compute(tcx: Qcx::DepContext, key: Self::Key) -> Self::Value;
 
     fn try_load_from_disk(qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk<Qcx, Self>;
 

compiler/rustc_query_system/src/query/plumbing.rs

@@ -246,7 +246,7 @@ fn try_start<'b, Qcx>(
 
     /// Completes the query by updating the query cache with the `result`,
     /// signals the waiter and forgets the JobOwner, so it won't poison the query
-    fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex) -> C::Stored
+    fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex)
     where
         C: QueryCache<Key = K>,
     {
@@ -257,23 +257,22 @@ fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex)
         // Forget ourself so our destructor won't poison the query
         mem::forget(self);
 
-        let (job, result) = {
+        // Mark as complete before we remove the job from the active state
-            let job = {
+        // so no other thread can re-execute this query.
-                #[cfg(parallel_compiler)]
+        cache.complete(key.clone(), result, dep_node_index);
-                let mut lock = state.active.get_shard_by_value(&key).lock();
+
-                #[cfg(not(parallel_compiler))]
+        let job = {
-                let mut lock = state.active.lock();
+            #[cfg(parallel_compiler)]
-                match lock.remove(&key).unwrap() {
+            let mut lock = state.active.get_shard_by_value(&key).lock();
-                    QueryResult::Started(job) => job,
+            #[cfg(not(parallel_compiler))]
-                    QueryResult::Poisoned => panic!(),
+            let mut lock = state.active.lock();
-                }
+            match lock.remove(&key).unwrap() {
-            };
+                QueryResult::Started(job) => job,
-            let result = cache.complete(key, result, dep_node_index);
+                QueryResult::Poisoned => panic!(),
-            (job, result)
+            }
         };
 
         job.signal_complete();
-        result
     }
 }
 
@@ -336,7 +335,7 @@ enum TryGetJob<'tcx, K, D>
 /// which will be used if the query is not in the cache and we need
 /// to compute it.
 #[inline]
-pub fn try_get_cached<Tcx, C>(tcx: Tcx, cache: &C, key: &C::Key) -> Option<C::Stored>
+pub fn try_get_cached<Tcx, C>(tcx: Tcx, cache: &C, key: &C::Key) -> Option<C::Value>
 where
     C: QueryCache,
     Tcx: DepContext,
@@ -358,7 +357,7 @@ fn try_execute_query<Q, Qcx>(
     span: Span,
     key: Q::Key,
     dep_node: Option<DepNode<Qcx::DepKind>>,
-) -> (Q::Stored, Option<DepNodeIndex>)
+) -> (Q::Value, Option<DepNodeIndex>)
 where
     Q: QueryConfig<Qcx>,
     Qcx: QueryContext,
@@ -390,7 +389,7 @@ fn try_execute_query<Q, Qcx>(
                     );
                 }
             }
-            let result = job.complete(cache, result, dep_node_index);
+            job.complete(cache, result, dep_node_index);
             (result, Some(dep_node_index))
         }
         TryGetJob::Cycle(error) => {
@@ -426,9 +425,8 @@ fn execute_job<Q, Qcx>(
     // Fast path for when incr. comp. is off.
     if !dep_graph.is_fully_enabled() {
         let prof_timer = qcx.dep_context().profiler().query_provider();
-        let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || {
+        let result =
-            Q::compute(qcx, &key)(*qcx.dep_context(), key)
+            qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || Q::compute(*qcx.dep_context(), key));
-        });
         let dep_node_index = dep_graph.next_virtual_depnode_index();
         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
         return (result, dep_node_index);
@@ -455,7 +453,7 @@ fn execute_job<Q, Qcx>(
         qcx.start_query(job_id, Q::DEPTH_LIMIT, Some(&diagnostics), || {
             if Q::ANON {
                 return dep_graph.with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || {
-                    Q::compute(qcx, &key)(*qcx.dep_context(), key)
+                    Q::compute(*qcx.dep_context(), key)
                 });
             }
 
@@ -463,8 +461,7 @@ fn execute_job<Q, Qcx>(
             let dep_node =
                 dep_node_opt.unwrap_or_else(|| Q::construct_dep_node(*qcx.dep_context(), &key));
 
-            let task = Q::compute(qcx, &key);
+            dep_graph.with_task(dep_node, *qcx.dep_context(), key, Q::compute, Q::HASH_RESULT)
-            dep_graph.with_task(dep_node, *qcx.dep_context(), key, task, Q::HASH_RESULT)
         });
 
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
@@ -555,7 +552,7 @@ fn try_load_from_disk_and_cache_in_memory<Q, Qcx>(
     let prof_timer = qcx.dep_context().profiler().query_provider();
 
     // The dep-graph for this computation is already in-place.
-    let result = dep_graph.with_ignore(|| Q::compute(qcx, key)(*qcx.dep_context(), key.clone()));
+    let result = dep_graph.with_ignore(|| Q::compute(*qcx.dep_context(), key.clone()));
 
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -727,7 +724,7 @@ pub enum QueryMode {
     Ensure,
 }
 
-pub fn get_query<Q, Qcx, D>(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Stored>
+pub fn get_query<Q, Qcx, D>(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Value>
 where
     D: DepKind,
     Q: QueryConfig<Qcx>,