mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Factor query arena allocation out from query caches

Browse Source
This commit is contained in:
John Kåre Alsaker 2023-02-08 19:53:48 +01:00
parent 4b34c7b766
commit a51a20531d
9 changed files with 160 additions and 254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
compiler/rustc_interface/src/passes.rs
View File

@ -676,9 +676,7 @@ pub fn create_global_ctxt<'tcx>(
callback(sess, &mut local_providers, &mut extern_providers);
}
let queries = queries.get_or_init(|| {
TcxQueries::new(local_providers, extern_providers, query_result_on_disk_cache)
});
let queries = queries.get_or_init(|| TcxQueries::new(query_result_on_disk_cache));
sess.time("setup_global_ctxt", || {
gcx_cell.get_or_init(move || {
@ -690,6 +688,8 @@ pub fn create_global_ctxt<'tcx>(
untracked,
dep_graph,
queries.on_disk_cache.as_ref().map(OnDiskCache::as_dyn),
local_providers,
extern_providers,
queries.as_dyn(),
rustc_query_impl::query_callbacks(arena),
)

2
compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
View File

@ -114,7 +114,7 @@ macro_rules! provide_one {
fn $name<'tcx>(
$tcx: TyCtxt<'tcx>,
def_id_arg: ty::query::query_keys::$name<'tcx>,
) -> ty::query::query_values::$name<'tcx> {
) -> ty::query::query_provided::$name<'tcx> {
let _prof_timer =
$tcx.prof.generic_activity(concat!("metadata_decode_entry_", stringify!($name)));

8
compiler/rustc_middle/src/ty/context.rs
View File

@ -18,6 +18,8 @@
use crate::thir::Thir;
use crate::traits;
use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData};
use crate::ty::query::ExternProviders;
use crate::ty::query::Providers;
use crate::ty::query::{self, TyCtxtAt};
use crate::ty::{
self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, DefIdTree, FloatTy, FloatVar,
@ -479,7 +481,7 @@ pub struct GlobalCtxt<'tcx> {
pub on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
pub queries: &'tcx dyn query::QueryEngine<'tcx>,
pub query_caches: query::QueryCaches<'tcx>,
pub query_system: query::QuerySystem<'tcx>,
pub(crate) query_kinds: &'tcx [DepKindStruct<'tcx>],
// Internal caches for metadata decoding. No need to track deps on this.
@ -639,6 +641,8 @@ pub fn create_global_ctxt(
untracked: Untracked,
dep_graph: DepGraph,
on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
local_providers: Providers,
extern_providers: ExternProviders,
queries: &'tcx dyn query::QueryEngine<'tcx>,
query_kinds: &'tcx [DepKindStruct<'tcx>],
) -> GlobalCtxt<'tcx> {
@ -664,7 +668,7 @@ pub fn create_global_ctxt(
untracked,
on_disk_cache,
queries,
query_caches: query::QueryCaches::default(),
query_system: query::QuerySystem::new(local_providers, extern_providers),
query_kinds,
ty_rcache: Default::default(),
pred_rcache: Default::default(),

98
compiler/rustc_middle/src/ty/query.rs
View File

@ -1,3 +1,5 @@
#![allow(unused_parens)]
use crate::dep_graph;
use crate::infer::canonical::{self, Canonical};
use crate::lint::LintExpectation;
@ -34,6 +36,7 @@
use crate::ty::util::AlwaysRequiresDrop;
use crate::ty::GeneratorDiagnosticData;
use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, UnusedGenericParams};
use rustc_arena::TypedArena;
use rustc_ast as ast;
use rustc_ast::expand::allocator::AllocatorKind;
use rustc_attr as attr;
@ -41,6 +44,7 @@
use rustc_data_structures::steal::Steal;
use rustc_data_structures::svh::Svh;
use rustc_data_structures::sync::Lrc;
use rustc_data_structures::sync::WorkerLocal;
use rustc_data_structures::unord::UnordSet;
use rustc_errors::ErrorGuaranteed;
use rustc_hir as hir;
@ -66,6 +70,24 @@
pub(crate) use rustc_query_system::query::QueryJobId;
use rustc_query_system::query::*;
pub struct QuerySystem<'tcx> {
pub local_providers: Box<Providers>,
pub extern_providers: Box<ExternProviders>,
pub arenas: QueryArenas<'tcx>,
pub caches: QueryCaches<'tcx>,
}
impl<'tcx> QuerySystem<'tcx> {
pub fn new(local_providers: Providers, extern_providers: ExternProviders) -> Self {
QuerySystem {
local_providers: Box::new(local_providers),
extern_providers: Box::new(extern_providers),
arenas: Default::default(),
caches: Default::default(),
}
}
}
#[derive(Copy, Clone)]
pub struct TyCtxtAt<'tcx> {
pub tcx: TyCtxt<'tcx>,
@ -112,10 +134,10 @@ macro_rules! query_helper_param_ty {
}
macro_rules! query_if_arena {
([] $arena:ty, $no_arena:ty) => {
([] $arena:tt $no_arena:tt) => {
$no_arena
};
([(arena_cache) $($rest:tt)*] $arena:ty, $no_arena:ty) => {
([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => {
$arena
};
([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
@ -131,7 +153,7 @@ macro_rules! separate_provide_extern_decl {
for<'tcx> fn(
TyCtxt<'tcx>,
query_keys::$name<'tcx>,
) -> query_values::$name<'tcx>
) -> query_provided::$name<'tcx>
};
([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
separate_provide_extern_decl!([$($modifiers)*][$($args)*])
@ -183,30 +205,62 @@ pub mod query_keys {
$(pub type $name<'tcx> = $($K)*;)*
}
#[allow(nonstandard_style, unused_lifetimes, unused_parens)]
#[allow(nonstandard_style, unused_lifetimes)]
pub mod query_values {
use super::*;
$(pub type $name<'tcx> = query_if_arena!([$($modifiers)*] <$V as Deref>::Target, $V);)*
$(pub type $name<'tcx> = $V;)*
}
#[allow(nonstandard_style, unused_lifetimes, unused_parens)]
#[allow(nonstandard_style, unused_lifetimes)]
pub mod query_provided {
use super::*;
$(
pub type $name<'tcx> = query_if_arena!([$($modifiers)*] (<$V as Deref>::Target) ($V));
)*
}
#[allow(nonstandard_style, unused_lifetimes)]
pub mod query_provided_to_value {
use super::*;
$(
#[inline]
pub fn $name<'tcx>(
_tcx: TyCtxt<'tcx>,
value: query_provided::$name<'tcx>,
) -> query_values::$name<'tcx> {
query_if_arena!([$($modifiers)*]
(&*_tcx.query_system.arenas.$name.alloc(value))
(value)
)
}
)*
}
#[allow(nonstandard_style, unused_lifetimes)]
pub mod query_storage {
use super::*;
$(
pub type $name<'tcx> = query_if_arena!([$($modifiers)*]
<<$($K)* as Key>::CacheSelector
as CacheSelector<'tcx, <$V as Deref>::Target>>::ArenaCache,
<<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache
);
pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache;
)*
}
#[allow(nonstandard_style, unused_lifetimes)]
pub mod query_stored {
use super::*;
pub struct QueryArenas<'tcx> {
$($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
(WorkerLocal<TypedArena<<$V as Deref>::Target>>)
()
),)*
}
$(pub type $name<'tcx> = $V;)*
impl Default for QueryArenas<'_> {
fn default() -> Self {
Self {
$($name: query_if_arena!([$($modifiers)*]
(WorkerLocal::new(|_| Default::default()))
()
),)*
}
}
}
#[derive(Default)]
@ -221,7 +275,7 @@ pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
let key = key.into_query_param();
opt_remap_env_constness!([$($modifiers)*][key]);
match try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key) {
match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
Some(_) => return,
None => self.tcx.queries.$name(self.tcx, DUMMY_SP, key, QueryMode::Ensure),
};
@ -246,7 +300,7 @@ pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
let key = key.into_query_param();
opt_remap_env_constness!([$($modifiers)*][key]);
match try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key) {
match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
Some(value) => value,
None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(),
}
@ -257,7 +311,7 @@ pub struct Providers {
$(pub $name: for<'tcx> fn(
TyCtxt<'tcx>,
query_keys::$name<'tcx>,
) -> query_values::$name<'tcx>,)*
) -> query_provided::$name<'tcx>,)*
}
pub struct ExternProviders {
@ -334,12 +388,13 @@ macro_rules! define_feedable {
$(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
$(#[$attr])*
#[inline(always)]
pub fn $name(self, value: query_values::$name<'tcx>) -> $V {
pub fn $name(self, value: query_provided::$name<'tcx>) -> $V {
let key = self.key().into_query_param();
opt_remap_env_constness!([$($modifiers)*][key]);
let tcx = self.tcx;
let cache = &tcx.query_caches.$name;
let value = query_provided_to_value::$name(tcx, value);
let cache = &tcx.query_system.caches.$name;
match try_get_cached(tcx, cache, &key) {
Some(old) => {
@ -357,7 +412,8 @@ pub fn $name(self, value: query_values::$name<'tcx>) -> $V {
&value,
hash_result!([$($modifiers)*]),
);
cache.complete(key, value, dep_node_index)
cache.complete(key, value, dep_node_index);
value
}
}
}

6
compiler/rustc_query_impl/src/lib.rs
View File

@ -21,8 +21,10 @@
use rustc_middle::arena::Arena;
use rustc_middle::dep_graph::{self, DepKindStruct};
use rustc_middle::query::Key;
use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
use rustc_middle::ty::query::QueryEngine;
use rustc_middle::ty::query::{
query_keys, query_provided, query_provided_to_value, query_storage, query_values,
};
use rustc_middle::ty::TyCtxt;
use rustc_span::Span;

65
compiler/rustc_query_impl/src/plumbing.rs
View File

@ -278,13 +278,13 @@ macro_rules! hash_result {
macro_rules! get_provider {
([][$tcx:expr, $name:ident, $key:expr]) => {{
$tcx.queries.local_providers.$name
$tcx.query_system.local_providers.$name
}};
([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
if $key.query_crate_is_local() {
$tcx.queries.local_providers.$name
$tcx.query_system.local_providers.$name
} else {
$tcx.queries.extern_providers.$name
$tcx.query_system.extern_providers.$name
}
}};
([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
@ -293,14 +293,14 @@ macro_rules! get_provider {
}
macro_rules! should_ever_cache_on_disk {
([]) => {{
None
([]$yes:tt $no:tt) => {{
$no
}};
([(cache) $($rest:tt)*]) => {{
Some($crate::plumbing::try_load_from_disk::<Self::Value>)
([(cache) $($rest:tt)*]$yes:tt $no:tt) => {{
$yes
}};
([$other:tt $($modifiers:tt)*]) => {
should_ever_cache_on_disk!([$($modifiers)*])
([$other:tt $($modifiers:tt)*]$yes:tt $no:tt) => {
should_ever_cache_on_disk!([$($modifiers)*]$yes $no)
};
}
@ -472,7 +472,6 @@ mod queries {
$(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> {
type Key = query_keys::$name<'tcx>;
type Value = query_values::$name<'tcx>;
type Stored = query_stored::$name<'tcx>;
const NAME: &'static str = stringify!($name);
#[inline]
@ -493,24 +492,40 @@ fn query_state<'a>(tcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, crate::dep
fn query_cache<'a>(tcx: QueryCtxt<'tcx>) -> &'a Self::Cache
where 'tcx:'a
{
&tcx.query_caches.$name
&tcx.query_system.caches.$name
}
fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Stored {
fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
tcx.$name(key)
}
#[inline]
// key is only sometimes used
#[allow(unused_variables)]
fn compute(qcx: QueryCtxt<'tcx>, key: &Self::Key) -> fn(TyCtxt<'tcx>, Self::Key) -> Self::Value {
get_provider!([$($modifiers)*][qcx, $name, key])
fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
query_provided_to_value::$name(
tcx,
get_provider!([$($modifiers)*][tcx, $name, key])(tcx, key)
)
}
#[inline]
fn try_load_from_disk(qcx: QueryCtxt<'tcx>, key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self> {
let cache_on_disk = Self::cache_on_disk(qcx.tcx, key);
if cache_on_disk { should_ever_cache_on_disk!([$($modifiers)*]) } else { None }
fn try_load_from_disk(_qcx: QueryCtxt<'tcx>, _key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self> {
should_ever_cache_on_disk!([$($modifiers)*] {
if Self::cache_on_disk(_qcx.tcx, _key) {
Some(|qcx: QueryCtxt<'tcx>, dep_node| {
let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>(
qcx,
dep_node
);
value.map(|value| query_provided_to_value::$name(qcx.tcx, value))
})
} else {
None
}
} {
None
})
}
const ANON: bool = is_anon!([$($modifiers)*]);
@ -633,7 +648,7 @@ pub(super) const fn $name<'tcx>() -> QueryStruct<'tcx> { QueryStruct {
$crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
tcx,
stringify!($name),
&tcx.query_caches.$name,
&tcx.query_system.caches.$name,
string_cache,
)
},
@ -649,18 +664,12 @@ pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct<'
}
}
use crate::{ExternProviders, OnDiskCache, Providers};
use crate::OnDiskCache;
impl<'tcx> Queries<'tcx> {
pub fn new(
local_providers: Providers,
extern_providers: ExternProviders,
on_disk_cache: Option<OnDiskCache<'tcx>>,
) -> Self {
pub fn new(on_disk_cache: Option<OnDiskCache<'tcx>>) -> Self {
use crate::query_structs;
Queries {
local_providers: Box::new(local_providers),
extern_providers: Box::new(extern_providers),
query_structs: make_dep_kind_array!(query_structs).to_vec(),
on_disk_cache,
jobs: AtomicU64::new(1),
@ -674,8 +683,6 @@ macro_rules! define_queries_struct {
input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
#[derive(Default)]
pub struct Queries<'tcx> {
local_providers: Box<Providers>,
extern_providers: Box<ExternProviders>,
query_structs: Vec<$crate::plumbing::QueryStruct<'tcx>>,
pub on_disk_cache: Option<OnDiskCache<'tcx>>,
jobs: AtomicU64,
@ -725,7 +732,7 @@ fn $name(
span: Span,
key: <queries::$name<'tcx> as QueryConfig<QueryCtxt<'tcx>>>::Key,
mode: QueryMode,
) -> Option<query_stored::$name<'tcx>> {
) -> Option<query_values::$name<'tcx>> {
let qcx = QueryCtxt { tcx, queries: self };
get_query::<queries::$name<'tcx>, _, rustc_middle::dep_graph::DepKind>(qcx, span, key, mode)
})*

171
compiler/rustc_query_system/src/query/caches.rs
View File

@ -1,12 +1,10 @@
use crate::dep_graph::DepNodeIndex;
use rustc_arena::TypedArena;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sharded;
#[cfg(parallel_compiler)]
use rustc_data_structures::sharded::Sharded;
use rustc_data_structures::sync::Lock;
use rustc_data_structures::sync::WorkerLocal;
use rustc_index::vec::{Idx, IndexVec};
use std::fmt::Debug;
use std::hash::Hash;
@ -16,12 +14,10 @@ pub trait CacheSelector<'tcx, V> {
type Cache
where
V: Copy;
type ArenaCache;
}
pub trait QueryStorage {
type Value: Debug;
type Stored: Copy;
type Value: Copy;
}
pub trait QueryCache: QueryStorage + Sized {
@ -31,9 +27,9 @@ pub trait QueryCache: QueryStorage + Sized {
/// It returns the shard index and a lock guard to the shard,
/// which will be used if the query is not in the cache and we need
/// to compute it.
fn lookup(&self, key: &Self::Key) -> Option<(Self::Stored, DepNodeIndex)>;
fn lookup(&self, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>;
fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex) -> Self::Stored;
fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex);
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex));
}
@ -44,7 +40,6 @@ impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector<'tcx, V> for DefaultCacheSelecto
type Cache = DefaultCache<K, V>
where
V: Copy;
type ArenaCache = ArenaCache<'tcx, K, V>;
}
pub struct DefaultCache<K, V> {
@ -62,7 +57,6 @@ fn default() -> Self {
impl<K: Eq + Hash, V: Copy + Debug> QueryStorage for DefaultCache<K, V> {
type Value = V;
type Stored = V;
}
impl<K, V> QueryCache for DefaultCache<K, V>
@ -85,7 +79,7 @@ fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
}
#[inline]
fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
fn complete(&self, key: K, value: V, index: DepNodeIndex) {
#[cfg(parallel_compiler)]
let mut lock = self.cache.get_shard_by_value(&key).lock();
#[cfg(not(parallel_compiler))]
@ -93,7 +87,6 @@ fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
// We may be overwriting another value. This is all right, since the dep-graph
// will check that the fingerprint matches.
lock.insert(key, (value, index));
value
}
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
@ -122,7 +115,6 @@ impl<'tcx, V: 'tcx> CacheSelector<'tcx, V> for SingleCacheSelector {
type Cache = SingleCache<V>
where
V: Copy;
type ArenaCache = ArenaCache<'tcx, (), V>;
}
pub struct SingleCache<V> {
@ -137,7 +129,6 @@ fn default() -> Self {
impl<V: Copy + Debug> QueryStorage for SingleCache<V> {
type Value = V;
type Stored = V;
}
impl<V> QueryCache for SingleCache<V>
@ -152,9 +143,8 @@ fn lookup(&self, _key: &()) -> Option<(V, DepNodeIndex)> {
}
#[inline]
fn complete(&self, _key: (), value: V, index: DepNodeIndex) -> Self::Stored {
fn complete(&self, _key: (), value: V, index: DepNodeIndex) {
*self.cache.lock() = Some((value, index));
value
}
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
@ -162,85 +152,12 @@ fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
}
}
pub struct ArenaCache<'tcx, K, V> {
arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
#[cfg(parallel_compiler)]
cache: Sharded<FxHashMap<K, &'tcx (V, DepNodeIndex)>>,
#[cfg(not(parallel_compiler))]
cache: Lock<FxHashMap<K, &'tcx (V, DepNodeIndex)>>,
}
impl<'tcx, K, V> Default for ArenaCache<'tcx, K, V> {
fn default() -> Self {
ArenaCache { arena: WorkerLocal::new(|_| TypedArena::default()), cache: Default::default() }
}
}
impl<'tcx, K: Eq + Hash, V: Debug + 'tcx> QueryStorage for ArenaCache<'tcx, K, V> {
type Value = V;
type Stored = &'tcx V;
}
impl<'tcx, K, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V>
where
K: Eq + Hash + Clone + Debug,
V: Debug,
{
type Key = K;
#[inline(always)]
fn lookup(&self, key: &K) -> Option<(&'tcx V, DepNodeIndex)> {
let key_hash = sharded::make_hash(key);
#[cfg(parallel_compiler)]
let lock = self.cache.get_shard_by_hash(key_hash).lock();
#[cfg(not(parallel_compiler))]
let lock = self.cache.lock();
let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key);
if let Some((_, value)) = result { Some((&value.0, value.1)) } else { None }
}
#[inline]
fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
let value = self.arena.alloc((value, index));
let value = unsafe { &*(value as *const _) };
#[cfg(parallel_compiler)]
let mut lock = self.cache.get_shard_by_value(&key).lock();
#[cfg(not(parallel_compiler))]
let mut lock = self.cache.lock();
// We may be overwriting another value. This is all right, since the dep-graph
// will check that the fingerprint matches.
lock.insert(key, value);
&value.0
}
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
#[cfg(parallel_compiler)]
{
let shards = self.cache.lock_shards();
for shard in shards.iter() {
for (k, v) in shard.iter() {
f(k, &v.0, v.1);
}
}
}
#[cfg(not(parallel_compiler))]
{
let map = self.cache.lock();
for (k, v) in map.iter() {
f(k, &v.0, v.1);
}
}
}
}
pub struct VecCacheSelector<K>(PhantomData<K>);
impl<'tcx, K: Idx, V: 'tcx> CacheSelector<'tcx, V> for VecCacheSelector<K> {
type Cache = VecCache<K, V>
where
V: Copy;
type ArenaCache = VecArenaCache<'tcx, K, V>;
}
pub struct VecCache<K: Idx, V> {
@ -258,7 +175,6 @@ fn default() -> Self {
impl<K: Eq + Idx, V: Copy + Debug> QueryStorage for VecCache<K, V> {
type Value = V;
type Stored = V;
}
impl<K, V> QueryCache for VecCache<K, V>
@ -278,87 +194,12 @@ fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
}
#[inline]
fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
fn complete(&self, key: K, value: V, index: DepNodeIndex) {
#[cfg(parallel_compiler)]
let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
#[cfg(not(parallel_compiler))]
let mut lock = self.cache.lock();
lock.insert(key, (value, index));
value
}
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
#[cfg(parallel_compiler)]
{
let shards = self.cache.lock_shards();
for shard in shards.iter() {
for (k, v) in shard.iter_enumerated() {
if let Some(v) = v {
f(&k, &v.0, v.1);
}
}
}
}
#[cfg(not(parallel_compiler))]
{
let map = self.cache.lock();
for (k, v) in map.iter_enumerated() {
if let Some(v) = v {
f(&k, &v.0, v.1);
}
}
}
}
}
pub struct VecArenaCache<'tcx, K: Idx, V> {
arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
#[cfg(parallel_compiler)]
cache: Sharded<IndexVec<K, Option<&'tcx (V, DepNodeIndex)>>>,
#[cfg(not(parallel_compiler))]
cache: Lock<IndexVec<K, Option<&'tcx (V, DepNodeIndex)>>>,
}
impl<'tcx, K: Idx, V> Default for VecArenaCache<'tcx, K, V> {
fn default() -> Self {
VecArenaCache {
arena: WorkerLocal::new(|_| TypedArena::default()),
cache: Default::default(),
}
}
}
impl<'tcx, K: Eq + Idx, V: Debug + 'tcx> QueryStorage for VecArenaCache<'tcx, K, V> {
type Value = V;
type Stored = &'tcx V;
}
impl<'tcx, K, V: 'tcx> QueryCache for VecArenaCache<'tcx, K, V>
where
K: Eq + Idx + Clone + Debug,
V: Debug,
{
type Key = K;
#[inline(always)]
fn lookup(&self, key: &K) -> Option<(&'tcx V, DepNodeIndex)> {
#[cfg(parallel_compiler)]
let lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
#[cfg(not(parallel_compiler))]
let lock = self.cache.lock();
if let Some(Some(value)) = lock.get(*key) { Some((&value.0, value.1)) } else { None }
}
#[inline]
fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
let value = self.arena.alloc((value, index));
let value = unsafe { &*(value as *const _) };
#[cfg(parallel_compiler)]
let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
#[cfg(not(parallel_compiler))]
let mut lock = self.cache.lock();
lock.insert(key, value);
&value.0
}
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {

9
compiler/rustc_query_system/src/query/config.rs
View File

@ -20,10 +20,9 @@ pub trait QueryConfig<Qcx: QueryContext> {
const NAME: &'static str;
type Key: DepNodeParams<Qcx::DepContext> + Eq + Hash + Clone + Debug;
type Value: Debug;
type Stored: Debug + Copy + std::borrow::Borrow<Self::Value>;
type Value: Debug + Copy;
type Cache: QueryCache<Key = Self::Key, Stored = Self::Stored, Value = Self::Value>;
type Cache: QueryCache<Key = Self::Key, Value = Self::Value>;
// Don't use this method to access query results, instead use the methods on TyCtxt
fn query_state<'a>(tcx: Qcx) -> &'a QueryState<Self::Key, Qcx::DepKind>
@ -38,9 +37,9 @@ fn query_cache<'a>(tcx: Qcx) -> &'a Self::Cache
fn cache_on_disk(tcx: Qcx::DepContext, key: &Self::Key) -> bool;
// Don't use this method to compute query results, instead use the methods on TyCtxt
fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Stored;
fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Value;
fn compute(tcx: Qcx, key: &Self::Key) -> fn(Qcx::DepContext, Self::Key) -> Self::Value;
fn compute(tcx: Qcx::DepContext, key: Self::Key) -> Self::Value;
fn try_load_from_disk(qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk<Qcx, Self>;

49
compiler/rustc_query_system/src/query/plumbing.rs
View File

@ -246,7 +246,7 @@ fn try_start<'b, Qcx>(
/// Completes the query by updating the query cache with the `result`,
/// signals the waiter and forgets the JobOwner, so it won't poison the query
fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex) -> C::Stored
fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex)
where
C: QueryCache<Key = K>,
{
@ -257,23 +257,22 @@ fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex)
// Forget ourself so our destructor won't poison the query
mem::forget(self);
let (job, result) = {
let job = {
#[cfg(parallel_compiler)]
let mut lock = state.active.get_shard_by_value(&key).lock();
#[cfg(not(parallel_compiler))]
let mut lock = state.active.lock();
match lock.remove(&key).unwrap() {
QueryResult::Started(job) => job,
QueryResult::Poisoned => panic!(),
}
};
let result = cache.complete(key, result, dep_node_index);
(job, result)
// Mark as complete before we remove the job from the active state
// so no other thread can re-execute this query.
cache.complete(key.clone(), result, dep_node_index);
let job = {
#[cfg(parallel_compiler)]
let mut lock = state.active.get_shard_by_value(&key).lock();
#[cfg(not(parallel_compiler))]
let mut lock = state.active.lock();
match lock.remove(&key).unwrap() {
QueryResult::Started(job) => job,
QueryResult::Poisoned => panic!(),
}
};
job.signal_complete();
result
}
}
@ -336,7 +335,7 @@ enum TryGetJob<'tcx, K, D>
/// which will be used if the query is not in the cache and we need
/// to compute it.
#[inline]
pub fn try_get_cached<Tcx, C>(tcx: Tcx, cache: &C, key: &C::Key) -> Option<C::Stored>
pub fn try_get_cached<Tcx, C>(tcx: Tcx, cache: &C, key: &C::Key) -> Option<C::Value>
where
C: QueryCache,
Tcx: DepContext,
@ -358,7 +357,7 @@ fn try_execute_query<Q, Qcx>(
span: Span,
key: Q::Key,
dep_node: Option<DepNode<Qcx::DepKind>>,
) -> (Q::Stored, Option<DepNodeIndex>)
) -> (Q::Value, Option<DepNodeIndex>)
where
Q: QueryConfig<Qcx>,
Qcx: QueryContext,
@ -390,7 +389,7 @@ fn try_execute_query<Q, Qcx>(
);
}
}
let result = job.complete(cache, result, dep_node_index);
job.complete(cache, result, dep_node_index);
(result, Some(dep_node_index))
}
TryGetJob::Cycle(error) => {
@ -426,9 +425,8 @@ fn execute_job<Q, Qcx>(
// Fast path for when incr. comp. is off.
if !dep_graph.is_fully_enabled() {
let prof_timer = qcx.dep_context().profiler().query_provider();
let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || {
Q::compute(qcx, &key)(*qcx.dep_context(), key)
});
let result =
qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || Q::compute(*qcx.dep_context(), key));
let dep_node_index = dep_graph.next_virtual_depnode_index();
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
return (result, dep_node_index);
@ -455,7 +453,7 @@ fn execute_job<Q, Qcx>(
qcx.start_query(job_id, Q::DEPTH_LIMIT, Some(&diagnostics), || {
if Q::ANON {
return dep_graph.with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || {
Q::compute(qcx, &key)(*qcx.dep_context(), key)
Q::compute(*qcx.dep_context(), key)
});
}
@ -463,8 +461,7 @@ fn execute_job<Q, Qcx>(
let dep_node =
dep_node_opt.unwrap_or_else(|| Q::construct_dep_node(*qcx.dep_context(), &key));
let task = Q::compute(qcx, &key);
dep_graph.with_task(dep_node, *qcx.dep_context(), key, task, Q::HASH_RESULT)
dep_graph.with_task(dep_node, *qcx.dep_context(), key, Q::compute, Q::HASH_RESULT)
});
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
@ -555,7 +552,7 @@ fn try_load_from_disk_and_cache_in_memory<Q, Qcx>(
let prof_timer = qcx.dep_context().profiler().query_provider();
// The dep-graph for this computation is already in-place.
let result = dep_graph.with_ignore(|| Q::compute(qcx, key)(*qcx.dep_context(), key.clone()));
let result = dep_graph.with_ignore(|| Q::compute(*qcx.dep_context(), key.clone()));
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
@ -727,7 +724,7 @@ pub enum QueryMode {
Ensure,
}
pub fn get_query<Q, Qcx, D>(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Stored>
pub fn get_query<Q, Qcx, D>(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Value>
where
D: DepKind,
Q: QueryConfig<Qcx>,