From b74cb78d63879bcca4e8039d45cd1878e0687b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Wed, 16 Aug 2023 16:49:40 +0200 Subject: [PATCH 1/2] Remove conditional use of `Sharded` from query state --- .../rustc_query_system/src/query/plumbing.rs | 53 ++++--------------- 1 file changed, 10 insertions(+), 43 deletions(-) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 4adb4eb7475..74b8aaf4f27 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -12,12 +12,13 @@ use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobI use crate::query::SerializedDepNodeIndex; use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame}; use crate::HandleCycleError; +#[cfg(parallel_compiler)] +use rustc_data_structures::cold_path; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sharded::Sharded; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::sync::Lock; -#[cfg(parallel_compiler)] -use rustc_data_structures::{cold_path, sharded::Sharded}; use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError}; use rustc_span::{Span, DUMMY_SP}; use std::cell::Cell; @@ -30,10 +31,7 @@ use thin_vec::ThinVec; use super::QueryConfig; pub struct QueryState { - #[cfg(parallel_compiler)] active: Sharded>>, - #[cfg(not(parallel_compiler))] - active: Lock>>, } /// Indicates the state of a query for a given key in a query map. @@ -52,15 +50,8 @@ where D: DepKind, { pub fn all_inactive(&self) -> bool { - #[cfg(parallel_compiler)] - { - let shards = self.active.lock_shards(); - shards.iter().all(|shard| shard.is_empty()) - } - #[cfg(not(parallel_compiler))] - { - self.active.lock().is_empty() - } + let shards = self.active.lock_shards(); + shards.iter().all(|shard| shard.is_empty()) } pub fn try_collect_active_jobs( @@ -71,26 +62,11 @@ where ) -> Option<()> { let mut active = Vec::new(); - #[cfg(parallel_compiler)] - { - // We use try_lock_shards here since we are called from the - // deadlock handler, and this shouldn't be locked. - let shards = self.active.try_lock_shards()?; - for shard in shards.iter() { - for (k, v) in shard.iter() { - if let QueryResult::Started(ref job) = *v { - active.push((*k, job.clone())); - } - } - } - } - #[cfg(not(parallel_compiler))] - { - // We use try_lock here since we are called from the - // deadlock handler, and this shouldn't be locked. - // (FIXME: Is this relevant for non-parallel compilers? It doesn't - // really hurt much.) - for (k, v) in self.active.try_lock()?.iter() { + // We use try_lock_shards here since we are called from the + // deadlock handler, and this shouldn't be locked. + let shards = self.active.try_lock_shards()?; + for shard in shards.iter() { + for (k, v) in shard.iter() { if let QueryResult::Started(ref job) = *v { active.push((*k, job.clone())); } @@ -184,10 +160,7 @@ where cache.complete(key, result, dep_node_index); let job = { - #[cfg(parallel_compiler)] let mut lock = state.active.get_shard_by_value(&key).lock(); - #[cfg(not(parallel_compiler))] - let mut lock = state.active.lock(); match lock.remove(&key).unwrap() { QueryResult::Started(job) => job, QueryResult::Poisoned => panic!(), @@ -209,10 +182,7 @@ where // Poison the query so jobs waiting on it panic. let state = self.state; let job = { - #[cfg(parallel_compiler)] let mut shard = state.active.get_shard_by_value(&self.key).lock(); - #[cfg(not(parallel_compiler))] - let mut shard = state.active.lock(); let job = match shard.remove(&self.key).unwrap() { QueryResult::Started(job) => job, QueryResult::Poisoned => panic!(), @@ -325,10 +295,7 @@ where Qcx: QueryContext, { let state = query.query_state(qcx); - #[cfg(parallel_compiler)] let mut state_lock = state.active.get_shard_by_value(&key).lock(); - #[cfg(not(parallel_compiler))] - let mut state_lock = state.active.lock(); // For the parallel compiler we need to check both the query cache and query state structures // while holding the state lock to ensure that 1) the query has not yet completed and 2) the From f458b112f88e1dbcd5072733c79e25328f9f24f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 17 Aug 2023 11:07:50 +0200 Subject: [PATCH 2/2] Optimize `lock_shards` --- compiler/rustc_data_structures/src/sharded.rs | 25 +++++++++---- compiler/rustc_middle/src/ty/context.rs | 37 ++++++++++--------- .../rustc_query_system/src/query/caches.rs | 6 +-- .../rustc_query_system/src/query/plumbing.rs | 8 ++-- 4 files changed, 42 insertions(+), 34 deletions(-) diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index 52ab5a7fb14..0f769c1f3bf 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -2,9 +2,12 @@ use crate::fx::{FxHashMap, FxHasher}; #[cfg(parallel_compiler)] use crate::sync::{is_dyn_thread_safe, CacheAligned}; use crate::sync::{Lock, LockGuard}; +#[cfg(parallel_compiler)] +use itertools::Either; use std::borrow::Borrow; use std::collections::hash_map::RawEntryMut; use std::hash::{Hash, Hasher}; +use std::iter; use std::mem; // 32 shards is sufficient to reduce contention on an 8-core Ryzen 7 1700, @@ -70,19 +73,27 @@ impl Sharded { } } - pub fn lock_shards(&self) -> Vec> { + #[inline] + pub fn lock_shards(&self) -> impl Iterator> { match self { - Self::Single(single) => vec![single.lock()], + #[cfg(not(parallel_compiler))] + Self::Single(single) => iter::once(single.lock()), #[cfg(parallel_compiler)] - Self::Shards(shards) => shards.iter().map(|shard| shard.0.lock()).collect(), + Self::Single(single) => Either::Left(iter::once(single.lock())), + #[cfg(parallel_compiler)] + Self::Shards(shards) => Either::Right(shards.iter().map(|shard| shard.0.lock())), } } - pub fn try_lock_shards(&self) -> Option>> { + #[inline] + pub fn try_lock_shards(&self) -> impl Iterator>> { match self { - Self::Single(single) => Some(vec![single.try_lock()?]), + #[cfg(not(parallel_compiler))] + Self::Single(single) => iter::once(single.try_lock()), #[cfg(parallel_compiler)] - Self::Shards(shards) => shards.iter().map(|shard| shard.0.try_lock()).collect(), + Self::Single(single) => Either::Left(iter::once(single.try_lock())), + #[cfg(parallel_compiler)] + Self::Shards(shards) => Either::Right(shards.iter().map(|shard| shard.0.try_lock())), } } } @@ -101,7 +112,7 @@ pub type ShardedHashMap = Sharded>; impl ShardedHashMap { pub fn len(&self) -> usize { - self.lock_shards().iter().map(|shard| shard.len()).sum() + self.lock_shards().map(|shard| shard.len()).sum() } } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index be839e03cff..e2f9e299566 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1296,25 +1296,26 @@ macro_rules! sty_debug_print { }; $(let mut $variant = total;)* - let shards = tcx.interners.type_.lock_shards(); - let types = shards.iter().flat_map(|shard| shard.keys()); - for &InternedInSet(t) in types { - let variant = match t.internee { - ty::Bool | ty::Char | ty::Int(..) | ty::Uint(..) | - ty::Float(..) | ty::Str | ty::Never => continue, - ty::Error(_) => /* unimportant */ continue, - $(ty::$variant(..) => &mut $variant,)* - }; - let lt = t.flags.intersects(ty::TypeFlags::HAS_RE_INFER); - let ty = t.flags.intersects(ty::TypeFlags::HAS_TY_INFER); - let ct = t.flags.intersects(ty::TypeFlags::HAS_CT_INFER); + for shard in tcx.interners.type_.lock_shards() { + let types = shard.keys(); + for &InternedInSet(t) in types { + let variant = match t.internee { + ty::Bool | ty::Char | ty::Int(..) | ty::Uint(..) | + ty::Float(..) | ty::Str | ty::Never => continue, + ty::Error(_) => /* unimportant */ continue, + $(ty::$variant(..) => &mut $variant,)* + }; + let lt = t.flags.intersects(ty::TypeFlags::HAS_RE_INFER); + let ty = t.flags.intersects(ty::TypeFlags::HAS_TY_INFER); + let ct = t.flags.intersects(ty::TypeFlags::HAS_CT_INFER); - variant.total += 1; - total.total += 1; - if lt { total.lt_infer += 1; variant.lt_infer += 1 } - if ty { total.ty_infer += 1; variant.ty_infer += 1 } - if ct { total.ct_infer += 1; variant.ct_infer += 1 } - if lt && ty && ct { total.all_infer += 1; variant.all_infer += 1 } + variant.total += 1; + total.total += 1; + if lt { total.lt_infer += 1; variant.lt_infer += 1 } + if ty { total.ty_infer += 1; variant.ty_infer += 1 } + if ct { total.ct_infer += 1; variant.ct_infer += 1 } + if lt && ty && ct { total.all_infer += 1; variant.all_infer += 1 } + } } writeln!(fmt, "Ty interner total ty lt ct all")?; $(writeln!(fmt, " {:18}: {uses:6} {usespc:4.1}%, \ diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index 4ba9d53a92f..a96230fdc94 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -70,8 +70,7 @@ where } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - let shards = self.cache.lock_shards(); - for shard in shards.iter() { + for shard in self.cache.lock_shards() { for (k, v) in shard.iter() { f(k, &v.0, v.1); } @@ -160,8 +159,7 @@ where } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - let shards = self.cache.lock_shards(); - for shard in shards.iter() { + for shard in self.cache.lock_shards() { for (k, v) in shard.iter_enumerated() { if let Some(v) = v { f(&k, &v.0, v.1); diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 74b8aaf4f27..e8a22a7c5da 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -50,8 +50,7 @@ where D: DepKind, { pub fn all_inactive(&self) -> bool { - let shards = self.active.lock_shards(); - shards.iter().all(|shard| shard.is_empty()) + self.active.lock_shards().all(|shard| shard.is_empty()) } pub fn try_collect_active_jobs( @@ -64,9 +63,8 @@ where // We use try_lock_shards here since we are called from the // deadlock handler, and this shouldn't be locked. - let shards = self.active.try_lock_shards()?; - for shard in shards.iter() { - for (k, v) in shard.iter() { + for shard in self.active.try_lock_shards() { + for (k, v) in shard?.iter() { if let QueryResult::Started(ref job) = *v { active.push((*k, job.clone())); }