mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #123058 - lukas-code:clauses, r=lcnr

Browse Source 
[perf] cache type info for ParamEnv

This is an attempt to mitigate some of the perf regressions in https://github.com/rust-lang/rust/pull/122553#issuecomment-2007563027, but seems worth to test and land separately, since it is mostly unrelated to that PR.
This commit is contained in:
bors 2024-04-07 02:07:20 +00:00
commit 087ae978a1
14 changed files with 286 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
compiler/rustc_hir_analysis/src/collect/item_bounds.rs
View File

@ -165,10 +165,7 @@ pub(super) fn explicit_item_bounds_with_filter(
ty::EarlyBinder::bind(bounds)
}
pub(super) fn item_bounds(
tcx: TyCtxt<'_>,
def_id: DefId,
) -> ty::EarlyBinder<&'_ ty::List<ty::Clause<'_>>> {
pub(super) fn item_bounds(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<ty::Clauses<'_>> {
tcx.explicit_item_bounds(def_id).map_bound(|bounds| {
tcx.mk_clauses_from_iter(util::elaborate(tcx, bounds.iter().map(|&(bound, _span)| bound)))
})
@ -177,7 +174,7 @@ pub(super) fn item_bounds(
pub(super) fn item_super_predicates(
tcx: TyCtxt<'_>,
def_id: DefId,
) -> ty::EarlyBinder<&'_ ty::List<ty::Clause<'_>>> {
) -> ty::EarlyBinder<ty::Clauses<'_>> {
tcx.explicit_item_super_predicates(def_id).map_bound(|bounds| {
tcx.mk_clauses_from_iter(
util::elaborate(tcx, bounds.iter().map(|&(bound, _span)| bound)).filter_only_self(),
@ -188,12 +185,12 @@ pub(super) fn item_super_predicates(
pub(super) fn item_non_self_assumptions(
tcx: TyCtxt<'_>,
def_id: DefId,
) -> ty::EarlyBinder<&'_ ty::List<ty::Clause<'_>>> {
) -> ty::EarlyBinder<ty::Clauses<'_>> {
let all_bounds: FxIndexSet<_> = tcx.item_bounds(def_id).skip_binder().iter().collect();
let own_bounds: FxIndexSet<_> =
tcx.item_super_predicates(def_id).skip_binder().iter().collect();
if all_bounds.len() == own_bounds.len() {
ty::EarlyBinder::bind(ty::List::empty())
ty::EarlyBinder::bind(ty::ListWithCachedTypeInfo::empty())
} else {
ty::EarlyBinder::bind(tcx.mk_clauses_from_iter(all_bounds.difference(&own_bounds).copied()))
}

4
compiler/rustc_middle/src/query/erase.rs
View File

@ -67,6 +67,10 @@ impl<T> EraseType for &'_ ty::List<T> {
type Result = [u8; size_of::<&'static ty::List<()>>()];
}
impl<T> EraseType for &'_ ty::ListWithCachedTypeInfo<T> {
type Result = [u8; size_of::<&'static ty::ListWithCachedTypeInfo<()>>()];
}
impl<I: rustc_index::Idx, T> EraseType for &'_ rustc_index::IndexSlice<I, T> {
type Result = [u8; size_of::<&'static rustc_index::IndexSlice<u32, ()>>()];
}

2
compiler/rustc_middle/src/query/keys.rs
View File

@ -432,7 +432,7 @@ impl<'tcx> Key for (Ty<'tcx>, Ty<'tcx>) {
}
}
impl<'tcx> Key for &'tcx ty::List<ty::Clause<'tcx>> {
impl<'tcx> Key for ty::Clauses<'tcx> {
type Cache<V> = DefaultCache<Self, V>;
fn default_span(&self, _: TyCtxt<'_>) -> Span {

8
compiler/rustc_middle/src/query/mod.rs
View File

@ -398,15 +398,15 @@ rustc_queries! {
/// ```
///
/// Bounds from the parent (e.g. with nested impl trait) are not included.
query item_bounds(key: DefId) -> ty::EarlyBinder<&'tcx ty::List<ty::Clause<'tcx>>> {
query item_bounds(key: DefId) -> ty::EarlyBinder<ty::Clauses<'tcx>> {
desc { |tcx| "elaborating item bounds for `{}`", tcx.def_path_str(key) }
}
query item_super_predicates(key: DefId) -> ty::EarlyBinder<&'tcx ty::List<ty::Clause<'tcx>>> {
query item_super_predicates(key: DefId) -> ty::EarlyBinder<ty::Clauses<'tcx>> {
desc { |tcx| "elaborating item assumptions for `{}`", tcx.def_path_str(key) }
}
query item_non_self_assumptions(key: DefId) -> ty::EarlyBinder<&'tcx ty::List<ty::Clause<'tcx>>> {
query item_non_self_assumptions(key: DefId) -> ty::EarlyBinder<ty::Clauses<'tcx>> {
desc { |tcx| "elaborating item assumptions for `{}`", tcx.def_path_str(key) }
}
@ -2156,7 +2156,7 @@ rustc_queries! {
desc { "resolving instance `{}`", ty::Instance::new(key.value.0, key.value.1) }
}
query reveal_opaque_types_in_bounds(key: &'tcx ty::List<ty::Clause<'tcx>>) -> &'tcx ty::List<ty::Clause<'tcx>> {
query reveal_opaque_types_in_bounds(key: ty::Clauses<'tcx>) -> ty::Clauses<'tcx> {
desc { "revealing opaque types in `{:?}`", key }
}

6
compiler/rustc_middle/src/ty/codec.rs
View File

@ -414,7 +414,9 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<ty
}
}
impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<ty::Clause<'tcx>> {
impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
for ty::ListWithCachedTypeInfo<ty::Clause<'tcx>>
{
fn decode(decoder: &mut D) -> &'tcx Self {
let len = decoder.read_usize();
decoder.interner().mk_clauses_from_iter(
@ -461,7 +463,7 @@ impl_decodable_via_ref! {
&'tcx mir::BorrowCheckResult<'tcx>,
&'tcx mir::coverage::CodeRegion,
&'tcx ty::List<ty::BoundVariableKind>,
&'tcx ty::List<ty::Clause<'tcx>>,
&'tcx ty::ListWithCachedTypeInfo<ty::Clause<'tcx>>,
&'tcx ty::List<FieldIdx>,
&'tcx ty::List<(VariantIdx, FieldIdx)>,
}

61
compiler/rustc_middle/src/ty/context.rs
View File

@ -25,10 +25,10 @@ use crate::traits::solve::{
ExternalConstraints, ExternalConstraintsData, PredefinedOpaques, PredefinedOpaquesData,
};
use crate::ty::{
self, AdtDef, AdtDefData, AdtKind, Binder, Clause, Const, ConstData, GenericParamDefKind,
ImplPolarity, List, ParamConst, ParamTy, PolyExistentialPredicate, PolyFnSig, Predicate,
PredicateKind, PredicatePolarity, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty,
TyKind, TyVid, TypeVisitable, Visibility,
self, AdtDef, AdtDefData, AdtKind, Binder, Clause, Clauses, Const, ConstData,
GenericParamDefKind, ImplPolarity, List, ListWithCachedTypeInfo, ParamConst, ParamTy,
PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind, PredicatePolarity, Region,
RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyVid, TypeVisitable, Visibility,
};
use crate::ty::{GenericArg, GenericArgs, GenericArgsRef};
use rustc_ast::{self as ast, attr};
@ -130,6 +130,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
type SubtypePredicate = ty::SubtypePredicate<'tcx>;
type CoercePredicate = ty::CoercePredicate<'tcx>;
type ClosureKind = ty::ClosureKind;
type Clauses = ty::Clauses<'tcx>;
fn mk_canonical_var_infos(self, infos: &[ty::CanonicalVarInfo<Self>]) -> Self::CanonicalVars {
self.mk_canonical_var_infos(infos)
@ -152,7 +153,7 @@ pub struct CtxtInterners<'tcx> {
region: InternedSet<'tcx, RegionKind<'tcx>>,
poly_existential_predicates: InternedSet<'tcx, List<PolyExistentialPredicate<'tcx>>>,
predicate: InternedSet<'tcx, WithCachedTypeInfo<ty::Binder<'tcx, PredicateKind<'tcx>>>>,
clauses: InternedSet<'tcx, List<Clause<'tcx>>>,
clauses: InternedSet<'tcx, ListWithCachedTypeInfo<Clause<'tcx>>>,
projs: InternedSet<'tcx, List<ProjectionKind>>,
place_elems: InternedSet<'tcx, List<PlaceElem<'tcx>>>,
const_: InternedSet<'tcx, WithCachedTypeInfo<ConstData<'tcx>>>,
@ -286,6 +287,24 @@ impl<'tcx> CtxtInterners<'tcx> {
.0,
))
}
fn intern_clauses(&self, clauses: &[Clause<'tcx>]) -> Clauses<'tcx> {
if clauses.is_empty() {
ListWithCachedTypeInfo::empty()
} else {
self.clauses
.intern_ref(clauses, || {
let flags = super::flags::FlagComputation::for_clauses(clauses);
InternedInSet(ListWithCachedTypeInfo::from_arena(
&*self.arena,
flags.into(),
clauses,
))
})
.0
}
}
}
// For these preinterned values, an alternative would be to have
@ -1783,6 +1802,29 @@ impl<'tcx, T: Hash> Hash for InternedInSet<'tcx, List<T>> {
}
}
impl<'tcx, T> Borrow<[T]> for InternedInSet<'tcx, ListWithCachedTypeInfo<T>> {
fn borrow(&self) -> &[T] {
&self.0[..]
}
}
impl<'tcx, T: PartialEq> PartialEq for InternedInSet<'tcx, ListWithCachedTypeInfo<T>> {
fn eq(&self, other: &InternedInSet<'tcx, ListWithCachedTypeInfo<T>>) -> bool {
// The `Borrow` trait requires that `x.borrow() == y.borrow()` equals
// `x == y`.
self.0[..] == other.0[..]
}
}
impl<'tcx, T: Eq> Eq for InternedInSet<'tcx, ListWithCachedTypeInfo<T>> {}
impl<'tcx, T: Hash> Hash for InternedInSet<'tcx, ListWithCachedTypeInfo<T>> {
fn hash<H: Hasher>(&self, s: &mut H) {
// The `Borrow` trait requires that `x.borrow().hash(s) == x.hash(s)`.
self.0[..].hash(s)
}
}
macro_rules! direct_interners {
($($name:ident: $vis:vis $method:ident($ty:ty): $ret_ctor:ident -> $ret_ty:ty,)+) => {
$(impl<'tcx> Borrow<$ty> for InternedInSet<'tcx, $ty> {
@ -1841,7 +1883,7 @@ macro_rules! slice_interners {
List::empty()
} else {
self.interners.$field.intern_ref(v, || {
InternedInSet(List::from_arena(&*self.arena, v))
InternedInSet(List::from_arena(&*self.arena, (), v))
}).0
}
})+
@ -1858,7 +1900,6 @@ slice_interners!(
type_lists: pub mk_type_list(Ty<'tcx>),
canonical_var_infos: pub mk_canonical_var_infos(CanonicalVarInfo<'tcx>),
poly_existential_predicates: intern_poly_existential_predicates(PolyExistentialPredicate<'tcx>),
clauses: intern_clauses(Clause<'tcx>),
projs: pub mk_projs(ProjectionKind),
place_elems: pub mk_place_elems(PlaceElem<'tcx>),
bound_variable_kinds: pub mk_bound_variable_kinds(ty::BoundVariableKind),
@ -2163,11 +2204,11 @@ impl<'tcx> TyCtxt<'tcx> {
self.intern_poly_existential_predicates(eps)
}
pub fn mk_clauses(self, clauses: &[Clause<'tcx>]) -> &'tcx List<Clause<'tcx>> {
pub fn mk_clauses(self, clauses: &[Clause<'tcx>]) -> Clauses<'tcx> {
// FIXME consider asking the input slice to be sorted to avoid
// re-interning permutations, in which case that would be asserted
// here.
self.intern_clauses(clauses)
self.interners.intern_clauses(clauses)
}
pub fn mk_local_def_ids(self, clauses: &[LocalDefId]) -> &'tcx List<LocalDefId> {
@ -2231,7 +2272,7 @@ impl<'tcx> TyCtxt<'tcx> {
pub fn mk_clauses_from_iter<I, T>(self, iter: I) -> T::Output
where
I: Iterator<Item = T>,
T: CollectAndApply<Clause<'tcx>, &'tcx List<Clause<'tcx>>>,
T: CollectAndApply<Clause<'tcx>, Clauses<'tcx>>,
{
T::collect_and_apply(iter, |xs| self.mk_clauses(xs))
}

9
compiler/rustc_middle/src/ty/flags.rs
View File

@ -35,6 +35,15 @@ impl FlagComputation {
result
}
pub fn for_clauses(clauses: &[ty::Clause<'_>]) -> FlagComputation {
let mut result = FlagComputation::new();
for c in clauses {
result.add_flags(c.as_predicate().flags());
result.add_exclusive_binder(c.as_predicate().outer_exclusive_binder());
}
result
}
fn add_flags(&mut self, flags: TypeFlags) {
self.flags = self.flags | flags;
}

9
compiler/rustc_middle/src/ty/impls_ty.rs
View File

@ -11,19 +11,20 @@ use rustc_data_structures::stable_hasher::HashingControls;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey};
use rustc_query_system::ich::StableHashingContext;
use std::cell::RefCell;
use std::ptr;
impl<'a, 'tcx, T> HashStable<StableHashingContext<'a>> for &'tcx ty::List<T>
impl<'a, 'tcx, H, T> HashStable<StableHashingContext<'a>> for &'tcx ty::list::RawList<H, T>
where
T: HashStable<StableHashingContext<'a>>,
{
fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
thread_local! {
static CACHE: RefCell<FxHashMap<(usize, usize, HashingControls), Fingerprint>> =
static CACHE: RefCell<FxHashMap<(*const (), HashingControls), Fingerprint>> =
RefCell::new(Default::default());
}
let hash = CACHE.with(|cache| {
let key = (self.as_ptr() as usize, self.len(), hcx.hashing_controls());
let key = (ptr::from_ref(*self).cast::<()>(), hcx.hashing_controls());
if let Some(&hash) = cache.borrow().get(&key) {
return hash;
}
@ -40,7 +41,7 @@ where
}
}
impl<'a, 'tcx, T> ToStableHashKey<StableHashingContext<'a>> for &'tcx ty::List<T>
impl<'a, 'tcx, H, T> ToStableHashKey<StableHashingContext<'a>> for &'tcx ty::list::RawList<H, T>
where
T: HashStable<StableHashingContext<'a>>,
{

204
compiler/rustc_middle/src/ty/list.rs
View File

@ -1,7 +1,8 @@
use super::flags::FlagComputation;
use super::{DebruijnIndex, DebugWithInfcx, InferCtxtLike, TyCtxt, TypeFlags, WithInfcx};
use crate::arena::Arena;
use rustc_data_structures::aligned::{align_of, Aligned};
use rustc_serialize::{Encodable, Encoder};
use rustc_type_ir::{InferCtxtLike, WithInfcx};
use std::alloc::Layout;
use std::cmp::Ordering;
use std::fmt;
@ -12,6 +13,9 @@ use std::ops::Deref;
use std::ptr;
use std::slice;
#[cfg(parallel_compiler)]
use rustc_data_structures::sync::DynSync;
/// `List<T>` is a bit like `&[T]`, but with some critical differences.
/// - IMPORTANT: Every `List<T>` is *required* to have unique contents. The
/// type's correctness relies on this, *but it does not enforce it*.
@ -28,15 +32,27 @@ use std::slice;
/// - `T` must be `Copy`. This lets `List<T>` be stored in a dropless arena and
/// iterators return a `T` rather than a `&T`.
/// - `T` must not be zero-sized.
#[repr(C)]
pub struct List<T> {
len: usize,
pub type List<T> = RawList<(), T>;
/// A generic type that can be used to prepend a [`List`] with some header.
///
/// The header will be ignored for value-based operations like [`PartialEq`],
/// [`Hash`] and [`Encodable`].
#[repr(C)]
pub struct RawList<H, T> {
skel: ListSkeleton<H, T>,
opaque: OpaqueListContents,
}
/// A [`RawList`] without the unsized tail. This type is used for layout computation
/// and constructing empty lists.
#[repr(C)]
struct ListSkeleton<H, T> {
header: H,
len: usize,
/// Although this claims to be a zero-length array, in practice `len`
/// elements are actually present.
data: [T; 0],
opaque: OpaqueListContents,
}
extern "C" {
@ -45,35 +61,17 @@ extern "C" {
type OpaqueListContents;
}
impl<T> List<T> {
/// Returns a reference to the (unique, static) empty list.
impl<H, T> RawList<H, T> {
#[inline(always)]
pub fn empty<'a>() -> &'a List<T> {
#[repr(align(64))]
struct MaxAlign;
assert!(mem::align_of::<T>() <= mem::align_of::<MaxAlign>());
#[repr(C)]
struct InOrder<T, U>(T, U);
// The empty slice is static and contains a single `0` usize (for the
// length) that is 64-byte aligned, thus featuring the necessary
// trailing padding for elements with up to 64-byte alignment.
static EMPTY_SLICE: InOrder<usize, MaxAlign> = InOrder(0, MaxAlign);
unsafe { &*(std::ptr::addr_of!(EMPTY_SLICE) as *const List<T>) }
}
pub fn len(&self) -> usize {
self.len
self.skel.len
}
#[inline(always)]
pub fn as_slice(&self) -> &[T] {
self
}
}
impl<T: Copy> List<T> {
/// Allocates a list from `arena` and copies the contents of `slice` into it.
///
/// WARNING: the contents *must be unique*, such that no list with these
@ -84,20 +82,31 @@ impl<T: Copy> List<T> {
/// (because the empty list exists statically, and is available via
/// `empty()`).
#[inline]
pub(super) fn from_arena<'tcx>(arena: &'tcx Arena<'tcx>, slice: &[T]) -> &'tcx List<T> {
pub(super) fn from_arena<'tcx>(
arena: &'tcx Arena<'tcx>,
header: H,
slice: &[T],
) -> &'tcx RawList<H, T>
where
T: Copy,
{
assert!(!mem::needs_drop::<T>());
assert!(mem::size_of::<T>() != 0);
assert!(!slice.is_empty());
let (layout, _offset) =
Layout::new::<usize>().extend(Layout::for_value::<[T]>(slice)).unwrap();
let mem = arena.dropless.alloc_raw(layout) as *mut List<T>;
Layout::new::<ListSkeleton<H, T>>().extend(Layout::for_value::<[T]>(slice)).unwrap();
let mem = arena.dropless.alloc_raw(layout) as *mut RawList<H, T>;
unsafe {
// Write the header
ptr::addr_of_mut!((*mem).skel.header).write(header);
// Write the length
ptr::addr_of_mut!((*mem).len).write(slice.len());
ptr::addr_of_mut!((*mem).skel.len).write(slice.len());
// Write the elements
ptr::addr_of_mut!((*mem).data)
ptr::addr_of_mut!((*mem).skel.data)
.cast::<T>()
.copy_from_nonoverlapping(slice.as_ptr(), slice.len());
@ -110,17 +119,44 @@ impl<T: Copy> List<T> {
//
// This would be weird, as `self.into_iter` iterates over `T` directly.
#[inline(always)]
pub fn iter(&self) -> <&'_ List<T> as IntoIterator>::IntoIter {
pub fn iter(&self) -> <&'_ RawList<H, T> as IntoIterator>::IntoIter
where
T: Copy,
{
self.into_iter()
}
}
impl<T: fmt::Debug> fmt::Debug for List<T> {
macro_rules! impl_list_empty {
($header_ty:ty, $header_init:expr) => {
impl<T> RawList<$header_ty, T> {
/// Returns a reference to the (per header unique, static) empty list.
#[inline(always)]
pub fn empty<'a>() -> &'a RawList<$header_ty, T> {
#[repr(align(64))]
struct MaxAlign;
static EMPTY: ListSkeleton<$header_ty, MaxAlign> =
ListSkeleton { header: $header_init, len: 0, data: [] };
assert!(mem::align_of::<T>() <= mem::align_of::<MaxAlign>());
// SAFETY: `EMPTY` is sufficiently aligned to be an empty list for all
// types with `align_of(T) <= align_of(MaxAlign)`, which we checked above.
unsafe { &*(std::ptr::addr_of!(EMPTY) as *const RawList<$header_ty, T>) }
}
}
};
}
impl_list_empty!((), ());
impl<H, T: fmt::Debug> fmt::Debug for RawList<H, T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
(**self).fmt(f)
}
}
impl<'tcx, T: super::DebugWithInfcx<TyCtxt<'tcx>>> super::DebugWithInfcx<TyCtxt<'tcx>> for List<T> {
impl<'tcx, H, T: DebugWithInfcx<TyCtxt<'tcx>>> DebugWithInfcx<TyCtxt<'tcx>> for RawList<H, T> {
fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
this: WithInfcx<'_, Infcx, &Self>,
f: &mut core::fmt::Formatter<'_>,
@ -129,40 +165,40 @@ impl<'tcx, T: super::DebugWithInfcx<TyCtxt<'tcx>>> super::DebugWithInfcx<TyCtxt<
}
}
impl<S: Encoder, T: Encodable<S>> Encodable<S> for List<T> {
impl<H, S: Encoder, T: Encodable<S>> Encodable<S> for RawList<H, T> {
#[inline]
fn encode(&self, s: &mut S) {
(**self).encode(s);
}
}
impl<T: PartialEq> PartialEq for List<T> {
impl<H, T: PartialEq> PartialEq for RawList<H, T> {
#[inline]
fn eq(&self, other: &List<T>) -> bool {
fn eq(&self, other: &RawList<H, T>) -> bool {
// Pointer equality implies list equality (due to the unique contents
// assumption).
ptr::eq(self, other)
}
}
impl<T: Eq> Eq for List<T> {}
impl<H, T: Eq> Eq for RawList<H, T> {}
impl<T> Ord for List<T>
impl<H, T> Ord for RawList<H, T>
where
T: Ord,
{
fn cmp(&self, other: &List<T>) -> Ordering {
fn cmp(&self, other: &RawList<H, T>) -> Ordering {
// Pointer equality implies list equality (due to the unique contents
// assumption), but the contents must be compared otherwise.
if self == other { Ordering::Equal } else { <[T] as Ord>::cmp(&**self, &**other) }
}
}
impl<T> PartialOrd for List<T>
impl<H, T> PartialOrd for RawList<H, T>
where
T: PartialOrd,
{
fn partial_cmp(&self, other: &List<T>) -> Option<Ordering> {
fn partial_cmp(&self, other: &RawList<H, T>) -> Option<Ordering> {
// Pointer equality implies list equality (due to the unique contents
// assumption), but the contents must be compared otherwise.
if self == other {
@ -173,16 +209,16 @@ where
}
}
impl<T> Hash for List<T> {
impl<Hdr, T> Hash for RawList<Hdr, T> {
#[inline]
fn hash<H: Hasher>(&self, s: &mut H) {
// Pointer hashing is sufficient (due to the unique contents
// assumption).
(self as *const List<T>).hash(s)
ptr::from_ref(self).hash(s)
}
}
impl<T> Deref for List<T> {
impl<H, T> Deref for RawList<H, T> {
type Target = [T];
#[inline(always)]
fn deref(&self) -> &[T] {
@ -190,14 +226,19 @@ impl<T> Deref for List<T> {
}
}
impl<T> AsRef<[T]> for List<T> {
impl<H, T> AsRef<[T]> for RawList<H, T> {
#[inline(always)]
fn as_ref(&self) -> &[T] {
unsafe { slice::from_raw_parts(self.data.as_ptr(), self.len) }
let data_ptr = ptr::addr_of!(self.skel.data).cast::<T>();
// SAFETY: `data_ptr` has the same provenance as `self` and can therefore
// access the `self.skel.len` elements stored at `self.skel.data`.
// Note that we specifically don't reborrow `&self.skel.data`, because that
// would give us a pointer with provenance over 0 bytes.
unsafe { slice::from_raw_parts(data_ptr, self.skel.len) }
}
}
impl<'a, T: Copy> IntoIterator for &'a List<T> {
impl<'a, H, T: Copy> IntoIterator for &'a RawList<H, T> {
type Item = T;
type IntoIter = iter::Copied<<&'a [T] as IntoIterator>::IntoIter>;
#[inline(always)]
@ -206,27 +247,56 @@ impl<'a, T: Copy> IntoIterator for &'a List<T> {
}
}
unsafe impl<T: Sync> Sync for List<T> {}
unsafe impl<H: Sync, T: Sync> Sync for RawList<H, T> {}
// We need this since `List` uses extern type `OpaqueListContents`.
#[cfg(parallel_compiler)]
use rustc_data_structures::sync::DynSync;
use super::TyCtxt;
#[cfg(parallel_compiler)]
unsafe impl<T: DynSync> DynSync for List<T> {}
unsafe impl<H: DynSync, T: DynSync> DynSync for RawList<H, T> {}
// Safety:
// Layouts of `Equivalent<T>` and `List<T>` are the same, modulo opaque tail,
// thus aligns of `Equivalent<T>` and `List<T>` must be the same.
unsafe impl<T> Aligned for List<T> {
const ALIGN: ptr::Alignment = {
#[repr(C)]
struct Equivalent<T> {
_len: usize,
_data: [T; 0],
}
align_of::<Equivalent<T>>()
};
// Layouts of `ListSkeleton<H, T>` and `RawList<H, T>` are the same, modulo opaque tail,
// thus aligns of `ListSkeleton<H, T>` and `RawList<H, T>` must be the same.
unsafe impl<H, T> Aligned for RawList<H, T> {
const ALIGN: ptr::Alignment = align_of::<ListSkeleton<H, T>>();
}
/// A [`List`] that additionally stores type information inline to speed up
/// [`TypeVisitableExt`](super::TypeVisitableExt) operations.
pub type ListWithCachedTypeInfo<T> = RawList<TypeInfo, T>;
impl<T> ListWithCachedTypeInfo<T> {
#[inline(always)]
pub fn flags(&self) -> TypeFlags {
self.skel.header.flags
}
#[inline(always)]
pub fn outer_exclusive_binder(&self) -> DebruijnIndex {
self.skel.header.outer_exclusive_binder
}
}
impl_list_empty!(TypeInfo, TypeInfo::empty());
/// The additional info that is stored in [`ListWithCachedTypeInfo`].
#[repr(C)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TypeInfo {
flags: TypeFlags,
outer_exclusive_binder: DebruijnIndex,
}
impl TypeInfo {
const fn empty() -> Self {
Self { flags: TypeFlags::empty(), outer_exclusive_binder: super::INNERMOST }
}
}
impl From<FlagComputation> for TypeInfo {
fn from(computation: FlagComputation) -> TypeInfo {
TypeInfo {
flags: computation.flags,
outer_exclusive_binder: computation.outer_exclusive_binder,
}
}
}

26
compiler/rustc_middle/src/ty/mod.rs
View File

@ -89,7 +89,7 @@ pub use self::context::{
TyCtxt, TyCtxtFeed,
};
pub use self::instance::{Instance, InstanceDef, ReifyReason, ShortInstance, UnusedGenericParams};
pub use self::list::List;
pub use self::list::{List, ListWithCachedTypeInfo};
pub use self::parameterized::ParameterizedOverTcx;
pub use self::predicate::{
Clause, ClauseKind, CoercePredicate, ExistentialPredicate, ExistentialProjection,
@ -1034,6 +1034,18 @@ impl PlaceholderLike for PlaceholderConst {
}
}
pub type Clauses<'tcx> = &'tcx ListWithCachedTypeInfo<Clause<'tcx>>;
impl<'tcx> rustc_type_ir::visit::Flags for Clauses<'tcx> {
fn flags(&self) -> TypeFlags {
(**self).flags()
}
fn outer_exclusive_binder(&self) -> DebruijnIndex {
(**self).outer_exclusive_binder()
}
}
/// When interacting with the type system we must provide information about the
/// environment. `ParamEnv` is the type that represents this information. See the
/// [dev guide chapter][param_env_guide] for more information.
@ -1053,7 +1065,7 @@ pub struct ParamEnv<'tcx> {
/// want `Reveal::All`.
///
/// Note: This is packed, use the reveal() method to access it.
packed: CopyTaggedPtr<&'tcx List<Clause<'tcx>>, ParamTag, true>,
packed: CopyTaggedPtr<Clauses<'tcx>, ParamTag, true>,
}
#[derive(Copy, Clone)]
@ -1112,11 +1124,11 @@ impl<'tcx> ParamEnv<'tcx> {
/// [param_env_guide]: https://rustc-dev-guide.rust-lang.org/param_env/param_env_summary.html
#[inline]
pub fn empty() -> Self {
Self::new(List::empty(), Reveal::UserFacing)
Self::new(ListWithCachedTypeInfo::empty(), Reveal::UserFacing)
}
#[inline]
pub fn caller_bounds(self) -> &'tcx List<Clause<'tcx>> {
pub fn caller_bounds(self) -> Clauses<'tcx> {
self.packed.pointer()
}
@ -1134,12 +1146,12 @@ impl<'tcx> ParamEnv<'tcx> {
/// or invoke `param_env.with_reveal_all()`.
#[inline]
pub fn reveal_all() -> Self {
Self::new(List::empty(), Reveal::All)
Self::new(ListWithCachedTypeInfo::empty(), Reveal::All)
}
/// Construct a trait environment with the given set of predicates.
#[inline]
pub fn new(caller_bounds: &'tcx List<Clause<'tcx>>, reveal: Reveal) -> Self {
pub fn new(caller_bounds: Clauses<'tcx>, reveal: Reveal) -> Self {
ty::ParamEnv { packed: CopyTaggedPtr::new(caller_bounds, ParamTag { reveal }) }
}
@ -1168,7 +1180,7 @@ impl<'tcx> ParamEnv<'tcx> {
/// Returns this same environment but with no caller bounds.
#[inline]
pub fn without_caller_bounds(self) -> Self {
Self::new(List::empty(), self.reveal())
Self::new(ListWithCachedTypeInfo::empty(), self.reveal())
}
/// Creates a pair of param-env and value for use in queries.

14
compiler/rustc_middle/src/ty/structural_impls.rs
View File

@ -712,7 +712,19 @@ impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
}
}
impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<ty::Clause<'tcx>> {
impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> V::Result {
visitor.visit_clauses(self)
}
}
impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
fn super_visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> V::Result {
self.as_slice().visit_with(visitor)
}
}
impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
self,
folder: &mut F,

20
compiler/rustc_middle/src/ty/util.rs
View File

@ -1608,16 +1608,18 @@ pub fn is_trivially_const_drop(ty: Ty<'_>) -> bool {
/// let v = self.iter().map(|p| p.fold_with(folder)).collect::<SmallVec<[_; 8]>>();
/// folder.tcx().intern_*(&v)
/// ```
pub fn fold_list<'tcx, F, T>(
list: &'tcx ty::List<T>,
pub fn fold_list<'tcx, F, L, T>(
list: L,
folder: &mut F,
intern: impl FnOnce(TyCtxt<'tcx>, &[T]) -> &'tcx ty::List<T>,
) -> Result<&'tcx ty::List<T>, F::Error>
intern: impl FnOnce(TyCtxt<'tcx>, &[T]) -> L,
) -> Result<L, F::Error>
where
F: FallibleTypeFolder<TyCtxt<'tcx>>,
L: AsRef<[T]>,
T: TypeFoldable<TyCtxt<'tcx>> + PartialEq + Copy,
{
let mut iter = list.iter();
let slice = list.as_ref();
let mut iter = slice.iter().copied();
// Look for the first element that changed
match iter.by_ref().enumerate().find_map(|(i, t)| match t.try_fold_with(folder) {
Ok(new_t) if new_t == t => None,
@ -1625,8 +1627,8 @@ where
}) {
Some((i, Ok(new_t))) => {
// An element changed, prepare to intern the resulting list
let mut new_list = SmallVec::<[_; 8]>::with_capacity(list.len());
new_list.extend_from_slice(&list[..i]);
let mut new_list = SmallVec::<[_; 8]>::with_capacity(slice.len());
new_list.extend_from_slice(&slice[..i]);
new_list.push(new_t);
for t in iter {
new_list.push(t.try_fold_with(folder)?)
@ -1647,8 +1649,8 @@ pub struct AlwaysRequiresDrop;
/// with their underlying types.
pub fn reveal_opaque_types_in_bounds<'tcx>(
tcx: TyCtxt<'tcx>,
val: &'tcx ty::List<ty::Clause<'tcx>>,
) -> &'tcx ty::List<ty::Clause<'tcx>> {
val: ty::Clauses<'tcx>,
) -> ty::Clauses<'tcx> {
let mut visitor = OpaqueTypeExpander {
seen_opaque_tys: FxHashSet::default(),
expanded_cache: FxHashMap::default(),

1
compiler/rustc_type_ir/src/interner.rs
View File

@ -100,6 +100,7 @@ pub trait Interner: Sized + Copy {
type SubtypePredicate: Copy + Debug + Hash + Eq;
type CoercePredicate: Copy + Debug + Hash + Eq;
type ClosureKind: Copy + Debug + Hash + Eq;
type Clauses: Copy + Debug + Hash + Eq + TypeSuperVisitable<Self> + Flags;
fn mk_canonical_var_infos(self, infos: &[CanonicalVarInfo<Self>]) -> Self::CanonicalVars;
}

23
compiler/rustc_type_ir/src/visit.rs
View File

@ -110,6 +110,10 @@ pub trait TypeVisitor<I: Interner>: Sized {
fn visit_predicate(&mut self, p: I::Predicate) -> Self::Result {
p.super_visit_with(self)
}
fn visit_clauses(&mut self, p: I::Clauses) -> Self::Result {
p.super_visit_with(self)
}
}
///////////////////////////////////////////////////////////////////////////
@ -423,6 +427,16 @@ impl<I: Interner> TypeVisitor<I> for HasTypeFlagsVisitor {
ControlFlow::Continue(())
}
}
#[inline]
fn visit_clauses(&mut self, clauses: I::Clauses) -> Self::Result {
// Note: no `super_visit_with` call.
if clauses.flags().intersects(self.flags) {
ControlFlow::Break(FoundFlags)
} else {
ControlFlow::Continue(())
}
}
}
#[derive(Debug, PartialEq, Eq, Copy, Clone)]
@ -515,6 +529,15 @@ impl<I: Interner> TypeVisitor<I> for HasEscapingVarsVisitor {
ControlFlow::Continue(())
}
}
#[inline]
fn visit_clauses(&mut self, clauses: I::Clauses) -> Self::Result {
if clauses.outer_exclusive_binder() > self.outer_index {
ControlFlow::Break(FoundEscapingVars)
} else {
ControlFlow::Continue(())
}
}
}
struct HasErrorVisitor;