From fcc477fbd0871d8fab045bd7e9524172ab10e51c Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Sun, 24 Mar 2024 22:49:31 +0100 Subject: [PATCH 1/2] cache type info for ParamEnv --- .../src/collect/item_bounds.rs | 11 +- compiler/rustc_middle/src/query/erase.rs | 4 + compiler/rustc_middle/src/query/keys.rs | 2 +- compiler/rustc_middle/src/query/mod.rs | 8 +- compiler/rustc_middle/src/ty/codec.rs | 6 +- compiler/rustc_middle/src/ty/context.rs | 59 +++- compiler/rustc_middle/src/ty/flags.rs | 9 + compiler/rustc_middle/src/ty/impls_ty.rs | 10 + compiler/rustc_middle/src/ty/list.rs | 284 +++++++++++++++--- compiler/rustc_middle/src/ty/mod.rs | 26 +- .../rustc_middle/src/ty/structural_impls.rs | 14 +- compiler/rustc_middle/src/ty/util.rs | 20 +- compiler/rustc_type_ir/src/interner.rs | 1 + compiler/rustc_type_ir/src/visit.rs | 23 ++ 14 files changed, 403 insertions(+), 74 deletions(-) diff --git a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs index f1b14adcb7a..c2205815ba6 100644 --- a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs +++ b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs @@ -165,10 +165,7 @@ pub(super) fn explicit_item_bounds_with_filter( ty::EarlyBinder::bind(bounds) } -pub(super) fn item_bounds( - tcx: TyCtxt<'_>, - def_id: DefId, -) -> ty::EarlyBinder<&'_ ty::List>> { +pub(super) fn item_bounds(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder> { tcx.explicit_item_bounds(def_id).map_bound(|bounds| { tcx.mk_clauses_from_iter(util::elaborate(tcx, bounds.iter().map(|&(bound, _span)| bound))) }) @@ -177,7 +174,7 @@ pub(super) fn item_bounds( pub(super) fn item_super_predicates( tcx: TyCtxt<'_>, def_id: DefId, -) -> ty::EarlyBinder<&'_ ty::List>> { +) -> ty::EarlyBinder> { tcx.explicit_item_super_predicates(def_id).map_bound(|bounds| { tcx.mk_clauses_from_iter( util::elaborate(tcx, bounds.iter().map(|&(bound, _span)| bound)).filter_only_self(), @@ -188,12 +185,12 @@ pub(super) fn item_super_predicates( pub(super) fn item_non_self_assumptions( tcx: TyCtxt<'_>, def_id: DefId, -) -> ty::EarlyBinder<&'_ ty::List>> { +) -> ty::EarlyBinder> { let all_bounds: FxIndexSet<_> = tcx.item_bounds(def_id).skip_binder().iter().collect(); let own_bounds: FxIndexSet<_> = tcx.item_super_predicates(def_id).skip_binder().iter().collect(); if all_bounds.len() == own_bounds.len() { - ty::EarlyBinder::bind(ty::List::empty()) + ty::EarlyBinder::bind(ty::ListWithCachedTypeInfo::empty()) } else { ty::EarlyBinder::bind(tcx.mk_clauses_from_iter(all_bounds.difference(&own_bounds).copied())) } diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs index d3da49c26a2..320d49ea646 100644 --- a/compiler/rustc_middle/src/query/erase.rs +++ b/compiler/rustc_middle/src/query/erase.rs @@ -67,6 +67,10 @@ impl EraseType for &'_ ty::List { type Result = [u8; size_of::<&'static ty::List<()>>()]; } +impl EraseType for &'_ ty::ListWithCachedTypeInfo { + type Result = [u8; size_of::<&'static ty::ListWithCachedTypeInfo<()>>()]; +} + impl EraseType for &'_ rustc_index::IndexSlice { type Result = [u8; size_of::<&'static rustc_index::IndexSlice>()]; } diff --git a/compiler/rustc_middle/src/query/keys.rs b/compiler/rustc_middle/src/query/keys.rs index 9cbc4d10146..c1548eb99f5 100644 --- a/compiler/rustc_middle/src/query/keys.rs +++ b/compiler/rustc_middle/src/query/keys.rs @@ -432,7 +432,7 @@ impl<'tcx> Key for (Ty<'tcx>, Ty<'tcx>) { } } -impl<'tcx> Key for &'tcx ty::List> { +impl<'tcx> Key for ty::Clauses<'tcx> { type Cache = DefaultCache; fn default_span(&self, _: TyCtxt<'_>) -> Span { diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 62a60a650ec..5ef7a20f460 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -398,15 +398,15 @@ rustc_queries! { /// ``` /// /// Bounds from the parent (e.g. with nested impl trait) are not included. - query item_bounds(key: DefId) -> ty::EarlyBinder<&'tcx ty::List>> { + query item_bounds(key: DefId) -> ty::EarlyBinder> { desc { |tcx| "elaborating item bounds for `{}`", tcx.def_path_str(key) } } - query item_super_predicates(key: DefId) -> ty::EarlyBinder<&'tcx ty::List>> { + query item_super_predicates(key: DefId) -> ty::EarlyBinder> { desc { |tcx| "elaborating item assumptions for `{}`", tcx.def_path_str(key) } } - query item_non_self_assumptions(key: DefId) -> ty::EarlyBinder<&'tcx ty::List>> { + query item_non_self_assumptions(key: DefId) -> ty::EarlyBinder> { desc { |tcx| "elaborating item assumptions for `{}`", tcx.def_path_str(key) } } @@ -2156,7 +2156,7 @@ rustc_queries! { desc { "resolving instance `{}`", ty::Instance::new(key.value.0, key.value.1) } } - query reveal_opaque_types_in_bounds(key: &'tcx ty::List>) -> &'tcx ty::List> { + query reveal_opaque_types_in_bounds(key: ty::Clauses<'tcx>) -> ty::Clauses<'tcx> { desc { "revealing opaque types in `{:?}`", key } } diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs index e7a1679b151..9068961d736 100644 --- a/compiler/rustc_middle/src/ty/codec.rs +++ b/compiler/rustc_middle/src/ty/codec.rs @@ -414,7 +414,9 @@ impl<'tcx, D: TyDecoder>> RefDecodable<'tcx, D> for ty::List>> RefDecodable<'tcx, D> for ty::List> { +impl<'tcx, D: TyDecoder>> RefDecodable<'tcx, D> + for ty::ListWithCachedTypeInfo> +{ fn decode(decoder: &mut D) -> &'tcx Self { let len = decoder.read_usize(); decoder.interner().mk_clauses_from_iter( @@ -461,7 +463,7 @@ impl_decodable_via_ref! { &'tcx mir::BorrowCheckResult<'tcx>, &'tcx mir::coverage::CodeRegion, &'tcx ty::List, - &'tcx ty::List>, + &'tcx ty::ListWithCachedTypeInfo>, &'tcx ty::List, &'tcx ty::List<(VariantIdx, FieldIdx)>, } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index b8d68c1b8be..4977023a668 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -25,10 +25,10 @@ use crate::traits::solve::{ ExternalConstraints, ExternalConstraintsData, PredefinedOpaques, PredefinedOpaquesData, }; use crate::ty::{ - self, AdtDef, AdtDefData, AdtKind, Binder, Clause, Const, ConstData, GenericParamDefKind, - ImplPolarity, List, ParamConst, ParamTy, PolyExistentialPredicate, PolyFnSig, Predicate, - PredicateKind, PredicatePolarity, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, - TyKind, TyVid, TypeVisitable, Visibility, + self, AdtDef, AdtDefData, AdtKind, Binder, Clause, Clauses, Const, ConstData, + GenericParamDefKind, ImplPolarity, List, ListWithCachedTypeInfo, ParamConst, ParamTy, + PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind, PredicatePolarity, Region, + RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyVid, TypeVisitable, Visibility, }; use crate::ty::{GenericArg, GenericArgs, GenericArgsRef}; use rustc_ast::{self as ast, attr}; @@ -130,6 +130,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> { type SubtypePredicate = ty::SubtypePredicate<'tcx>; type CoercePredicate = ty::CoercePredicate<'tcx>; type ClosureKind = ty::ClosureKind; + type Clauses = ty::Clauses<'tcx>; fn mk_canonical_var_infos(self, infos: &[ty::CanonicalVarInfo]) -> Self::CanonicalVars { self.mk_canonical_var_infos(infos) @@ -152,7 +153,7 @@ pub struct CtxtInterners<'tcx> { region: InternedSet<'tcx, RegionKind<'tcx>>, poly_existential_predicates: InternedSet<'tcx, List>>, predicate: InternedSet<'tcx, WithCachedTypeInfo>>>, - clauses: InternedSet<'tcx, List>>, + clauses: InternedSet<'tcx, ListWithCachedTypeInfo>>, projs: InternedSet<'tcx, List>, place_elems: InternedSet<'tcx, List>>, const_: InternedSet<'tcx, WithCachedTypeInfo>>, @@ -286,6 +287,24 @@ impl<'tcx> CtxtInterners<'tcx> { .0, )) } + + fn intern_clauses(&self, clauses: &[Clause<'tcx>]) -> Clauses<'tcx> { + if clauses.is_empty() { + ListWithCachedTypeInfo::empty() + } else { + self.clauses + .intern_ref(clauses, || { + let flags = super::flags::FlagComputation::for_clauses(clauses); + + InternedInSet(ListWithCachedTypeInfo::from_arena( + &*self.arena, + flags.into(), + clauses, + )) + }) + .0 + } + } } // For these preinterned values, an alternative would be to have @@ -1775,6 +1794,29 @@ impl<'tcx, T: Hash> Hash for InternedInSet<'tcx, List> { } } +impl<'tcx, T> Borrow<[T]> for InternedInSet<'tcx, ListWithCachedTypeInfo> { + fn borrow(&self) -> &[T] { + &self.0[..] + } +} + +impl<'tcx, T: PartialEq> PartialEq for InternedInSet<'tcx, ListWithCachedTypeInfo> { + fn eq(&self, other: &InternedInSet<'tcx, ListWithCachedTypeInfo>) -> bool { + // The `Borrow` trait requires that `x.borrow() == y.borrow()` equals + // `x == y`. + self.0[..] == other.0[..] + } +} + +impl<'tcx, T: Eq> Eq for InternedInSet<'tcx, ListWithCachedTypeInfo> {} + +impl<'tcx, T: Hash> Hash for InternedInSet<'tcx, ListWithCachedTypeInfo> { + fn hash(&self, s: &mut H) { + // The `Borrow` trait requires that `x.borrow().hash(s) == x.hash(s)`. + self.0[..].hash(s) + } +} + macro_rules! direct_interners { ($($name:ident: $vis:vis $method:ident($ty:ty): $ret_ctor:ident -> $ret_ty:ty,)+) => { $(impl<'tcx> Borrow<$ty> for InternedInSet<'tcx, $ty> { @@ -1850,7 +1892,6 @@ slice_interners!( type_lists: pub mk_type_list(Ty<'tcx>), canonical_var_infos: pub mk_canonical_var_infos(CanonicalVarInfo<'tcx>), poly_existential_predicates: intern_poly_existential_predicates(PolyExistentialPredicate<'tcx>), - clauses: intern_clauses(Clause<'tcx>), projs: pub mk_projs(ProjectionKind), place_elems: pub mk_place_elems(PlaceElem<'tcx>), bound_variable_kinds: pub mk_bound_variable_kinds(ty::BoundVariableKind), @@ -2155,11 +2196,11 @@ impl<'tcx> TyCtxt<'tcx> { self.intern_poly_existential_predicates(eps) } - pub fn mk_clauses(self, clauses: &[Clause<'tcx>]) -> &'tcx List> { + pub fn mk_clauses(self, clauses: &[Clause<'tcx>]) -> Clauses<'tcx> { // FIXME consider asking the input slice to be sorted to avoid // re-interning permutations, in which case that would be asserted // here. - self.intern_clauses(clauses) + self.interners.intern_clauses(clauses) } pub fn mk_local_def_ids(self, clauses: &[LocalDefId]) -> &'tcx List { @@ -2223,7 +2264,7 @@ impl<'tcx> TyCtxt<'tcx> { pub fn mk_clauses_from_iter(self, iter: I) -> T::Output where I: Iterator, - T: CollectAndApply, &'tcx List>>, + T: CollectAndApply, Clauses<'tcx>>, { T::collect_and_apply(iter, |xs| self.mk_clauses(xs)) } diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs index ca9c762611e..5feb6ef76d5 100644 --- a/compiler/rustc_middle/src/ty/flags.rs +++ b/compiler/rustc_middle/src/ty/flags.rs @@ -35,6 +35,15 @@ impl FlagComputation { result } + pub fn for_clauses(clauses: &[ty::Clause<'_>]) -> FlagComputation { + let mut result = FlagComputation::new(); + for c in clauses { + result.add_flags(c.as_predicate().flags()); + result.add_exclusive_binder(c.as_predicate().outer_exclusive_binder()); + } + result + } + fn add_flags(&mut self, flags: TypeFlags) { self.flags = self.flags | flags; } diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs index 129d947697e..9747a506fa0 100644 --- a/compiler/rustc_middle/src/ty/impls_ty.rs +++ b/compiler/rustc_middle/src/ty/impls_ty.rs @@ -55,6 +55,16 @@ where } } +impl<'a, 'tcx, T> HashStable> for &'tcx ty::ListWithCachedTypeInfo +where + T: HashStable>, +{ + #[inline] + fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) { + self.as_list().hash_stable(hcx, hasher); + } +} + impl<'a> ToStableHashKey> for SimplifiedType { type KeyType = Fingerprint; diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs index 336c2dce114..08078673d79 100644 --- a/compiler/rustc_middle/src/ty/list.rs +++ b/compiler/rustc_middle/src/ty/list.rs @@ -1,7 +1,8 @@ +use super::flags::FlagComputation; +use super::{DebruijnIndex, DebugWithInfcx, InferCtxtLike, TyCtxt, TypeFlags, WithInfcx}; use crate::arena::Arena; use rustc_data_structures::aligned::{align_of, Aligned}; use rustc_serialize::{Encodable, Encoder}; -use rustc_type_ir::{InferCtxtLike, WithInfcx}; use std::alloc::Layout; use std::cmp::Ordering; use std::fmt; @@ -12,6 +13,9 @@ use std::ops::Deref; use std::ptr; use std::slice; +#[cfg(parallel_compiler)] +use rustc_data_structures::sync::DynSync; + /// `List` is a bit like `&[T]`, but with some critical differences. /// - IMPORTANT: Every `List` is *required* to have unique contents. The /// type's correctness relies on this, *but it does not enforce it*. @@ -30,13 +34,32 @@ use std::slice; /// - `T` must not be zero-sized. #[repr(C)] pub struct List { + skel: ListSkeleton, + opaque: OpaqueListContents, +} + +/// This type defines the statically known field offsets and alignment of a [`List`]. +/// +/// The alignment of a `List` cannot be determined, because it has an extern type tail. +/// We use `ListSkeleton` instead of `List` whenever computing the alignment is required, +/// for example: +/// - Implementing the [`Aligned`] trait, which is required for [`CopyTaggedPtr`]. +/// - Projecting from [`ListWithCachedTypeInfo`] to `List`, which requires computing the padding +/// between the cached type info and the list, which requires computing the list's alignment. +/// +/// Note that `ListSkeleton` is `Sized`, but **it's size is not correct**, as it is missing the +/// dynamically sized list tail. Do not create a `ListSkeleton` on the stack. +/// +/// FIXME: This can be removed once we properly support `!Sized + Aligned + Thin` types. +/// +/// [`CopyTaggedPtr`]: rustc_data_structures::tagged_ptr::CopyTaggedPtr +#[repr(C)] +struct ListSkeleton { len: usize, /// Although this claims to be a zero-length array, in practice `len` /// elements are actually present. data: [T; 0], - - opaque: OpaqueListContents, } extern "C" { @@ -49,25 +72,15 @@ impl List { /// Returns a reference to the (unique, static) empty list. #[inline(always)] pub fn empty<'a>() -> &'a List { - #[repr(align(64))] - struct MaxAlign; - - assert!(mem::align_of::() <= mem::align_of::()); - - #[repr(C)] - struct InOrder(T, U); - - // The empty slice is static and contains a single `0` usize (for the - // length) that is 64-byte aligned, thus featuring the necessary - // trailing padding for elements with up to 64-byte alignment. - static EMPTY_SLICE: InOrder = InOrder(0, MaxAlign); - unsafe { &*(std::ptr::addr_of!(EMPTY_SLICE) as *const List) } + ListWithCachedTypeInfo::empty() } + #[inline(always)] pub fn len(&self) -> usize { - self.len + self.skel.len } + #[inline(always)] pub fn as_slice(&self) -> &[T] { self } @@ -94,10 +107,10 @@ impl List { let mem = arena.dropless.alloc_raw(layout) as *mut List; unsafe { // Write the length - ptr::addr_of_mut!((*mem).len).write(slice.len()); + ptr::addr_of_mut!((*mem).skel.len).write(slice.len()); // Write the elements - ptr::addr_of_mut!((*mem).data) + ptr::addr_of_mut!((*mem).skel.data) .cast::() .copy_from_nonoverlapping(slice.as_ptr(), slice.len()); @@ -120,7 +133,7 @@ impl fmt::Debug for List { (**self).fmt(f) } } -impl<'tcx, T: super::DebugWithInfcx>> super::DebugWithInfcx> for List { +impl<'tcx, T: DebugWithInfcx>> DebugWithInfcx> for List { fn fmt>>( this: WithInfcx<'_, Infcx, &Self>, f: &mut core::fmt::Formatter<'_>, @@ -178,7 +191,7 @@ impl Hash for List { fn hash(&self, s: &mut H) { // Pointer hashing is sufficient (due to the unique contents // assumption). - (self as *const List).hash(s) + ptr::from_ref(self).hash(s) } } @@ -193,7 +206,12 @@ impl Deref for List { impl AsRef<[T]> for List { #[inline(always)] fn as_ref(&self) -> &[T] { - unsafe { slice::from_raw_parts(self.data.as_ptr(), self.len) } + let data_ptr = ptr::addr_of!(self.skel.data).cast::(); + // SAFETY: `data_ptr` has the same provenance as `self` and can therefore + // access the `self.skel.len` elements stored at `self.skel.data`. + // Note that we specifically don't reborrow `&self.skel.data`, because that + // would give us a pointer with provenance over 0 bytes. + unsafe { slice::from_raw_parts(data_ptr, self.skel.len) } } } @@ -210,23 +228,221 @@ unsafe impl Sync for List {} // We need this since `List` uses extern type `OpaqueListContents`. #[cfg(parallel_compiler)] -use rustc_data_structures::sync::DynSync; - -use super::TyCtxt; -#[cfg(parallel_compiler)] unsafe impl DynSync for List {} // Safety: -// Layouts of `Equivalent` and `List` are the same, modulo opaque tail, -// thus aligns of `Equivalent` and `List` must be the same. +// Layouts of `ListSkeleton` and `List` are the same, modulo opaque tail, +// thus aligns of `ListSkeleton` and `List` must be the same. unsafe impl Aligned for List { - const ALIGN: ptr::Alignment = { + const ALIGN: ptr::Alignment = align_of::>(); +} + +/// A [`List`] that additionally stores type information inline to speed up +/// [`TypeVisitableExt`](super::TypeVisitableExt) operations. +#[repr(C)] +pub struct ListWithCachedTypeInfo { + skel: ListWithCachedTypeInfoSkeleton, + opaque: OpaqueListContents, +} + +/// The additional info that is stored in [`ListWithCachedTypeInfo`]. +#[repr(C)] +pub struct TypeInfo { + flags: TypeFlags, + outer_exclusive_binder: DebruijnIndex, +} + +impl From for TypeInfo { + fn from(computation: FlagComputation) -> TypeInfo { + TypeInfo { + flags: computation.flags, + outer_exclusive_binder: computation.outer_exclusive_binder, + } + } +} + +/// This type is similar to [`ListSkeleton`], but for [`ListWithCachedTypeInfo`]. +/// It is used for computing the alignment of a [`ListWithCachedTypeInfo`]. +#[repr(C)] +struct ListWithCachedTypeInfoSkeleton { + info: TypeInfo, + // N.B.: There may be padding between these two fields. We cannot use `List` directly + // here, because it has an unknown alignment which makes computing the amount of padding + // and therefore projecting from `&ListWithCachedTypeInfo` to `&List` impossible. + list: ListSkeleton, +} + +impl ListWithCachedTypeInfo { + #[inline(always)] + pub fn empty<'a>() -> &'a ListWithCachedTypeInfo { + #[repr(align(64))] + struct MaxAlign; + #[repr(C)] - struct Equivalent { - _len: usize, - _data: [T; 0], + struct Empty { + info: TypeInfo, + zero: [u8; 2 * mem::align_of::() - mem::size_of::()], + align: MaxAlign, } - align_of::>() - }; + static EMPTY: Empty = Empty { + info: TypeInfo { flags: TypeFlags::empty(), outer_exclusive_binder: super::INNERMOST }, + zero: [0; 2 * mem::align_of::() - mem::size_of::()], + align: MaxAlign, + }; + + assert!(mem::align_of::() <= mem::align_of::()); + + // The layout of the empty `ListWithCachedTypeInfo` must be one of the following, + // depending on the alignment of `T`: + // + // On 64-bit platforms: + // F = flags (32 bit), B = outer_exclusive_binder (32 bit), LL = len (64 bit) + // align(T) <= 8: FBLL + // align(T) = 16: FB..LL.. + // align(T) = 32: FB......LL...... + // align(T) = 64: FB..............LL.............. + // + // On 32-bit platforms: + // F = flags (32 bit), B = outer_exclusive_binder (32 bit), L = len (32 bit) + // align(T) <= 4: FBL + // align(T) = 8: FBL. + // align(T) = 16: FB..L... + // align(T) = 32: FB......L....... + // align(T) = 64: FB..............L............... + // + // We zero out every possible location of `len` so that `EMPTY` is a valid + // `ListWithCachedTypeInfo` for all `T` with alignment up to 64 bytes. + unsafe { &*(std::ptr::addr_of!(EMPTY) as *const ListWithCachedTypeInfo) } + } + + #[inline] + pub(super) fn from_arena<'tcx>( + arena: &'tcx Arena<'tcx>, + info: TypeInfo, + slice: &[T], + ) -> &'tcx ListWithCachedTypeInfo + where + T: Copy, + { + assert!(!mem::needs_drop::()); + assert!(mem::size_of::() != 0); + assert!(!slice.is_empty()); + + let (list_layout, _offset) = + Layout::new::().extend(Layout::for_value::<[T]>(slice)).unwrap(); + + let (layout, _offset) = Layout::new::().extend(list_layout).unwrap(); + + let mem = arena.dropless.alloc_raw(layout) as *mut ListWithCachedTypeInfo; + unsafe { + // Write the cached type info + ptr::addr_of_mut!((*mem).skel.info).write(info); + + // Write the length + ptr::addr_of_mut!((*mem).skel.list.len).write(slice.len()); + + // Write the elements + ptr::addr_of_mut!((*mem).skel.list.data) + .cast::() + .copy_from_nonoverlapping(slice.as_ptr(), slice.len()); + + &*mem + } + } + + #[inline(always)] + pub fn as_list(&self) -> &List { + self + } + + #[inline(always)] + pub fn flags(&self) -> TypeFlags { + self.skel.info.flags + } + + #[inline(always)] + pub fn outer_exclusive_binder(&self) -> DebruijnIndex { + self.skel.info.outer_exclusive_binder + } +} + +impl Deref for ListWithCachedTypeInfo { + type Target = List; + #[inline(always)] + fn deref(&self) -> &List { + let list_ptr = ptr::addr_of!(self.skel.list) as *const List; + unsafe { &*list_ptr } + } +} + +impl AsRef<[T]> for ListWithCachedTypeInfo { + #[inline(always)] + fn as_ref(&self) -> &[T] { + (&**self).as_ref() + } +} + +impl fmt::Debug for ListWithCachedTypeInfo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (**self).fmt(f) + } +} +impl<'tcx, T: DebugWithInfcx>> DebugWithInfcx> + for ListWithCachedTypeInfo +{ + fn fmt>>( + this: WithInfcx<'_, Infcx, &Self>, + f: &mut core::fmt::Formatter<'_>, + ) -> core::fmt::Result { + DebugWithInfcx::fmt(this.map(|this| &**this), f) + } +} + +impl> Encodable for ListWithCachedTypeInfo { + #[inline] + fn encode(&self, s: &mut S) { + (**self).encode(s); + } +} + +impl PartialEq for ListWithCachedTypeInfo { + #[inline] + fn eq(&self, other: &ListWithCachedTypeInfo) -> bool { + // Pointer equality implies list equality (due to the unique contents + // assumption). + ptr::eq(self, other) + } +} + +impl Eq for ListWithCachedTypeInfo {} + +impl Hash for ListWithCachedTypeInfo { + #[inline] + fn hash(&self, s: &mut H) { + // Pointer hashing is sufficient (due to the unique contents + // assumption). + ptr::from_ref(self).hash(s) + } +} + +impl<'a, T: Copy> IntoIterator for &'a ListWithCachedTypeInfo { + type Item = T; + type IntoIter = iter::Copied<<&'a [T] as IntoIterator>::IntoIter>; + #[inline(always)] + fn into_iter(self) -> Self::IntoIter { + (**self).into_iter() + } +} + +unsafe impl Sync for ListWithCachedTypeInfo {} + +#[cfg(parallel_compiler)] +unsafe impl DynSync for ListWithCachedTypeInfo {} + +// Safety: +// Layouts of `ListWithCachedTypeInfoSkeleton` and `ListWithCachedTypeInfo` +// are the same, modulo opaque tail, thus their aligns must be the same. +unsafe impl Aligned for ListWithCachedTypeInfo { + const ALIGN: ptr::Alignment = align_of::>(); } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index fb56c71c517..9bf5205edaa 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -89,7 +89,7 @@ pub use self::context::{ TyCtxt, TyCtxtFeed, }; pub use self::instance::{Instance, InstanceDef, ReifyReason, ShortInstance, UnusedGenericParams}; -pub use self::list::List; +pub use self::list::{List, ListWithCachedTypeInfo}; pub use self::parameterized::ParameterizedOverTcx; pub use self::predicate::{ Clause, ClauseKind, CoercePredicate, ExistentialPredicate, ExistentialProjection, @@ -1034,6 +1034,18 @@ impl PlaceholderLike for PlaceholderConst { } } +pub type Clauses<'tcx> = &'tcx ListWithCachedTypeInfo>; + +impl<'tcx> rustc_type_ir::visit::Flags for Clauses<'tcx> { + fn flags(&self) -> TypeFlags { + (**self).flags() + } + + fn outer_exclusive_binder(&self) -> DebruijnIndex { + (**self).outer_exclusive_binder() + } +} + /// When interacting with the type system we must provide information about the /// environment. `ParamEnv` is the type that represents this information. See the /// [dev guide chapter][param_env_guide] for more information. @@ -1053,7 +1065,7 @@ pub struct ParamEnv<'tcx> { /// want `Reveal::All`. /// /// Note: This is packed, use the reveal() method to access it. - packed: CopyTaggedPtr<&'tcx List>, ParamTag, true>, + packed: CopyTaggedPtr, ParamTag, true>, } #[derive(Copy, Clone)] @@ -1112,11 +1124,11 @@ impl<'tcx> ParamEnv<'tcx> { /// [param_env_guide]: https://rustc-dev-guide.rust-lang.org/param_env/param_env_summary.html #[inline] pub fn empty() -> Self { - Self::new(List::empty(), Reveal::UserFacing) + Self::new(ListWithCachedTypeInfo::empty(), Reveal::UserFacing) } #[inline] - pub fn caller_bounds(self) -> &'tcx List> { + pub fn caller_bounds(self) -> Clauses<'tcx> { self.packed.pointer() } @@ -1134,12 +1146,12 @@ impl<'tcx> ParamEnv<'tcx> { /// or invoke `param_env.with_reveal_all()`. #[inline] pub fn reveal_all() -> Self { - Self::new(List::empty(), Reveal::All) + Self::new(ListWithCachedTypeInfo::empty(), Reveal::All) } /// Construct a trait environment with the given set of predicates. #[inline] - pub fn new(caller_bounds: &'tcx List>, reveal: Reveal) -> Self { + pub fn new(caller_bounds: Clauses<'tcx>, reveal: Reveal) -> Self { ty::ParamEnv { packed: CopyTaggedPtr::new(caller_bounds, ParamTag { reveal }) } } @@ -1168,7 +1180,7 @@ impl<'tcx> ParamEnv<'tcx> { /// Returns this same environment but with no caller bounds. #[inline] pub fn without_caller_bounds(self) -> Self { - Self::new(List::empty(), self.reveal()) + Self::new(ListWithCachedTypeInfo::empty(), self.reveal()) } /// Creates a pair of param-env and value for use in queries. diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 0e7010e67d7..8231c0214cb 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -712,7 +712,19 @@ impl<'tcx> TypeSuperVisitable> for ty::Predicate<'tcx> { } } -impl<'tcx> TypeFoldable> for &'tcx ty::List> { +impl<'tcx> TypeVisitable> for ty::Clauses<'tcx> { + fn visit_with>>(&self, visitor: &mut V) -> V::Result { + visitor.visit_clauses(self) + } +} + +impl<'tcx> TypeSuperVisitable> for ty::Clauses<'tcx> { + fn super_visit_with>>(&self, visitor: &mut V) -> V::Result { + self.as_slice().visit_with(visitor) + } +} + +impl<'tcx> TypeFoldable> for ty::Clauses<'tcx> { fn try_fold_with>>( self, folder: &mut F, diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index f74bba134ab..cef15b29a85 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -1608,16 +1608,18 @@ pub fn is_trivially_const_drop(ty: Ty<'_>) -> bool { /// let v = self.iter().map(|p| p.fold_with(folder)).collect::>(); /// folder.tcx().intern_*(&v) /// ``` -pub fn fold_list<'tcx, F, T>( - list: &'tcx ty::List, +pub fn fold_list<'tcx, F, L, T>( + list: L, folder: &mut F, - intern: impl FnOnce(TyCtxt<'tcx>, &[T]) -> &'tcx ty::List, -) -> Result<&'tcx ty::List, F::Error> + intern: impl FnOnce(TyCtxt<'tcx>, &[T]) -> L, +) -> Result where F: FallibleTypeFolder>, + L: AsRef<[T]>, T: TypeFoldable> + PartialEq + Copy, { - let mut iter = list.iter(); + let slice = list.as_ref(); + let mut iter = slice.iter().copied(); // Look for the first element that changed match iter.by_ref().enumerate().find_map(|(i, t)| match t.try_fold_with(folder) { Ok(new_t) if new_t == t => None, @@ -1625,8 +1627,8 @@ where }) { Some((i, Ok(new_t))) => { // An element changed, prepare to intern the resulting list - let mut new_list = SmallVec::<[_; 8]>::with_capacity(list.len()); - new_list.extend_from_slice(&list[..i]); + let mut new_list = SmallVec::<[_; 8]>::with_capacity(slice.len()); + new_list.extend_from_slice(&slice[..i]); new_list.push(new_t); for t in iter { new_list.push(t.try_fold_with(folder)?) @@ -1647,8 +1649,8 @@ pub struct AlwaysRequiresDrop; /// with their underlying types. pub fn reveal_opaque_types_in_bounds<'tcx>( tcx: TyCtxt<'tcx>, - val: &'tcx ty::List>, -) -> &'tcx ty::List> { + val: ty::Clauses<'tcx>, +) -> ty::Clauses<'tcx> { let mut visitor = OpaqueTypeExpander { seen_opaque_tys: FxHashSet::default(), expanded_cache: FxHashMap::default(), diff --git a/compiler/rustc_type_ir/src/interner.rs b/compiler/rustc_type_ir/src/interner.rs index 7a2885dd3bb..7d6862726f0 100644 --- a/compiler/rustc_type_ir/src/interner.rs +++ b/compiler/rustc_type_ir/src/interner.rs @@ -100,6 +100,7 @@ pub trait Interner: Sized + Copy { type SubtypePredicate: Copy + Debug + Hash + Eq; type CoercePredicate: Copy + Debug + Hash + Eq; type ClosureKind: Copy + Debug + Hash + Eq; + type Clauses: Copy + Debug + Hash + Eq + TypeSuperVisitable + Flags; fn mk_canonical_var_infos(self, infos: &[CanonicalVarInfo]) -> Self::CanonicalVars; } diff --git a/compiler/rustc_type_ir/src/visit.rs b/compiler/rustc_type_ir/src/visit.rs index 839e75dba4c..35fef8d0762 100644 --- a/compiler/rustc_type_ir/src/visit.rs +++ b/compiler/rustc_type_ir/src/visit.rs @@ -110,6 +110,10 @@ pub trait TypeVisitor: Sized { fn visit_predicate(&mut self, p: I::Predicate) -> Self::Result { p.super_visit_with(self) } + + fn visit_clauses(&mut self, p: I::Clauses) -> Self::Result { + p.super_visit_with(self) + } } /////////////////////////////////////////////////////////////////////////// @@ -423,6 +427,16 @@ impl TypeVisitor for HasTypeFlagsVisitor { ControlFlow::Continue(()) } } + + #[inline] + fn visit_clauses(&mut self, clauses: I::Clauses) -> Self::Result { + // Note: no `super_visit_with` call. + if clauses.flags().intersects(self.flags) { + ControlFlow::Break(FoundFlags) + } else { + ControlFlow::Continue(()) + } + } } #[derive(Debug, PartialEq, Eq, Copy, Clone)] @@ -515,6 +529,15 @@ impl TypeVisitor for HasEscapingVarsVisitor { ControlFlow::Continue(()) } } + + #[inline] + fn visit_clauses(&mut self, clauses: I::Clauses) -> Self::Result { + if clauses.outer_exclusive_binder() > self.outer_index { + ControlFlow::Break(FoundEscapingVars) + } else { + ControlFlow::Continue(()) + } + } } struct HasErrorVisitor; From 94d61d899f1ce6a8178385447740600d64aa1dae Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Sat, 6 Apr 2024 17:20:30 +0200 Subject: [PATCH 2/2] add RawList --- compiler/rustc_middle/src/ty/context.rs | 2 +- compiler/rustc_middle/src/ty/impls_ty.rs | 19 +- compiler/rustc_middle/src/ty/list.rs | 338 +++++++---------------- 3 files changed, 102 insertions(+), 257 deletions(-) diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 4977023a668..0e7af711696 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1875,7 +1875,7 @@ macro_rules! slice_interners { List::empty() } else { self.interners.$field.intern_ref(v, || { - InternedInSet(List::from_arena(&*self.arena, v)) + InternedInSet(List::from_arena(&*self.arena, (), v)) }).0 } })+ diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs index 9747a506fa0..ef7a7a99ff7 100644 --- a/compiler/rustc_middle/src/ty/impls_ty.rs +++ b/compiler/rustc_middle/src/ty/impls_ty.rs @@ -11,19 +11,20 @@ use rustc_data_structures::stable_hasher::HashingControls; use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey}; use rustc_query_system::ich::StableHashingContext; use std::cell::RefCell; +use std::ptr; -impl<'a, 'tcx, T> HashStable> for &'tcx ty::List +impl<'a, 'tcx, H, T> HashStable> for &'tcx ty::list::RawList where T: HashStable>, { fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) { thread_local! { - static CACHE: RefCell> = + static CACHE: RefCell> = RefCell::new(Default::default()); } let hash = CACHE.with(|cache| { - let key = (self.as_ptr() as usize, self.len(), hcx.hashing_controls()); + let key = (ptr::from_ref(*self).cast::<()>(), hcx.hashing_controls()); if let Some(&hash) = cache.borrow().get(&key) { return hash; } @@ -40,7 +41,7 @@ where } } -impl<'a, 'tcx, T> ToStableHashKey> for &'tcx ty::List +impl<'a, 'tcx, H, T> ToStableHashKey> for &'tcx ty::list::RawList where T: HashStable>, { @@ -55,16 +56,6 @@ where } } -impl<'a, 'tcx, T> HashStable> for &'tcx ty::ListWithCachedTypeInfo -where - T: HashStable>, -{ - #[inline] - fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) { - self.as_list().hash_stable(hcx, hasher); - } -} - impl<'a> ToStableHashKey> for SimplifiedType { type KeyType = Fingerprint; diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs index 08078673d79..04e165c8a73 100644 --- a/compiler/rustc_middle/src/ty/list.rs +++ b/compiler/rustc_middle/src/ty/list.rs @@ -32,31 +32,24 @@ use rustc_data_structures::sync::DynSync; /// - `T` must be `Copy`. This lets `List` be stored in a dropless arena and /// iterators return a `T` rather than a `&T`. /// - `T` must not be zero-sized. +pub type List = RawList<(), T>; + +/// A generic type that can be used to prepend a [`List`] with some header. +/// +/// The header will be ignored for value-based operations like [`PartialEq`], +/// [`Hash`] and [`Encodable`]. #[repr(C)] -pub struct List { - skel: ListSkeleton, +pub struct RawList { + skel: ListSkeleton, opaque: OpaqueListContents, } -/// This type defines the statically known field offsets and alignment of a [`List`]. -/// -/// The alignment of a `List` cannot be determined, because it has an extern type tail. -/// We use `ListSkeleton` instead of `List` whenever computing the alignment is required, -/// for example: -/// - Implementing the [`Aligned`] trait, which is required for [`CopyTaggedPtr`]. -/// - Projecting from [`ListWithCachedTypeInfo`] to `List`, which requires computing the padding -/// between the cached type info and the list, which requires computing the list's alignment. -/// -/// Note that `ListSkeleton` is `Sized`, but **it's size is not correct**, as it is missing the -/// dynamically sized list tail. Do not create a `ListSkeleton` on the stack. -/// -/// FIXME: This can be removed once we properly support `!Sized + Aligned + Thin` types. -/// -/// [`CopyTaggedPtr`]: rustc_data_structures::tagged_ptr::CopyTaggedPtr +/// A [`RawList`] without the unsized tail. This type is used for layout computation +/// and constructing empty lists. #[repr(C)] -struct ListSkeleton { +struct ListSkeleton { + header: H, len: usize, - /// Although this claims to be a zero-length array, in practice `len` /// elements are actually present. data: [T; 0], @@ -68,13 +61,7 @@ extern "C" { type OpaqueListContents; } -impl List { - /// Returns a reference to the (unique, static) empty list. - #[inline(always)] - pub fn empty<'a>() -> &'a List { - ListWithCachedTypeInfo::empty() - } - +impl RawList { #[inline(always)] pub fn len(&self) -> usize { self.skel.len @@ -84,9 +71,7 @@ impl List { pub fn as_slice(&self) -> &[T] { self } -} -impl List { /// Allocates a list from `arena` and copies the contents of `slice` into it. /// /// WARNING: the contents *must be unique*, such that no list with these @@ -97,15 +82,26 @@ impl List { /// (because the empty list exists statically, and is available via /// `empty()`). #[inline] - pub(super) fn from_arena<'tcx>(arena: &'tcx Arena<'tcx>, slice: &[T]) -> &'tcx List { + pub(super) fn from_arena<'tcx>( + arena: &'tcx Arena<'tcx>, + header: H, + slice: &[T], + ) -> &'tcx RawList + where + T: Copy, + { assert!(!mem::needs_drop::()); assert!(mem::size_of::() != 0); assert!(!slice.is_empty()); let (layout, _offset) = - Layout::new::().extend(Layout::for_value::<[T]>(slice)).unwrap(); - let mem = arena.dropless.alloc_raw(layout) as *mut List; + Layout::new::>().extend(Layout::for_value::<[T]>(slice)).unwrap(); + + let mem = arena.dropless.alloc_raw(layout) as *mut RawList; unsafe { + // Write the header + ptr::addr_of_mut!((*mem).skel.header).write(header); + // Write the length ptr::addr_of_mut!((*mem).skel.len).write(slice.len()); @@ -123,17 +119,44 @@ impl List { // // This would be weird, as `self.into_iter` iterates over `T` directly. #[inline(always)] - pub fn iter(&self) -> <&'_ List as IntoIterator>::IntoIter { + pub fn iter(&self) -> <&'_ RawList as IntoIterator>::IntoIter + where + T: Copy, + { self.into_iter() } } -impl fmt::Debug for List { +macro_rules! impl_list_empty { + ($header_ty:ty, $header_init:expr) => { + impl RawList<$header_ty, T> { + /// Returns a reference to the (per header unique, static) empty list. + #[inline(always)] + pub fn empty<'a>() -> &'a RawList<$header_ty, T> { + #[repr(align(64))] + struct MaxAlign; + + static EMPTY: ListSkeleton<$header_ty, MaxAlign> = + ListSkeleton { header: $header_init, len: 0, data: [] }; + + assert!(mem::align_of::() <= mem::align_of::()); + + // SAFETY: `EMPTY` is sufficiently aligned to be an empty list for all + // types with `align_of(T) <= align_of(MaxAlign)`, which we checked above. + unsafe { &*(std::ptr::addr_of!(EMPTY) as *const RawList<$header_ty, T>) } + } + } + }; +} + +impl_list_empty!((), ()); + +impl fmt::Debug for RawList { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) } } -impl<'tcx, T: DebugWithInfcx>> DebugWithInfcx> for List { +impl<'tcx, H, T: DebugWithInfcx>> DebugWithInfcx> for RawList { fn fmt>>( this: WithInfcx<'_, Infcx, &Self>, f: &mut core::fmt::Formatter<'_>, @@ -142,40 +165,40 @@ impl<'tcx, T: DebugWithInfcx>> DebugWithInfcx> for Lis } } -impl> Encodable for List { +impl> Encodable for RawList { #[inline] fn encode(&self, s: &mut S) { (**self).encode(s); } } -impl PartialEq for List { +impl PartialEq for RawList { #[inline] - fn eq(&self, other: &List) -> bool { + fn eq(&self, other: &RawList) -> bool { // Pointer equality implies list equality (due to the unique contents // assumption). ptr::eq(self, other) } } -impl Eq for List {} +impl Eq for RawList {} -impl Ord for List +impl Ord for RawList where T: Ord, { - fn cmp(&self, other: &List) -> Ordering { + fn cmp(&self, other: &RawList) -> Ordering { // Pointer equality implies list equality (due to the unique contents // assumption), but the contents must be compared otherwise. if self == other { Ordering::Equal } else { <[T] as Ord>::cmp(&**self, &**other) } } } -impl PartialOrd for List +impl PartialOrd for RawList where T: PartialOrd, { - fn partial_cmp(&self, other: &List) -> Option { + fn partial_cmp(&self, other: &RawList) -> Option { // Pointer equality implies list equality (due to the unique contents // assumption), but the contents must be compared otherwise. if self == other { @@ -186,7 +209,7 @@ where } } -impl Hash for List { +impl Hash for RawList { #[inline] fn hash(&self, s: &mut H) { // Pointer hashing is sufficient (due to the unique contents @@ -195,7 +218,7 @@ impl Hash for List { } } -impl Deref for List { +impl Deref for RawList { type Target = [T]; #[inline(always)] fn deref(&self) -> &[T] { @@ -203,7 +226,7 @@ impl Deref for List { } } -impl AsRef<[T]> for List { +impl AsRef<[T]> for RawList { #[inline(always)] fn as_ref(&self) -> &[T] { let data_ptr = ptr::addr_of!(self.skel.data).cast::(); @@ -215,7 +238,7 @@ impl AsRef<[T]> for List { } } -impl<'a, T: Copy> IntoIterator for &'a List { +impl<'a, H, T: Copy> IntoIterator for &'a RawList { type Item = T; type IntoIter = iter::Copied<<&'a [T] as IntoIterator>::IntoIter>; #[inline(always)] @@ -224,34 +247,51 @@ impl<'a, T: Copy> IntoIterator for &'a List { } } -unsafe impl Sync for List {} +unsafe impl Sync for RawList {} // We need this since `List` uses extern type `OpaqueListContents`. #[cfg(parallel_compiler)] -unsafe impl DynSync for List {} +unsafe impl DynSync for RawList {} // Safety: -// Layouts of `ListSkeleton` and `List` are the same, modulo opaque tail, -// thus aligns of `ListSkeleton` and `List` must be the same. -unsafe impl Aligned for List { - const ALIGN: ptr::Alignment = align_of::>(); +// Layouts of `ListSkeleton` and `RawList` are the same, modulo opaque tail, +// thus aligns of `ListSkeleton` and `RawList` must be the same. +unsafe impl Aligned for RawList { + const ALIGN: ptr::Alignment = align_of::>(); } /// A [`List`] that additionally stores type information inline to speed up /// [`TypeVisitableExt`](super::TypeVisitableExt) operations. -#[repr(C)] -pub struct ListWithCachedTypeInfo { - skel: ListWithCachedTypeInfoSkeleton, - opaque: OpaqueListContents, +pub type ListWithCachedTypeInfo = RawList; + +impl ListWithCachedTypeInfo { + #[inline(always)] + pub fn flags(&self) -> TypeFlags { + self.skel.header.flags + } + + #[inline(always)] + pub fn outer_exclusive_binder(&self) -> DebruijnIndex { + self.skel.header.outer_exclusive_binder + } } +impl_list_empty!(TypeInfo, TypeInfo::empty()); + /// The additional info that is stored in [`ListWithCachedTypeInfo`]. #[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct TypeInfo { flags: TypeFlags, outer_exclusive_binder: DebruijnIndex, } +impl TypeInfo { + const fn empty() -> Self { + Self { flags: TypeFlags::empty(), outer_exclusive_binder: super::INNERMOST } + } +} + impl From for TypeInfo { fn from(computation: FlagComputation) -> TypeInfo { TypeInfo { @@ -260,189 +300,3 @@ impl From for TypeInfo { } } } - -/// This type is similar to [`ListSkeleton`], but for [`ListWithCachedTypeInfo`]. -/// It is used for computing the alignment of a [`ListWithCachedTypeInfo`]. -#[repr(C)] -struct ListWithCachedTypeInfoSkeleton { - info: TypeInfo, - // N.B.: There may be padding between these two fields. We cannot use `List` directly - // here, because it has an unknown alignment which makes computing the amount of padding - // and therefore projecting from `&ListWithCachedTypeInfo` to `&List` impossible. - list: ListSkeleton, -} - -impl ListWithCachedTypeInfo { - #[inline(always)] - pub fn empty<'a>() -> &'a ListWithCachedTypeInfo { - #[repr(align(64))] - struct MaxAlign; - - #[repr(C)] - struct Empty { - info: TypeInfo, - zero: [u8; 2 * mem::align_of::() - mem::size_of::()], - align: MaxAlign, - } - - static EMPTY: Empty = Empty { - info: TypeInfo { flags: TypeFlags::empty(), outer_exclusive_binder: super::INNERMOST }, - zero: [0; 2 * mem::align_of::() - mem::size_of::()], - align: MaxAlign, - }; - - assert!(mem::align_of::() <= mem::align_of::()); - - // The layout of the empty `ListWithCachedTypeInfo` must be one of the following, - // depending on the alignment of `T`: - // - // On 64-bit platforms: - // F = flags (32 bit), B = outer_exclusive_binder (32 bit), LL = len (64 bit) - // align(T) <= 8: FBLL - // align(T) = 16: FB..LL.. - // align(T) = 32: FB......LL...... - // align(T) = 64: FB..............LL.............. - // - // On 32-bit platforms: - // F = flags (32 bit), B = outer_exclusive_binder (32 bit), L = len (32 bit) - // align(T) <= 4: FBL - // align(T) = 8: FBL. - // align(T) = 16: FB..L... - // align(T) = 32: FB......L....... - // align(T) = 64: FB..............L............... - // - // We zero out every possible location of `len` so that `EMPTY` is a valid - // `ListWithCachedTypeInfo` for all `T` with alignment up to 64 bytes. - unsafe { &*(std::ptr::addr_of!(EMPTY) as *const ListWithCachedTypeInfo) } - } - - #[inline] - pub(super) fn from_arena<'tcx>( - arena: &'tcx Arena<'tcx>, - info: TypeInfo, - slice: &[T], - ) -> &'tcx ListWithCachedTypeInfo - where - T: Copy, - { - assert!(!mem::needs_drop::()); - assert!(mem::size_of::() != 0); - assert!(!slice.is_empty()); - - let (list_layout, _offset) = - Layout::new::().extend(Layout::for_value::<[T]>(slice)).unwrap(); - - let (layout, _offset) = Layout::new::().extend(list_layout).unwrap(); - - let mem = arena.dropless.alloc_raw(layout) as *mut ListWithCachedTypeInfo; - unsafe { - // Write the cached type info - ptr::addr_of_mut!((*mem).skel.info).write(info); - - // Write the length - ptr::addr_of_mut!((*mem).skel.list.len).write(slice.len()); - - // Write the elements - ptr::addr_of_mut!((*mem).skel.list.data) - .cast::() - .copy_from_nonoverlapping(slice.as_ptr(), slice.len()); - - &*mem - } - } - - #[inline(always)] - pub fn as_list(&self) -> &List { - self - } - - #[inline(always)] - pub fn flags(&self) -> TypeFlags { - self.skel.info.flags - } - - #[inline(always)] - pub fn outer_exclusive_binder(&self) -> DebruijnIndex { - self.skel.info.outer_exclusive_binder - } -} - -impl Deref for ListWithCachedTypeInfo { - type Target = List; - #[inline(always)] - fn deref(&self) -> &List { - let list_ptr = ptr::addr_of!(self.skel.list) as *const List; - unsafe { &*list_ptr } - } -} - -impl AsRef<[T]> for ListWithCachedTypeInfo { - #[inline(always)] - fn as_ref(&self) -> &[T] { - (&**self).as_ref() - } -} - -impl fmt::Debug for ListWithCachedTypeInfo { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - (**self).fmt(f) - } -} -impl<'tcx, T: DebugWithInfcx>> DebugWithInfcx> - for ListWithCachedTypeInfo -{ - fn fmt>>( - this: WithInfcx<'_, Infcx, &Self>, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - DebugWithInfcx::fmt(this.map(|this| &**this), f) - } -} - -impl> Encodable for ListWithCachedTypeInfo { - #[inline] - fn encode(&self, s: &mut S) { - (**self).encode(s); - } -} - -impl PartialEq for ListWithCachedTypeInfo { - #[inline] - fn eq(&self, other: &ListWithCachedTypeInfo) -> bool { - // Pointer equality implies list equality (due to the unique contents - // assumption). - ptr::eq(self, other) - } -} - -impl Eq for ListWithCachedTypeInfo {} - -impl Hash for ListWithCachedTypeInfo { - #[inline] - fn hash(&self, s: &mut H) { - // Pointer hashing is sufficient (due to the unique contents - // assumption). - ptr::from_ref(self).hash(s) - } -} - -impl<'a, T: Copy> IntoIterator for &'a ListWithCachedTypeInfo { - type Item = T; - type IntoIter = iter::Copied<<&'a [T] as IntoIterator>::IntoIter>; - #[inline(always)] - fn into_iter(self) -> Self::IntoIter { - (**self).into_iter() - } -} - -unsafe impl Sync for ListWithCachedTypeInfo {} - -#[cfg(parallel_compiler)] -unsafe impl DynSync for ListWithCachedTypeInfo {} - -// Safety: -// Layouts of `ListWithCachedTypeInfoSkeleton` and `ListWithCachedTypeInfo` -// are the same, modulo opaque tail, thus their aligns must be the same. -unsafe impl Aligned for ListWithCachedTypeInfo { - const ALIGN: ptr::Alignment = align_of::>(); -}