Auto merge of #59536 - Zoxc:the-arena, r=eddyb

Introduce an arena type which may be used to allocate a list of types with destructors

You can also specify that you want deserializers for `&'tcx [T]` and `&'tcx T` for a type in the list, which will allocate those using the arena.

Based on https://github.com/rust-lang/rust/pull/59517 and https://github.com/rust-lang/rust/pull/59533. Look at the last commit for the interesting changes.

An alternative to https://github.com/rust-lang/rust/pull/56448. cc @michaelwoerister @eddyb

r? @oli-obk

src/librustc/arena.rs

@@ -0,0 +1,206 @@
+use arena::{TypedArena, DroplessArena};
+use std::mem;
+use std::ptr;
+use std::slice;
+use std::cell::RefCell;
+use std::marker::PhantomData;
+use smallvec::SmallVec;
+
+#[macro_export]
+macro_rules! arena_types {
+    ($macro:path, $args:tt, $tcx:lifetime) => (
+        $macro!($args, [
+            [] vtable_method: Option<(
+                rustc::hir::def_id::DefId,
+                rustc::ty::subst::SubstsRef<$tcx>
+            )>,
+            [few] mir_keys: rustc::util::nodemap::DefIdSet,
+            [decode] specialization_graph: rustc::traits::specialization_graph::Graph,
+        ], $tcx);
+    )
+}
+
+macro_rules! arena_for_type {
+    ([][$ty:ty]) => {
+        TypedArena<$ty>
+    };
+    ([few $(, $attrs:ident)*][$ty:ty]) => {
+        PhantomData<$ty>
+    };
+    ([$ignore:ident $(, $attrs:ident)*]$args:tt) => {
+        arena_for_type!([$($attrs),*]$args)
+    };
+}
+
+macro_rules! declare_arena {
+    ([], [$($a:tt $name:ident: $ty:ty,)*], $tcx:lifetime) => {
+        #[derive(Default)]
+        pub struct Arena<$tcx> {
+            dropless: DroplessArena,
+            drop: DropArena,
+            $($name: arena_for_type!($a[$ty]),)*
+        }
+    }
+}
+
+macro_rules! which_arena_for_type {
+    ([][$arena:expr]) => {
+        Some($arena)
+    };
+    ([few$(, $attrs:ident)*][$arena:expr]) => {
+        None
+    };
+    ([$ignore:ident$(, $attrs:ident)*]$args:tt) => {
+        which_arena_for_type!([$($attrs),*]$args)
+    };
+}
+
+macro_rules! impl_arena_allocatable {
+    ([], [$($a:tt $name:ident: $ty:ty,)*], $tcx:lifetime) => {
+        $(
+            impl ArenaAllocatable for $ty {}
+            unsafe impl<$tcx> ArenaField<$tcx> for $ty {
+                #[inline]
+                fn arena<'a>(_arena: &'a Arena<$tcx>) -> Option<&'a TypedArena<Self>> {
+                    which_arena_for_type!($a[&_arena.$name])
+                }
+            }
+        )*
+    }
+}
+
+arena_types!(declare_arena, [], 'tcx);
+
+arena_types!(impl_arena_allocatable, [], 'tcx);
+
+pub trait ArenaAllocatable {}
+
+impl<T: Copy> ArenaAllocatable for T {}
+
+pub unsafe trait ArenaField<'tcx>: Sized {
+    /// Returns a specific arena to allocate from.
+    /// If None is returned, the DropArena will be used.
+    fn arena<'a>(arena: &'a Arena<'tcx>) -> Option<&'a TypedArena<Self>>;
+}
+
+unsafe impl<'tcx, T> ArenaField<'tcx> for T {
+    #[inline]
+    default fn arena<'a>(_: &'a Arena<'tcx>) -> Option<&'a TypedArena<Self>> {
+        panic!()
+    }
+}
+
+impl<'tcx> Arena<'tcx> {
+    #[inline]
+    pub fn alloc<T: ArenaAllocatable>(&self, value: T) -> &mut T {
+        if !mem::needs_drop::<T>() {
+            return self.dropless.alloc(value);
+        }
+        match <T as ArenaField<'tcx>>::arena(self) {
+            Some(arena) => arena.alloc(value),
+            None => unsafe { self.drop.alloc(value) },
+        }
+    }
+
+    pub fn alloc_from_iter<
+        T: ArenaAllocatable,
+        I: IntoIterator<Item = T>
+    >(
+        &'a self,
+        iter: I
+    ) -> &'a mut [T] {
+        if !mem::needs_drop::<T>() {
+            return self.dropless.alloc_from_iter(iter);
+        }
+        match <T as ArenaField<'tcx>>::arena(self) {
+            Some(arena) => arena.alloc_from_iter(iter),
+            None => unsafe { self.drop.alloc_from_iter(iter) },
+        }
+    }
+}
+
+/// Calls the destructor for an object when dropped.
+struct DropType {
+    drop_fn: unsafe fn(*mut u8),
+    obj: *mut u8,
+}
+
+unsafe fn drop_for_type<T>(to_drop: *mut u8) {
+    std::ptr::drop_in_place(to_drop as *mut T)
+}
+
+impl Drop for DropType {
+    fn drop(&mut self) {
+        unsafe {
+            (self.drop_fn)(self.obj)
+        }
+    }
+}
+
+/// An arena which can be used to allocate any type.
+/// Allocating in this arena is unsafe since the type system
+/// doesn't know which types it contains. In order to
+/// allocate safely, you must store a PhantomData<T>
+/// alongside this arena for each type T you allocate.
+#[derive(Default)]
+struct DropArena {
+    /// A list of destructors to run when the arena drops.
+    /// Ordered so `destructors` gets dropped before the arena
+    /// since its destructor can reference memory in the arena.
+    destructors: RefCell<Vec<DropType>>,
+    arena: DroplessArena,
+}
+
+impl DropArena {
+    #[inline]
+    unsafe fn alloc<T>(&self, object: T) -> &mut T {
+        let mem = self.arena.alloc_raw(
+            mem::size_of::<T>(),
+            mem::align_of::<T>()
+        ) as *mut _ as *mut T;
+        // Write into uninitialized memory.
+        ptr::write(mem, object);
+        let result = &mut *mem;
+        // Record the destructor after doing the allocation as that may panic
+        // and would cause `object`'s destuctor to run twice if it was recorded before
+        self.destructors.borrow_mut().push(DropType {
+            drop_fn: drop_for_type::<T>,
+            obj: result as *mut T as *mut u8,
+        });
+        result
+    }
+
+    #[inline]
+    unsafe fn alloc_from_iter<T, I: IntoIterator<Item = T>>(&self, iter: I) -> &mut [T] {
+        let mut vec: SmallVec<[_; 8]> = iter.into_iter().collect();
+        if vec.is_empty() {
+            return &mut [];
+        }
+        let len = vec.len();
+
+        let start_ptr = self.arena.alloc_raw(
+            len.checked_mul(mem::size_of::<T>()).unwrap(),
+            mem::align_of::<T>()
+        ) as *mut _ as *mut T;
+
+        let mut destructors = self.destructors.borrow_mut();
+        // Reserve space for the destructors so we can't panic while adding them
+        destructors.reserve(len);
+
+        // Move the content to the arena by copying it and then forgetting
+        // the content of the SmallVec
+        vec.as_ptr().copy_to_nonoverlapping(start_ptr, len);
+        mem::forget(vec.drain());
+
+        // Record the destructors after doing the allocation as that may panic
+        // and would cause `object`'s destuctor to run twice if it was recorded before
+        for i in 0..len {
+            destructors.push(DropType {
+                drop_fn: drop_for_type::<T>,
+                obj: start_ptr.offset(i as isize) as *mut u8,
+            });
+        }
+
+        slice::from_raw_parts_mut(start_ptr, len)
+    }
+}

src/librustc/lib.rs

@@ -40,6 +40,7 @@
 #![cfg_attr(windows, feature(libc))]
 #![feature(never_type)]
 #![feature(exhaustive_patterns)]
+#![feature(overlapping_marker_traits)]
 #![feature(extern_types)]
 #![feature(nll)]
 #![feature(non_exhaustive)]
@@ -103,6 +104,8 @@
 #[macro_use]
 pub mod query;
 
+#[macro_use]
+pub mod arena;
 pub mod cfg;
 pub mod dep_graph;
 pub mod hir;

src/librustc/query/mod.rs

@@ -84,7 +84,7 @@
         /// Set of all the `DefId`s in this crate that have MIR associated with
         /// them. This includes all the body owners, but also things like struct
         /// constructors.
-        query mir_keys(_: CrateNum) -> Lrc<DefIdSet> {
+        query mir_keys(_: CrateNum) -> &'tcx DefIdSet {
             desc { "getting a list of all mir_keys" }
         }
 
@@ -515,7 +515,7 @@
 
     Other {
         query vtable_methods(key: ty::PolyTraitRef<'tcx>)
-                            -> Lrc<Vec<Option<(DefId, SubstsRef<'tcx>)>>> {
+                            -> &'tcx [Option<(DefId, SubstsRef<'tcx>)>] {
             no_force
             desc { |tcx| "finding all methods for trait {}", tcx.def_path_str(key.def_id()) }
         }
@@ -538,8 +538,7 @@
         query trait_impls_of(key: DefId) -> Lrc<ty::trait_def::TraitImpls> {
             desc { |tcx| "trait impls of `{}`", tcx.def_path_str(key) }
         }
-        query specialization_graph_of(_: DefId)
-            -> Lrc<specialization_graph::Graph> {}
+        query specialization_graph_of(_: DefId) -> &'tcx specialization_graph::Graph {}
         query is_object_safe(key: DefId) -> bool {
             desc { |tcx| "determine object safety of trait `{}`", tcx.def_path_str(key) }
         }

src/librustc/traits/mod.rs

@@ -26,7 +26,6 @@
 use crate::infer::outlives::env::OutlivesEnvironment;
 use crate::middle::region;
 use crate::mir::interpret::ErrorHandled;
-use rustc_data_structures::sync::Lrc;
 use rustc_macros::HashStable;
 use syntax::ast;
 use syntax_pos::{Span, DUMMY_SP};
@@ -984,11 +983,11 @@ fn substitute_normalize_and_test_predicates<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx
 fn vtable_methods<'a, 'tcx>(
     tcx: TyCtxt<'a, 'tcx, 'tcx>,
     trait_ref: ty::PolyTraitRef<'tcx>)
-    -> Lrc<Vec<Option<(DefId, SubstsRef<'tcx>)>>>
+    -> &'tcx [Option<(DefId, SubstsRef<'tcx>)>]
 {
     debug!("vtable_methods({:?})", trait_ref);
 
-    Lrc::new(
+    tcx.arena.alloc_from_iter(
         supertraits(tcx, trait_ref).flat_map(move |trait_ref| {
             let trait_methods = tcx.associated_items(trait_ref.def_id())
                 .filter(|item| item.kind == ty::AssociatedKind::Method);
@@ -1039,7 +1038,7 @@ fn vtable_methods<'a, 'tcx>(
 
                 Some((def_id, substs))
             })
-        }).collect()
+        })
     )
 }
 

src/librustc/traits/specialize/mod.rs

@@ -16,7 +16,6 @@
 use crate::lint;
 use crate::traits::{self, coherence, FutureCompatOverlapErrorKind, ObligationCause, TraitEngine};
 use rustc_data_structures::fx::FxHashSet;
-use rustc_data_structures::sync::Lrc;
 use syntax_pos::DUMMY_SP;
 use crate::traits::select::IntercrateAmbiguityCause;
 use crate::ty::{self, TyCtxt, TypeFoldable};
@@ -289,7 +288,7 @@ fn fulfill_implication<'a, 'gcx, 'tcx>(infcx: &InferCtxt<'a, 'gcx, 'tcx>,
 pub(super) fn specialization_graph_provider<'a, 'tcx>(
     tcx: TyCtxt<'a, 'tcx, 'tcx>,
     trait_id: DefId,
-) -> Lrc<specialization_graph::Graph> {
+) -> &'tcx specialization_graph::Graph {
     let mut sg = specialization_graph::Graph::new();
 
     let mut trait_impls = tcx.all_impls(trait_id);
@@ -383,7 +382,7 @@ pub(super) fn specialization_graph_provider<'a, 'tcx>(
         }
     }
 
-    Lrc::new(sg)
+    tcx.arena.alloc(sg)
 }
 
 /// Recovers the "impl X for Y" signature from `impl_def_id` and returns it as a

src/librustc/traits/specialize/specialization_graph.rs

@@ -7,7 +7,6 @@
 use crate::traits;
 use crate::ty::{self, TyCtxt, TypeFoldable};
 use crate::ty::fast_reject::{self, SimplifiedType};
-use rustc_data_structures::sync::Lrc;
 use syntax::ast::Ident;
 use crate::util::captures::Captures;
 use crate::util::nodemap::{DefIdMap, FxHashMap};
@@ -439,13 +438,13 @@ pub fn def_id(&self) -> DefId {
     }
 }
 
-pub struct Ancestors {
+pub struct Ancestors<'tcx> {
     trait_def_id: DefId,
-    specialization_graph: Lrc<Graph>,
+    specialization_graph: &'tcx Graph,
     current_source: Option<Node>,
 }
 
-impl Iterator for Ancestors {
+impl Iterator for Ancestors<'_> {
     type Item = Node;
     fn next(&mut self) -> Option<Node> {
         let cur = self.current_source.take();
@@ -476,7 +475,7 @@ pub fn map<U, F: FnOnce(T) -> U>(self, f: F) -> NodeItem<U> {
     }
 }
 
-impl<'a, 'gcx, 'tcx> Ancestors {
+impl<'a, 'gcx, 'tcx> Ancestors<'gcx> {
     /// Search the items from the given ancestors, returning each definition
     /// with the given name and the given kind.
     // FIXME(#35870): avoid closures being unexported due to `impl Trait`.
@@ -509,10 +508,10 @@ pub fn defs(
 
 /// Walk up the specialization ancestors of a given impl, starting with that
 /// impl itself.
-pub fn ancestors(tcx: TyCtxt<'_, '_, '_>,
+pub fn ancestors(tcx: TyCtxt<'_, 'tcx, '_>,
                  trait_def_id: DefId,
                  start_from_impl: DefId)
-                 -> Ancestors {
+                 -> Ancestors<'tcx> {
     let specialization_graph = tcx.specialization_graph_of(trait_def_id);
     Ancestors {
         trait_def_id,

src/librustc/ty/codec.rs

@@ -6,6 +6,7 @@
 // The functionality in here is shared between persisting to crate metadata and
 // persisting to incr. comp. caches.
 
+use crate::arena::ArenaAllocatable;
 use crate::hir::def_id::{DefId, CrateNum};
 use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
 use rustc_data_structures::fx::FxHashMap;
@@ -130,6 +131,26 @@ fn positioned_at_shorthand(&self) -> bool {
     }
 }
 
+#[inline]
+pub fn decode_arena_allocable<'a, 'tcx, D, T: ArenaAllocatable + Decodable>(
+    decoder: &mut D
+) -> Result<&'tcx T, D::Error>
+    where D: TyDecoder<'a, 'tcx>,
+          'tcx: 'a,
+{
+    Ok(decoder.tcx().arena.alloc(Decodable::decode(decoder)?))
+}
+
+#[inline]
+pub fn decode_arena_allocable_slice<'a, 'tcx, D, T: ArenaAllocatable + Decodable>(
+    decoder: &mut D
+) -> Result<&'tcx [T], D::Error>
+    where D: TyDecoder<'a, 'tcx>,
+          'tcx: 'a,
+{
+    Ok(decoder.tcx().arena.alloc_from_iter(<Vec<T> as Decodable>::decode(decoder)?))
+}
+
 #[inline]
 pub fn decode_cnum<'a, 'tcx, D>(decoder: &mut D) -> Result<CrateNum, D::Error>
     where D: TyDecoder<'a, 'tcx>,
@@ -273,6 +294,39 @@ macro_rules! __impl_decoder_methods {
     }
 }
 
+#[macro_export]
+macro_rules! impl_arena_allocatable_decoder {
+    ([]$args:tt) => {};
+    ([decode $(, $attrs:ident)*]
+     [[$DecoderName:ident [$($typaram:tt),*]], [$name:ident: $ty:ty], $tcx:lifetime]) => {
+        impl<$($typaram),*> SpecializedDecoder<&$tcx $ty> for $DecoderName<$($typaram),*> {
+            #[inline]
+            fn specialized_decode(&mut self) -> Result<&$tcx $ty, Self::Error> {
+                decode_arena_allocable(self)
+            }
+        }
+
+        impl<$($typaram),*> SpecializedDecoder<&$tcx [$ty]> for $DecoderName<$($typaram),*> {
+            #[inline]
+            fn specialized_decode(&mut self) -> Result<&$tcx [$ty], Self::Error> {
+                decode_arena_allocable_slice(self)
+            }
+        }
+    };
+    ([$ignore:ident $(, $attrs:ident)*]$args:tt) => {
+        impl_arena_allocatable_decoder!([$($attrs),*]$args);
+    };
+}
+
+#[macro_export]
+macro_rules! impl_arena_allocatable_decoders {
+    ($args:tt, [$($a:tt $name:ident: $ty:ty,)*], $tcx:lifetime) => {
+        $(
+            impl_arena_allocatable_decoder!($a [$args, [$name: $ty], $tcx]);
+        )*
+    }
+}
+
 #[macro_export]
 macro_rules! implement_ty_decoder {
     ($DecoderName:ident <$($typaram:tt),*>) => {
@@ -322,6 +376,8 @@ fn error(&mut self, err: &str) -> Self::Error {
             // the caller to pick any lifetime for 'tcx, including 'static,
             // by using the unspecialized proxies to them.
 
+            arena_types!(impl_arena_allocatable_decoders, [$DecoderName [$($typaram),*]], 'tcx);
+
             impl<$($typaram),*> SpecializedDecoder<CrateNum>
             for $DecoderName<$($typaram),*> {
                 fn specialized_decode(&mut self) -> Result<CrateNum, Self::Error> {

src/librustc/ty/context.rs

@@ -1,5 +1,6 @@
 //! Type context book-keeping.
 
+use crate::arena::Arena;
 use crate::dep_graph::DepGraph;
 use crate::dep_graph::{self, DepNode, DepConstructor};
 use crate::session::Session;
@@ -1003,6 +1004,7 @@ fn deref(&self) -> &Self::Target {
 }
 
 pub struct GlobalCtxt<'tcx> {
+    pub arena: WorkerLocal<Arena<'tcx>>,
     global_arenas: &'tcx WorkerLocal<GlobalArenas<'tcx>>,
     global_interners: CtxtInterners<'tcx>,
 
@@ -1262,6 +1264,7 @@ pub fn create_global_ctxt(
         GlobalCtxt {
             sess: s,
             cstore,
+            arena: WorkerLocal::new(|_| Arena::default()),
             global_arenas: &arenas.global,
             global_interners: interners,
             dep_graph,

src/librustc/ty/trait_def.rs

@@ -67,7 +67,7 @@ pub fn new(def_id: DefId,
 
     pub fn ancestors(&self, tcx: TyCtxt<'a, 'gcx, 'tcx>,
                      of_impl: DefId)
-                     -> specialization_graph::Ancestors {
+                     -> specialization_graph::Ancestors<'gcx> {
         specialization_graph::ancestors(tcx, self.def_id, of_impl)
     }
 }

src/librustc_mir/interpret/traits.rs

@@ -1,4 +1,3 @@
-use rustc_data_structures::sync::Lrc;
 use rustc::ty::{self, Ty};
 use rustc::ty::layout::{Size, Align, LayoutOf};
 use rustc::mir::interpret::{Scalar, Pointer, EvalResult, PointerArithmetic};
@@ -35,7 +34,7 @@ pub fn get_vtable(
 
             self.tcx.vtable_methods(trait_ref)
         } else {
-            Lrc::new(Vec::new())
+            &[]
         };
 
         let layout = self.layout_of(ty)?;

src/librustc_mir/transform/mod.rs

@@ -8,7 +8,6 @@
 use rustc::hir;
 use rustc::hir::intravisit::{self, Visitor, NestedVisitorMap};
 use rustc::util::nodemap::DefIdSet;
-use rustc_data_structures::sync::Lrc;
 use std::borrow::Cow;
 use syntax::ast;
 use syntax_pos::Span;
@@ -59,7 +58,7 @@ fn is_mir_available<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, def_id: DefId) -> boo
 /// Finds the full set of `DefId`s within the current crate that have
 /// MIR associated with them.
 fn mir_keys<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, krate: CrateNum)
-                      -> Lrc<DefIdSet> {
+                      -> &'tcx DefIdSet {
     assert_eq!(krate, LOCAL_CRATE);
 
     let mut set = DefIdSet::default();
@@ -94,7 +93,7 @@ fn nested_visit_map<'b>(&'b mut self) -> NestedVisitorMap<'b, 'tcx> {
         set: &mut set,
     }.as_deep_visitor());
 
-    Lrc::new(set)
+    tcx.arena.alloc(set)
 }
 
 fn mir_built<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, def_id: DefId) -> &'tcx Steal<Mir<'tcx>> {

src/libserialize/serialize.rs

@@ -911,4 +911,5 @@ impl<T: UseSpecializedDecodable> Decodable for T {
 impl<'a, T: ?Sized + Encodable> UseSpecializedEncodable for &'a T {}
 impl<T: ?Sized + Encodable> UseSpecializedEncodable for Box<T> {}
 impl<T: Decodable> UseSpecializedDecodable for Box<T> {}
-
+impl<'a, T: Decodable> UseSpecializedDecodable for &'a T {}
+impl<'a, T: Decodable> UseSpecializedDecodable for &'a [T] {}