diff --git a/src/librustc/arena.rs b/src/librustc/arena.rs new file mode 100644 index 00000000000..e9751a23f12 --- /dev/null +++ b/src/librustc/arena.rs @@ -0,0 +1,206 @@ +use arena::{TypedArena, DroplessArena}; +use std::mem; +use std::ptr; +use std::slice; +use std::cell::RefCell; +use std::marker::PhantomData; +use smallvec::SmallVec; + +#[macro_export] +macro_rules! arena_types { + ($macro:path, $args:tt, $tcx:lifetime) => ( + $macro!($args, [ + [] vtable_method: Option<( + rustc::hir::def_id::DefId, + rustc::ty::subst::SubstsRef<$tcx> + )>, + [few] mir_keys: rustc::util::nodemap::DefIdSet, + [decode] specialization_graph: rustc::traits::specialization_graph::Graph, + ], $tcx); + ) +} + +macro_rules! arena_for_type { + ([][$ty:ty]) => { + TypedArena<$ty> + }; + ([few $(, $attrs:ident)*][$ty:ty]) => { + PhantomData<$ty> + }; + ([$ignore:ident $(, $attrs:ident)*]$args:tt) => { + arena_for_type!([$($attrs),*]$args) + }; +} + +macro_rules! declare_arena { + ([], [$($a:tt $name:ident: $ty:ty,)*], $tcx:lifetime) => { + #[derive(Default)] + pub struct Arena<$tcx> { + dropless: DroplessArena, + drop: DropArena, + $($name: arena_for_type!($a[$ty]),)* + } + } +} + +macro_rules! which_arena_for_type { + ([][$arena:expr]) => { + Some($arena) + }; + ([few$(, $attrs:ident)*][$arena:expr]) => { + None + }; + ([$ignore:ident$(, $attrs:ident)*]$args:tt) => { + which_arena_for_type!([$($attrs),*]$args) + }; +} + +macro_rules! impl_arena_allocatable { + ([], [$($a:tt $name:ident: $ty:ty,)*], $tcx:lifetime) => { + $( + impl ArenaAllocatable for $ty {} + unsafe impl<$tcx> ArenaField<$tcx> for $ty { + #[inline] + fn arena<'a>(_arena: &'a Arena<$tcx>) -> Option<&'a TypedArena> { + which_arena_for_type!($a[&_arena.$name]) + } + } + )* + } +} + +arena_types!(declare_arena, [], 'tcx); + +arena_types!(impl_arena_allocatable, [], 'tcx); + +pub trait ArenaAllocatable {} + +impl ArenaAllocatable for T {} + +pub unsafe trait ArenaField<'tcx>: Sized { + /// Returns a specific arena to allocate from. + /// If None is returned, the DropArena will be used. + fn arena<'a>(arena: &'a Arena<'tcx>) -> Option<&'a TypedArena>; +} + +unsafe impl<'tcx, T> ArenaField<'tcx> for T { + #[inline] + default fn arena<'a>(_: &'a Arena<'tcx>) -> Option<&'a TypedArena> { + panic!() + } +} + +impl<'tcx> Arena<'tcx> { + #[inline] + pub fn alloc(&self, value: T) -> &mut T { + if !mem::needs_drop::() { + return self.dropless.alloc(value); + } + match >::arena(self) { + Some(arena) => arena.alloc(value), + None => unsafe { self.drop.alloc(value) }, + } + } + + pub fn alloc_from_iter< + T: ArenaAllocatable, + I: IntoIterator + >( + &'a self, + iter: I + ) -> &'a mut [T] { + if !mem::needs_drop::() { + return self.dropless.alloc_from_iter(iter); + } + match >::arena(self) { + Some(arena) => arena.alloc_from_iter(iter), + None => unsafe { self.drop.alloc_from_iter(iter) }, + } + } +} + +/// Calls the destructor for an object when dropped. +struct DropType { + drop_fn: unsafe fn(*mut u8), + obj: *mut u8, +} + +unsafe fn drop_for_type(to_drop: *mut u8) { + std::ptr::drop_in_place(to_drop as *mut T) +} + +impl Drop for DropType { + fn drop(&mut self) { + unsafe { + (self.drop_fn)(self.obj) + } + } +} + +/// An arena which can be used to allocate any type. +/// Allocating in this arena is unsafe since the type system +/// doesn't know which types it contains. In order to +/// allocate safely, you must store a PhantomData +/// alongside this arena for each type T you allocate. +#[derive(Default)] +struct DropArena { + /// A list of destructors to run when the arena drops. + /// Ordered so `destructors` gets dropped before the arena + /// since its destructor can reference memory in the arena. + destructors: RefCell>, + arena: DroplessArena, +} + +impl DropArena { + #[inline] + unsafe fn alloc(&self, object: T) -> &mut T { + let mem = self.arena.alloc_raw( + mem::size_of::(), + mem::align_of::() + ) as *mut _ as *mut T; + // Write into uninitialized memory. + ptr::write(mem, object); + let result = &mut *mem; + // Record the destructor after doing the allocation as that may panic + // and would cause `object`'s destuctor to run twice if it was recorded before + self.destructors.borrow_mut().push(DropType { + drop_fn: drop_for_type::, + obj: result as *mut T as *mut u8, + }); + result + } + + #[inline] + unsafe fn alloc_from_iter>(&self, iter: I) -> &mut [T] { + let mut vec: SmallVec<[_; 8]> = iter.into_iter().collect(); + if vec.is_empty() { + return &mut []; + } + let len = vec.len(); + + let start_ptr = self.arena.alloc_raw( + len.checked_mul(mem::size_of::()).unwrap(), + mem::align_of::() + ) as *mut _ as *mut T; + + let mut destructors = self.destructors.borrow_mut(); + // Reserve space for the destructors so we can't panic while adding them + destructors.reserve(len); + + // Move the content to the arena by copying it and then forgetting + // the content of the SmallVec + vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); + mem::forget(vec.drain()); + + // Record the destructors after doing the allocation as that may panic + // and would cause `object`'s destuctor to run twice if it was recorded before + for i in 0..len { + destructors.push(DropType { + drop_fn: drop_for_type::, + obj: start_ptr.offset(i as isize) as *mut u8, + }); + } + + slice::from_raw_parts_mut(start_ptr, len) + } +} diff --git a/src/librustc/lib.rs b/src/librustc/lib.rs index 0b75cb6c8a3..c5c2cbfcb89 100644 --- a/src/librustc/lib.rs +++ b/src/librustc/lib.rs @@ -40,6 +40,7 @@ #![cfg_attr(windows, feature(libc))] #![feature(never_type)] #![feature(exhaustive_patterns)] +#![feature(overlapping_marker_traits)] #![feature(extern_types)] #![feature(nll)] #![feature(non_exhaustive)] @@ -103,6 +104,8 @@ #[macro_use] pub mod query; +#[macro_use] +pub mod arena; pub mod cfg; pub mod dep_graph; pub mod hir; diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index cdf553d34e9..d0ad2c90668 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -84,7 +84,7 @@ /// Set of all the `DefId`s in this crate that have MIR associated with /// them. This includes all the body owners, but also things like struct /// constructors. - query mir_keys(_: CrateNum) -> Lrc { + query mir_keys(_: CrateNum) -> &'tcx DefIdSet { desc { "getting a list of all mir_keys" } } @@ -515,7 +515,7 @@ Other { query vtable_methods(key: ty::PolyTraitRef<'tcx>) - -> Lrc)>>> { + -> &'tcx [Option<(DefId, SubstsRef<'tcx>)>] { no_force desc { |tcx| "finding all methods for trait {}", tcx.def_path_str(key.def_id()) } } @@ -538,8 +538,7 @@ query trait_impls_of(key: DefId) -> Lrc { desc { |tcx| "trait impls of `{}`", tcx.def_path_str(key) } } - query specialization_graph_of(_: DefId) - -> Lrc {} + query specialization_graph_of(_: DefId) -> &'tcx specialization_graph::Graph {} query is_object_safe(key: DefId) -> bool { desc { |tcx| "determine object safety of trait `{}`", tcx.def_path_str(key) } } diff --git a/src/librustc/traits/mod.rs b/src/librustc/traits/mod.rs index 78c80b48ee8..d91c08b070a 100644 --- a/src/librustc/traits/mod.rs +++ b/src/librustc/traits/mod.rs @@ -26,7 +26,6 @@ use crate::infer::outlives::env::OutlivesEnvironment; use crate::middle::region; use crate::mir::interpret::ErrorHandled; -use rustc_data_structures::sync::Lrc; use rustc_macros::HashStable; use syntax::ast; use syntax_pos::{Span, DUMMY_SP}; @@ -984,11 +983,11 @@ fn substitute_normalize_and_test_predicates<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx fn vtable_methods<'a, 'tcx>( tcx: TyCtxt<'a, 'tcx, 'tcx>, trait_ref: ty::PolyTraitRef<'tcx>) - -> Lrc)>>> + -> &'tcx [Option<(DefId, SubstsRef<'tcx>)>] { debug!("vtable_methods({:?})", trait_ref); - Lrc::new( + tcx.arena.alloc_from_iter( supertraits(tcx, trait_ref).flat_map(move |trait_ref| { let trait_methods = tcx.associated_items(trait_ref.def_id()) .filter(|item| item.kind == ty::AssociatedKind::Method); @@ -1039,7 +1038,7 @@ fn vtable_methods<'a, 'tcx>( Some((def_id, substs)) }) - }).collect() + }) ) } diff --git a/src/librustc/traits/specialize/mod.rs b/src/librustc/traits/specialize/mod.rs index c576586fcad..384a5862cde 100644 --- a/src/librustc/traits/specialize/mod.rs +++ b/src/librustc/traits/specialize/mod.rs @@ -16,7 +16,6 @@ use crate::lint; use crate::traits::{self, coherence, FutureCompatOverlapErrorKind, ObligationCause, TraitEngine}; use rustc_data_structures::fx::FxHashSet; -use rustc_data_structures::sync::Lrc; use syntax_pos::DUMMY_SP; use crate::traits::select::IntercrateAmbiguityCause; use crate::ty::{self, TyCtxt, TypeFoldable}; @@ -289,7 +288,7 @@ fn fulfill_implication<'a, 'gcx, 'tcx>(infcx: &InferCtxt<'a, 'gcx, 'tcx>, pub(super) fn specialization_graph_provider<'a, 'tcx>( tcx: TyCtxt<'a, 'tcx, 'tcx>, trait_id: DefId, -) -> Lrc { +) -> &'tcx specialization_graph::Graph { let mut sg = specialization_graph::Graph::new(); let mut trait_impls = tcx.all_impls(trait_id); @@ -383,7 +382,7 @@ pub(super) fn specialization_graph_provider<'a, 'tcx>( } } - Lrc::new(sg) + tcx.arena.alloc(sg) } /// Recovers the "impl X for Y" signature from `impl_def_id` and returns it as a diff --git a/src/librustc/traits/specialize/specialization_graph.rs b/src/librustc/traits/specialize/specialization_graph.rs index 561859c7c31..dae1518d722 100644 --- a/src/librustc/traits/specialize/specialization_graph.rs +++ b/src/librustc/traits/specialize/specialization_graph.rs @@ -7,7 +7,6 @@ use crate::traits; use crate::ty::{self, TyCtxt, TypeFoldable}; use crate::ty::fast_reject::{self, SimplifiedType}; -use rustc_data_structures::sync::Lrc; use syntax::ast::Ident; use crate::util::captures::Captures; use crate::util::nodemap::{DefIdMap, FxHashMap}; @@ -439,13 +438,13 @@ pub fn def_id(&self) -> DefId { } } -pub struct Ancestors { +pub struct Ancestors<'tcx> { trait_def_id: DefId, - specialization_graph: Lrc, + specialization_graph: &'tcx Graph, current_source: Option, } -impl Iterator for Ancestors { +impl Iterator for Ancestors<'_> { type Item = Node; fn next(&mut self) -> Option { let cur = self.current_source.take(); @@ -476,7 +475,7 @@ pub fn map U>(self, f: F) -> NodeItem { } } -impl<'a, 'gcx, 'tcx> Ancestors { +impl<'a, 'gcx, 'tcx> Ancestors<'gcx> { /// Search the items from the given ancestors, returning each definition /// with the given name and the given kind. // FIXME(#35870): avoid closures being unexported due to `impl Trait`. @@ -509,10 +508,10 @@ pub fn defs( /// Walk up the specialization ancestors of a given impl, starting with that /// impl itself. -pub fn ancestors(tcx: TyCtxt<'_, '_, '_>, +pub fn ancestors(tcx: TyCtxt<'_, 'tcx, '_>, trait_def_id: DefId, start_from_impl: DefId) - -> Ancestors { + -> Ancestors<'tcx> { let specialization_graph = tcx.specialization_graph_of(trait_def_id); Ancestors { trait_def_id, diff --git a/src/librustc/ty/codec.rs b/src/librustc/ty/codec.rs index e7474345c00..a76cc3dfdec 100644 --- a/src/librustc/ty/codec.rs +++ b/src/librustc/ty/codec.rs @@ -6,6 +6,7 @@ // The functionality in here is shared between persisting to crate metadata and // persisting to incr. comp. caches. +use crate::arena::ArenaAllocatable; use crate::hir::def_id::{DefId, CrateNum}; use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos}; use rustc_data_structures::fx::FxHashMap; @@ -130,6 +131,26 @@ fn positioned_at_shorthand(&self) -> bool { } } +#[inline] +pub fn decode_arena_allocable<'a, 'tcx, D, T: ArenaAllocatable + Decodable>( + decoder: &mut D +) -> Result<&'tcx T, D::Error> + where D: TyDecoder<'a, 'tcx>, + 'tcx: 'a, +{ + Ok(decoder.tcx().arena.alloc(Decodable::decode(decoder)?)) +} + +#[inline] +pub fn decode_arena_allocable_slice<'a, 'tcx, D, T: ArenaAllocatable + Decodable>( + decoder: &mut D +) -> Result<&'tcx [T], D::Error> + where D: TyDecoder<'a, 'tcx>, + 'tcx: 'a, +{ + Ok(decoder.tcx().arena.alloc_from_iter( as Decodable>::decode(decoder)?)) +} + #[inline] pub fn decode_cnum<'a, 'tcx, D>(decoder: &mut D) -> Result where D: TyDecoder<'a, 'tcx>, @@ -273,6 +294,39 @@ macro_rules! __impl_decoder_methods { } } +#[macro_export] +macro_rules! impl_arena_allocatable_decoder { + ([]$args:tt) => {}; + ([decode $(, $attrs:ident)*] + [[$DecoderName:ident [$($typaram:tt),*]], [$name:ident: $ty:ty], $tcx:lifetime]) => { + impl<$($typaram),*> SpecializedDecoder<&$tcx $ty> for $DecoderName<$($typaram),*> { + #[inline] + fn specialized_decode(&mut self) -> Result<&$tcx $ty, Self::Error> { + decode_arena_allocable(self) + } + } + + impl<$($typaram),*> SpecializedDecoder<&$tcx [$ty]> for $DecoderName<$($typaram),*> { + #[inline] + fn specialized_decode(&mut self) -> Result<&$tcx [$ty], Self::Error> { + decode_arena_allocable_slice(self) + } + } + }; + ([$ignore:ident $(, $attrs:ident)*]$args:tt) => { + impl_arena_allocatable_decoder!([$($attrs),*]$args); + }; +} + +#[macro_export] +macro_rules! impl_arena_allocatable_decoders { + ($args:tt, [$($a:tt $name:ident: $ty:ty,)*], $tcx:lifetime) => { + $( + impl_arena_allocatable_decoder!($a [$args, [$name: $ty], $tcx]); + )* + } +} + #[macro_export] macro_rules! implement_ty_decoder { ($DecoderName:ident <$($typaram:tt),*>) => { @@ -322,6 +376,8 @@ fn error(&mut self, err: &str) -> Self::Error { // the caller to pick any lifetime for 'tcx, including 'static, // by using the unspecialized proxies to them. + arena_types!(impl_arena_allocatable_decoders, [$DecoderName [$($typaram),*]], 'tcx); + impl<$($typaram),*> SpecializedDecoder for $DecoderName<$($typaram),*> { fn specialized_decode(&mut self) -> Result { diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index aa5610739fd..7dc4dee3fbf 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -1,5 +1,6 @@ //! Type context book-keeping. +use crate::arena::Arena; use crate::dep_graph::DepGraph; use crate::dep_graph::{self, DepNode, DepConstructor}; use crate::session::Session; @@ -1003,6 +1004,7 @@ fn deref(&self) -> &Self::Target { } pub struct GlobalCtxt<'tcx> { + pub arena: WorkerLocal>, global_arenas: &'tcx WorkerLocal>, global_interners: CtxtInterners<'tcx>, @@ -1262,6 +1264,7 @@ pub fn create_global_ctxt( GlobalCtxt { sess: s, cstore, + arena: WorkerLocal::new(|_| Arena::default()), global_arenas: &arenas.global, global_interners: interners, dep_graph, diff --git a/src/librustc/ty/trait_def.rs b/src/librustc/ty/trait_def.rs index 143b5bf3762..58f21893de1 100644 --- a/src/librustc/ty/trait_def.rs +++ b/src/librustc/ty/trait_def.rs @@ -67,7 +67,7 @@ pub fn new(def_id: DefId, pub fn ancestors(&self, tcx: TyCtxt<'a, 'gcx, 'tcx>, of_impl: DefId) - -> specialization_graph::Ancestors { + -> specialization_graph::Ancestors<'gcx> { specialization_graph::ancestors(tcx, self.def_id, of_impl) } } diff --git a/src/librustc_mir/interpret/traits.rs b/src/librustc_mir/interpret/traits.rs index d76d3a33016..a451f2afb46 100644 --- a/src/librustc_mir/interpret/traits.rs +++ b/src/librustc_mir/interpret/traits.rs @@ -1,4 +1,3 @@ -use rustc_data_structures::sync::Lrc; use rustc::ty::{self, Ty}; use rustc::ty::layout::{Size, Align, LayoutOf}; use rustc::mir::interpret::{Scalar, Pointer, EvalResult, PointerArithmetic}; @@ -35,7 +34,7 @@ pub fn get_vtable( self.tcx.vtable_methods(trait_ref) } else { - Lrc::new(Vec::new()) + &[] }; let layout = self.layout_of(ty)?; diff --git a/src/librustc_mir/transform/mod.rs b/src/librustc_mir/transform/mod.rs index 8df0d72407b..27cb87f5dca 100644 --- a/src/librustc_mir/transform/mod.rs +++ b/src/librustc_mir/transform/mod.rs @@ -8,7 +8,6 @@ use rustc::hir; use rustc::hir::intravisit::{self, Visitor, NestedVisitorMap}; use rustc::util::nodemap::DefIdSet; -use rustc_data_structures::sync::Lrc; use std::borrow::Cow; use syntax::ast; use syntax_pos::Span; @@ -59,7 +58,7 @@ fn is_mir_available<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, def_id: DefId) -> boo /// Finds the full set of `DefId`s within the current crate that have /// MIR associated with them. fn mir_keys<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, krate: CrateNum) - -> Lrc { + -> &'tcx DefIdSet { assert_eq!(krate, LOCAL_CRATE); let mut set = DefIdSet::default(); @@ -94,7 +93,7 @@ fn nested_visit_map<'b>(&'b mut self) -> NestedVisitorMap<'b, 'tcx> { set: &mut set, }.as_deep_visitor()); - Lrc::new(set) + tcx.arena.alloc(set) } fn mir_built<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, def_id: DefId) -> &'tcx Steal> { diff --git a/src/libserialize/serialize.rs b/src/libserialize/serialize.rs index cf948078b08..8ef8c2b4c0a 100644 --- a/src/libserialize/serialize.rs +++ b/src/libserialize/serialize.rs @@ -911,4 +911,5 @@ impl Decodable for T { impl<'a, T: ?Sized + Encodable> UseSpecializedEncodable for &'a T {} impl UseSpecializedEncodable for Box {} impl UseSpecializedDecodable for Box {} - +impl<'a, T: Decodable> UseSpecializedDecodable for &'a T {} +impl<'a, T: Decodable> UseSpecializedDecodable for &'a [T] {}