use std::cell::RefCell; use std::collections::HashSet; use std::rc::Rc; use rustc::ty::{self, layout::Size}; use rustc::hir::{Mutability, MutMutable, MutImmutable}; use crate::{ EvalResult, EvalErrorKind, MiriEvalContext, HelpersEvalContextExt, Evaluator, MutValueVisitor, MemoryKind, MiriMemoryKind, RangeMap, AllocId, Allocation, AllocationExtra, Pointer, Immediate, ImmTy, PlaceTy, MPlaceTy, }; pub type Timestamp = u64; pub type CallId = u64; /// Information about which kind of borrow was used to create the reference this is tagged /// with. #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] pub enum Borrow { /// A unique (mutable) reference. Uniq(Timestamp), /// A shared reference. This is also used by raw pointers, which do not track details /// of how or when they were created, hence the timestamp is optional. /// Shr(Some(_)) does NOT mean that the destination of this reference is frozen; /// that depends on the type! Only those parts outside of an `UnsafeCell` are actually /// frozen. Shr(Option), } impl Borrow { #[inline(always)] pub fn is_shared(self) -> bool { match self { Borrow::Shr(_) => true, _ => false, } } #[inline(always)] pub fn is_unique(self) -> bool { match self { Borrow::Uniq(_) => true, _ => false, } } } impl Default for Borrow { fn default() -> Self { Borrow::Shr(None) } } /// An item in the per-location borrow stack #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] pub enum BorStackItem { /// Indicates the unique reference that may mutate. Uniq(Timestamp), /// Indicates that the location has been shared. Used for raw pointers, but /// also for shared references. The latter *additionally* get frozen /// when there is no `UnsafeCell`. Shr, /// A barrier, tracking the function it belongs to by its index on the call stack FnBarrier(CallId) } /// Extra per-location state #[derive(Clone, Debug)] pub struct Stack { borrows: Vec, // used as a stack; never empty frozen_since: Option, // virtual frozen "item" on top of the stack } impl Stack { #[inline(always)] pub fn is_frozen(&self) -> bool { self.frozen_since.is_some() } } /// What kind of reference is being used? #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] pub enum RefKind { /// &mut Unique, /// & without interior mutability Frozen, /// * (raw pointer) or & to `UnsafeCell` Raw, } /// What kind of access is being performed? #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] pub enum AccessKind { Read, Write, Dealloc, } /// Extra global state in the memory, available to the memory access hooks #[derive(Debug)] pub struct BarrierTracking { next_id: CallId, active_calls: HashSet, } pub type MemoryState = Rc>; impl Default for BarrierTracking { fn default() -> Self { BarrierTracking { next_id: 0, active_calls: HashSet::default(), } } } impl BarrierTracking { pub fn new_call(&mut self) -> CallId { let id = self.next_id; trace!("new_call: Assigning ID {}", id); self.active_calls.insert(id); self.next_id += 1; id } pub fn end_call(&mut self, id: CallId) { assert!(self.active_calls.remove(&id)); } fn is_active(&self, id: CallId) -> bool { self.active_calls.contains(&id) } } /// Extra global machine state #[derive(Clone, Debug)] pub struct State { clock: Timestamp } impl Default for State { fn default() -> Self { State { clock: 0 } } } impl State { fn increment_clock(&mut self) -> Timestamp { let val = self.clock; self.clock = val + 1; val } } /// Extra per-allocation state #[derive(Clone, Debug)] pub struct Stacks { // Even reading memory can have effects on the stack, so we need a `RefCell` here. stacks: RefCell>, barrier_tracking: MemoryState, } /// Core per-location operations: deref, access, create. /// We need to make at least the following things true: /// /// U1: After creating a Uniq, it is at the top (+unfrozen). /// U2: If the top is Uniq (+unfrozen), accesses must be through that Uniq or pop it. /// U3: If an access (deref sufficient?) happens with a Uniq, it requires the Uniq to be in the stack. /// /// F1: After creating a &, the parts outside `UnsafeCell` are frozen. /// F2: If a write access happens, it unfreezes. /// F3: If an access (well, a deref) happens with an & outside `UnsafeCell`, it requires the location to still be frozen. impl<'tcx> Stack { /// Deref `bor`: Check if the location is frozen and the tag in the stack. /// This dos *not* constitute an access! "Deref" refers to the `*` operator /// in Rust, and includs cases like `&*x` or `(*x).foo` where no or only part /// of the memory actually gets accessed. Also we cannot know if we are /// going to read or write. /// Returns the index of the item we matched, `None` if it was the frozen one. /// `kind` indicates which kind of reference is being dereferenced. fn deref( &self, bor: Borrow, kind: RefKind, ) -> Result, String> { // Exclude unique ref with frozen tag. if let (RefKind::Unique, Borrow::Shr(Some(_))) = (kind, bor) { return Err(format!("Encountered mutable reference with frozen tag ({:?})", bor)); } // Checks related to freezing match bor { Borrow::Shr(Some(bor_t)) if kind == RefKind::Frozen => { // We need the location to be frozen. This ensures F3. let frozen = self.frozen_since.map_or(false, |itm_t| itm_t <= bor_t); return if frozen { Ok(None) } else { Err(format!("Location is not frozen long enough")) } } Borrow::Shr(_) if self.frozen_since.is_some() => { return Ok(None) // Shared deref to frozen location, looking good } _ => {} // Not sufficient, go on looking. } // If we got here, we have to look for our item in the stack. for (idx, &itm) in self.borrows.iter().enumerate().rev() { match (itm, bor) { (BorStackItem::Uniq(itm_t), Borrow::Uniq(bor_t)) if itm_t == bor_t => { // Found matching unique item. This satisfies U3. return Ok(Some(idx)) } (BorStackItem::Shr, Borrow::Shr(_)) => { // Found matching shared/raw item. return Ok(Some(idx)) } // Go on looking. We ignore barriers! When an `&mut` and an `&` alias, // dereferencing the `&` is still possible (to reborrow), but doing // an access is not. _ => {} } } // If we got here, we did not find our item. We have to error to satisfy U3. Err(format!("Borrow being dereferenced ({:?}) does not exist on the stack", bor)) } /// Perform an actual memory access using `bor`. We do not know any types here /// or whether things should be frozen, but we *do* know if this is reading /// or writing. fn access( &mut self, bor: Borrow, kind: AccessKind, barrier_tracking: &BarrierTracking, ) -> EvalResult<'tcx> { // Check if we can match the frozen "item". // Not possible on writes! if self.is_frozen() { if kind == AccessKind::Read { // When we are frozen, we just accept all reads. No harm in this. // The deref already checked that `Uniq` items are in the stack, and that // the location is frozen if it should be. return Ok(()); } trace!("access: Unfreezing"); } // Unfreeze on writes. This ensures F2. self.frozen_since = None; // Pop the stack until we have something matching. while let Some(&itm) = self.borrows.last() { match (itm, bor) { (BorStackItem::FnBarrier(call), _) if barrier_tracking.is_active(call) => { return err!(MachineError(format!( "Stopping looking for borrow being accessed ({:?}) because of barrier ({})", bor, call ))) } (BorStackItem::Uniq(itm_t), Borrow::Uniq(bor_t)) if itm_t == bor_t => { // Found matching unique item. Continue after the match. } (BorStackItem::Shr, _) if kind == AccessKind::Read => { // When reading, everything can use a shared item! // We do not want to do this when writing: Writing to an `&mut` // should reaffirm its exclusivity (i.e., make sure it is // on top of the stack). Continue after the match. } (BorStackItem::Shr, Borrow::Shr(_)) => { // Found matching shared item. Continue after the match. } _ => { // Pop this, go on. This ensures U2. let itm = self.borrows.pop().unwrap(); trace!("access: Popping {:?}", itm); continue } } // If we got here, we found a matching item. Congratulations! // However, we are not done yet: If this access is deallocating, we must make sure // there are no active barriers remaining on the stack. if kind == AccessKind::Dealloc { for &itm in self.borrows.iter().rev() { match itm { BorStackItem::FnBarrier(call) if barrier_tracking.is_active(call) => { return err!(MachineError(format!( "Deallocating with active barrier ({})", call ))) } _ => {}, } } } // NOW we are done. return Ok(()) } // If we got here, we did not find our item. err!(MachineError(format!( "Borrow being accessed ({:?}) does not exist on the stack", bor ))) } /// Initiate `bor`; mostly this means pushing. /// This operation cannot fail; it is up to the caller to ensure that the precondition /// is met: We cannot push `Uniq` onto frozen stacks. /// `kind` indicates which kind of reference is being created. fn create(&mut self, bor: Borrow, kind: RefKind) { if self.frozen_since.is_some() { // A frozen location? Possible if we create a barrier, then push again. assert!(bor.is_shared(), "We should never try creating a unique borrow for a frozen stack"); trace!("create: Not doing anything on frozen location"); return; } // First, push. We do this even if we will later freeze, because we // will allow mutation of shared data at the expense of unfreezing. let itm = match bor { Borrow::Uniq(t) => BorStackItem::Uniq(t), Borrow::Shr(_) => BorStackItem::Shr, }; if *self.borrows.last().unwrap() == itm { // This is just an optimization, no functional change: Avoid stacking // multiple `Shr` on top of each other. assert!(bor.is_shared()); trace!("create: Sharing a shared location is a NOP"); } else { // This ensures U1. trace!("create: Pushing {:?}", itm); self.borrows.push(itm); } // Then, maybe freeze. This is part 2 of ensuring F1. if kind == RefKind::Frozen { let bor_t = match bor { Borrow::Shr(Some(t)) => t, _ => bug!("Creating illegal borrow {:?} for frozen ref", bor), }; trace!("create: Freezing"); self.frozen_since = Some(bor_t); } } /// Add a barrier fn barrier(&mut self, call: CallId) { let itm = BorStackItem::FnBarrier(call); if *self.borrows.last().unwrap() == itm { // This is just an optimization, no functional change: Avoid stacking // multiple identical barriers on top of each other. // This can happen when a function receives several shared references // that overlap. trace!("barrier: Avoiding redundant extra barrier"); } else { trace!("barrier: Pushing barrier for call {}", call); self.borrows.push(itm); } } } /// Higher-level per-location operations: deref, access, reborrow. impl<'tcx> Stacks { /// Check that this stack is fine with being dereferenced fn deref( &self, ptr: Pointer, size: Size, kind: RefKind, ) -> EvalResult<'tcx> { trace!("deref for tag {:?} as {:?}: {:?}, size {}", ptr.tag, kind, ptr, size.bytes()); let stacks = self.stacks.borrow(); for stack in stacks.iter(ptr.offset, size) { stack.deref(ptr.tag, kind).map_err(EvalErrorKind::MachineError)?; } Ok(()) } /// `ptr` got used, reflect that in the stack. fn access( &self, ptr: Pointer, size: Size, kind: AccessKind, ) -> EvalResult<'tcx> { trace!("{:?} access of tag {:?}: {:?}, size {}", kind, ptr.tag, ptr, size.bytes()); // Even reads can have a side-effect, by invalidating other references. // This is fundamentally necessary since `&mut` asserts that there // are no accesses through other references, not even reads. let barrier_tracking = self.barrier_tracking.borrow(); let mut stacks = self.stacks.borrow_mut(); for stack in stacks.iter_mut(ptr.offset, size) { stack.access(ptr.tag, kind, &*barrier_tracking)?; } Ok(()) } /// Reborrow the given pointer to the new tag for the given kind of reference. /// This works on `&self` because we might encounter references to constant memory. fn reborrow( &self, ptr: Pointer, size: Size, mut barrier: Option, new_bor: Borrow, new_kind: RefKind, ) -> EvalResult<'tcx> { assert_eq!(new_bor.is_unique(), new_kind == RefKind::Unique); trace!("reborrow for tag {:?} to {:?} as {:?}: {:?}, size {}", ptr.tag, new_bor, new_kind, ptr, size.bytes()); if new_kind == RefKind::Raw { // No barrier for raw, including `&UnsafeCell`. They can rightfully // alias with `&mut`. // FIXME: This means that the `dereferencable` attribute on non-frozen shared // references is incorrect! They are dereferencable when the function is // called, but might become non-dereferencable during the course of execution. // Also see [1], [2]. // // [1]: , // [2]: barrier = None; } let barrier_tracking = self.barrier_tracking.borrow(); let mut stacks = self.stacks.borrow_mut(); for stack in stacks.iter_mut(ptr.offset, size) { // Access source `ptr`, create new ref. let ptr_idx = stack.deref(ptr.tag, new_kind).map_err(EvalErrorKind::MachineError)?; // If we can deref the new tag already, and if that tag lives higher on // the stack than the one we come from, just use that. // IOW, we check if `new_bor` *already* is "derived from" `ptr.tag`. // This also checks frozenness, if required. let bor_redundant = barrier.is_none() && match (ptr_idx, stack.deref(new_bor, new_kind)) { // If the new borrow works with the frozen item, or else if it lives // above the old one in the stack, our job here is done. (_, Ok(None)) => true, (Some(ptr_idx), Ok(Some(new_idx))) if new_idx >= ptr_idx => true, // Otherwise we need to create a new borrow. _ => false, }; if bor_redundant { assert!(new_bor.is_shared(), "A unique reborrow can never be redundant"); trace!("reborrow is redundant"); continue; } // We need to do some actual work. let access_kind = if new_kind == RefKind::Unique { AccessKind::Write } else { AccessKind::Read }; stack.access(ptr.tag, access_kind, &*barrier_tracking)?; if let Some(call) = barrier { stack.barrier(call); } stack.create(new_bor, new_kind); } Ok(()) } } /// Hooks and glue impl AllocationExtra for Stacks { #[inline(always)] fn memory_allocated<'tcx>(size: Size, extra: &MemoryState) -> Self { let stack = Stack { borrows: vec![BorStackItem::Shr], frozen_since: None, }; Stacks { stacks: RefCell::new(RangeMap::new(size, stack)), barrier_tracking: Rc::clone(extra), } } #[inline(always)] fn memory_read<'tcx>( alloc: &Allocation, ptr: Pointer, size: Size, ) -> EvalResult<'tcx> { alloc.extra.access(ptr, size, AccessKind::Read) } #[inline(always)] fn memory_written<'tcx>( alloc: &mut Allocation, ptr: Pointer, size: Size, ) -> EvalResult<'tcx> { alloc.extra.access(ptr, size, AccessKind::Write) } #[inline(always)] fn memory_deallocated<'tcx>( alloc: &mut Allocation, ptr: Pointer, size: Size, ) -> EvalResult<'tcx> { alloc.extra.access(ptr, size, AccessKind::Dealloc) } } impl<'tcx> Stacks { /// Pushes the first item to the stacks. pub(crate) fn first_item( &mut self, itm: BorStackItem, size: Size ) { for stack in self.stacks.get_mut().iter_mut(Size::ZERO, size) { assert!(stack.borrows.len() == 1); assert_eq!(stack.borrows.pop().unwrap(), BorStackItem::Shr); stack.borrows.push(itm); } } } pub trait EvalContextExt<'tcx> { fn ptr_dereference( &self, place: MPlaceTy<'tcx, Borrow>, size: Size, mutability: Option, ) -> EvalResult<'tcx>; fn tag_new_allocation( &mut self, id: AllocId, kind: MemoryKind, ) -> Borrow; /// Reborrow the given place, returning the newly tagged ptr to it. fn reborrow( &mut self, place: MPlaceTy<'tcx, Borrow>, size: Size, fn_barrier: bool, new_bor: Borrow ) -> EvalResult<'tcx>; /// Retag an indidual pointer, returning the retagged version. fn retag_reference( &mut self, ptr: ImmTy<'tcx, Borrow>, mutbl: Mutability, fn_barrier: bool, two_phase: bool, ) -> EvalResult<'tcx, Immediate>; fn retag( &mut self, fn_entry: bool, two_phase: bool, place: PlaceTy<'tcx, Borrow> ) -> EvalResult<'tcx>; fn escape_to_raw( &mut self, place: MPlaceTy<'tcx, Borrow>, size: Size, ) -> EvalResult<'tcx>; } impl<'a, 'mir, 'tcx> EvalContextExt<'tcx> for MiriEvalContext<'a, 'mir, 'tcx> { fn tag_new_allocation( &mut self, id: AllocId, kind: MemoryKind, ) -> Borrow { let time = match kind { MemoryKind::Stack => { // New unique borrow. This `Uniq` is not accessible by the program, // so it will only ever be used when using the local directly (i.e., // not through a pointer). IOW, whenever we directly use a local this will pop // everything else off the stack, invalidating all previous pointers // and, in particular, *all* raw pointers. This subsumes the explicit // `reset` which the blog post [1] says to perform when accessing a local. // // [1] https://www.ralfj.de/blog/2018/08/07/stacked-borrows.html self.machine.stacked_borrows.increment_clock() } _ => { // Nothing to do for everything else return Borrow::default() } }; // Make this the active borrow for this allocation let alloc = self.memory_mut().get_mut(id).expect("This is a new allocation, it must still exist"); let size = Size::from_bytes(alloc.bytes.len() as u64); alloc.extra.first_item(BorStackItem::Uniq(time), size); Borrow::Uniq(time) } /// Called for value-to-place conversion. `mutability` is `None` for raw pointers. /// /// Note that this does NOT mean that all this memory will actually get accessed/referenced! /// We could be in the middle of `&(*var).1`. fn ptr_dereference( &self, place: MPlaceTy<'tcx, Borrow>, size: Size, mutability: Option, ) -> EvalResult<'tcx> { trace!("ptr_dereference: Accessing {} reference for {:?} (pointee {})", if let Some(mutability) = mutability { format!("{:?}", mutability) } else { format!("raw") }, place.ptr, place.layout.ty); let ptr = place.ptr.to_ptr()?; if mutability.is_none() { // No further checks on raw derefs -- only the access itself will be checked. return Ok(()); } // Get the allocation let alloc = self.memory().get(ptr.alloc_id)?; alloc.check_bounds(self, ptr, size)?; // If we got here, we do some checking, *but* we leave the tag unchanged. if let Borrow::Shr(Some(_)) = ptr.tag { assert_eq!(mutability, Some(MutImmutable)); // We need a frozen-sensitive check self.visit_freeze_sensitive(place, size, |cur_ptr, size, frozen| { let kind = if frozen { RefKind::Frozen } else { RefKind::Raw }; alloc.extra.deref(cur_ptr, size, kind) })?; } else { // Just treat this as one big chunk let kind = if mutability == Some(MutMutable) { RefKind::Unique } else { RefKind::Raw }; alloc.extra.deref(ptr, size, kind)?; } // All is good Ok(()) } /// The given place may henceforth be accessed through raw pointers. #[inline(always)] fn escape_to_raw( &mut self, place: MPlaceTy<'tcx, Borrow>, size: Size, ) -> EvalResult<'tcx> { self.reborrow(place, size, /*fn_barrier*/ false, Borrow::default())?; Ok(()) } fn reborrow( &mut self, place: MPlaceTy<'tcx, Borrow>, size: Size, fn_barrier: bool, new_bor: Borrow ) -> EvalResult<'tcx> { let ptr = place.ptr.to_ptr()?; let barrier = if fn_barrier { Some(self.frame().extra) } else { None }; trace!("reborrow: Creating new reference for {:?} (pointee {}): {:?}", ptr, place.layout.ty, new_bor); // Get the allocation. It might not be mutable, so we cannot use `get_mut`. let alloc = self.memory().get(ptr.alloc_id)?; alloc.check_bounds(self, ptr, size)?; // Update the stacks. if let Borrow::Shr(Some(_)) = new_bor { // Reference that cares about freezing. We need a frozen-sensitive reborrow. self.visit_freeze_sensitive(place, size, |cur_ptr, size, frozen| { let kind = if frozen { RefKind::Frozen } else { RefKind::Raw }; alloc.extra.reborrow(cur_ptr, size, barrier, new_bor, kind) })?; } else { // Just treat this as one big chunk. let kind = if new_bor.is_unique() { RefKind::Unique } else { RefKind::Raw }; alloc.extra.reborrow(ptr, size, barrier, new_bor, kind)?; } Ok(()) } fn retag_reference( &mut self, val: ImmTy<'tcx, Borrow>, mutbl: Mutability, fn_barrier: bool, two_phase: bool, ) -> EvalResult<'tcx, Immediate> { // We want a place for where the ptr *points to*, so we get one. let place = self.ref_to_mplace(val)?; let size = self.size_and_align_of_mplace(place)? .map(|(size, _)| size) .unwrap_or_else(|| place.layout.size); if size == Size::ZERO { // Nothing to do for ZSTs. return Ok(*val); } // Compute new borrow. let time = self.machine.stacked_borrows.increment_clock(); let new_bor = match mutbl { MutMutable => Borrow::Uniq(time), MutImmutable => Borrow::Shr(Some(time)), }; // Reborrow. self.reborrow(place, size, fn_barrier, new_bor)?; let new_place = place.with_tag(new_bor); // Handle two-phase borrows. if two_phase { // We immediately share it, to allow read accesses let two_phase_time = self.machine.stacked_borrows.increment_clock(); let two_phase_bor = Borrow::Shr(Some(two_phase_time)); self.reborrow(new_place, size, /*fn_barrier*/false, two_phase_bor)?; } // Return new ptr. Ok(new_place.to_ref()) } fn retag( &mut self, fn_entry: bool, two_phase: bool, place: PlaceTy<'tcx, Borrow> ) -> EvalResult<'tcx> { // Determine mutability and whether to add a barrier. // Cannot use `builtin_deref` because that reports *immutable* for `Box`, // making it useless. fn qualify(ty: ty::Ty<'_>, fn_entry: bool) -> Option<(Mutability, bool)> { match ty.sty { // References are simple ty::Ref(_, _, mutbl) => Some((mutbl, fn_entry)), // Boxes do not get a barrier: Barriers reflect that references outlive the call // they were passed in to; that's just not the case for boxes. ty::Adt(..) if ty.is_box() => Some((MutMutable, false)), _ => None, } } // We need a visitor to visit all references. However, that requires // a `MemPlace`, so we have a fast path for reference types that // avoids allocating. if let Some((mutbl, barrier)) = qualify(place.layout.ty, fn_entry) { // fast path let val = self.read_immediate(self.place_to_op(place)?)?; let val = self.retag_reference(val, mutbl, barrier, two_phase)?; self.write_immediate(val, place)?; return Ok(()); } let place = self.force_allocation(place)?; let mut visitor = RetagVisitor { ecx: self, fn_entry, two_phase }; visitor.visit_value(place)?; // The actual visitor struct RetagVisitor<'ecx, 'a, 'mir, 'tcx> { ecx: &'ecx mut MiriEvalContext<'a, 'mir, 'tcx>, fn_entry: bool, two_phase: bool, } impl<'ecx, 'a, 'mir, 'tcx> MutValueVisitor<'a, 'mir, 'tcx, Evaluator<'tcx>> for RetagVisitor<'ecx, 'a, 'mir, 'tcx> { type V = MPlaceTy<'tcx, Borrow>; #[inline(always)] fn ecx(&mut self) -> &mut MiriEvalContext<'a, 'mir, 'tcx> { &mut self.ecx } // Primitives of reference type, that is the one thing we are interested in. fn visit_primitive(&mut self, place: MPlaceTy<'tcx, Borrow>) -> EvalResult<'tcx> { // Cannot use `builtin_deref` because that reports *immutable* for `Box`, // making it useless. if let Some((mutbl, barrier)) = qualify(place.layout.ty, self.fn_entry) { let val = self.ecx.read_immediate(place.into())?; let val = self.ecx.retag_reference(val, mutbl, barrier, self.two_phase)?; self.ecx.write_immediate(val, place.into())?; } Ok(()) } } Ok(()) } }