diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs index 946546263ea..b81a4bfe149 100644 --- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs @@ -1,4 +1,4 @@ -use super::{CompileTimeEvalContext, CompileTimeInterpreter, ConstEvalErr, MemoryExtra}; +use super::{CompileTimeEvalContext, CompileTimeInterpreter, ConstEvalErr}; use crate::interpret::eval_nullary_intrinsic; use crate::interpret::{ intern_const_alloc_recursive, Allocation, ConstAlloc, ConstValue, CtfeValidationMode, GlobalId, @@ -100,8 +100,7 @@ pub(super) fn mk_eval_cx<'mir, 'tcx>( tcx, root_span, param_env, - CompileTimeInterpreter::new(tcx.const_eval_limit()), - MemoryExtra { can_access_statics }, + CompileTimeInterpreter::new(tcx.const_eval_limit(), can_access_statics), ) } @@ -285,10 +284,9 @@ pub fn eval_to_allocation_raw_provider<'tcx>( tcx, tcx.def_span(def.did), key.param_env, - CompileTimeInterpreter::new(tcx.const_eval_limit()), // Statics (and promoteds inside statics) may access other statics, because unlike consts // they do not have to behave "as if" they were evaluated at runtime. - MemoryExtra { can_access_statics: is_static }, + CompileTimeInterpreter::new(tcx.const_eval_limit(), /*can_access_statics:*/ is_static), ); let res = ecx.load_mir(cid.instance.def, cid.promoted); diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 2b58c1e8233..5aae4a90960 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -93,10 +93,7 @@ pub struct CompileTimeInterpreter<'mir, 'tcx> { /// The virtual call stack. pub(crate) stack: Vec>, -} -#[derive(Copy, Clone, Debug)] -pub struct MemoryExtra { /// We need to make sure consts never point to anything mutable, even recursively. That is /// relied on for pattern matching on consts with references. /// To achieve this, two pieces have to work together: @@ -107,8 +104,12 @@ pub struct MemoryExtra { } impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> { - pub(super) fn new(const_eval_limit: Limit) -> Self { - CompileTimeInterpreter { steps_remaining: const_eval_limit.0, stack: Vec::new() } + pub(super) fn new(const_eval_limit: Limit, can_access_statics: bool) -> Self { + CompileTimeInterpreter { + steps_remaining: const_eval_limit.0, + stack: Vec::new(), + can_access_statics, + } } } @@ -233,8 +234,6 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, type MemoryKind = MemoryKind; - type MemoryExtra = MemoryExtra; - const PANIC_ON_ALLOC_FAIL: bool = false; // will be raised as a proper error fn load_mir( @@ -345,7 +344,7 @@ fn call_intrinsic( Err(err) => throw_ub_format!("align has to be a power of 2, {}", err), }; - let ptr = ecx.memory.allocate( + let ptr = ecx.allocate_ptr( Size::from_bytes(size as u64), align, interpret::MemoryKind::Machine(MemoryKind::Heap), @@ -365,14 +364,14 @@ fn call_intrinsic( // If an allocation is created in an another const, // we don't deallocate it. - let (alloc_id, _, _) = ecx.memory.ptr_get_alloc(ptr)?; + let (alloc_id, _, _) = ecx.ptr_get_alloc_id(ptr)?; let is_allocated_in_another_const = matches!( ecx.tcx.get_global_alloc(alloc_id), Some(interpret::GlobalAlloc::Memory(_)) ); if !is_allocated_in_another_const { - ecx.memory.deallocate( + ecx.deallocate_ptr( ptr, Some((size, align)), interpret::MemoryKind::Machine(MemoryKind::Heap), @@ -472,7 +471,7 @@ fn stack_mut<'a>( } fn before_access_global( - memory_extra: &MemoryExtra, + machine: &Self, alloc_id: AllocId, alloc: ConstAllocation<'tcx>, static_def_id: Option, @@ -488,7 +487,7 @@ fn before_access_global( } } else { // Read access. These are usually allowed, with some exceptions. - if memory_extra.can_access_statics { + if machine.can_access_statics { // Machine configuration allows us read from anything (e.g., `static` initializer). Ok(()) } else if static_def_id.is_some() { diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs index 5be0a183cf6..a244b79ed07 100644 --- a/compiler/rustc_const_eval/src/interpret/cast.rs +++ b/compiler/rustc_const_eval/src/interpret/cast.rs @@ -56,7 +56,7 @@ pub fn cast( ) .ok_or_else(|| err_inval!(TooGeneric))?; - let fn_ptr = self.memory.create_fn_alloc(FnVal::Instance(instance)); + let fn_ptr = self.create_fn_alloc_ptr(FnVal::Instance(instance)); self.write_pointer(fn_ptr, dest)?; } _ => span_bug!(self.cur_span(), "reify fn pointer on {:?}", src.layout.ty), @@ -87,7 +87,7 @@ pub fn cast( substs, ty::ClosureKind::FnOnce, ); - let fn_ptr = self.memory.create_fn_alloc(FnVal::Instance(instance)); + let fn_ptr = self.create_fn_alloc_ptr(FnVal::Instance(instance)); self.write_pointer(fn_ptr, dest)?; } _ => span_bug!(self.cur_span(), "closure fn pointer on {:?}", src.layout.ty), @@ -153,8 +153,8 @@ pub fn misc_cast( return Ok(**src); } else { // Casting the metadata away from a fat ptr. - assert_eq!(src.layout.size, 2 * self.memory.pointer_size()); - assert_eq!(dest_layout.size, self.memory.pointer_size()); + assert_eq!(src.layout.size, 2 * self.pointer_size()); + assert_eq!(dest_layout.size, self.pointer_size()); assert!(src.layout.ty.is_unsafe_ptr()); return match **src { Immediate::ScalarPair(data, _) => Ok(data.into()), diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index 1b8186b5aad..a2ea0f516bf 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -22,9 +22,9 @@ use rustc_target::abi::{call::FnAbi, Align, HasDataLayout, Size, TargetDataLayout}; use super::{ - AllocCheck, AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, - MemPlace, MemPlaceMeta, Memory, MemoryKind, Operand, Place, PlaceTy, Pointer, - PointerArithmetic, Provenance, Scalar, ScalarMaybeUninit, StackPopJump, + AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace, + MemPlaceMeta, Memory, MemoryKind, Operand, Place, PlaceTy, PointerArithmetic, Provenance, + Scalar, ScalarMaybeUninit, StackPopJump, }; use crate::transform::validate::equal_up_to_regions; @@ -413,13 +413,12 @@ pub fn new( root_span: Span, param_env: ty::ParamEnv<'tcx>, machine: M, - memory_extra: M::MemoryExtra, ) -> Self { InterpCx { machine, tcx: tcx.at(root_span), param_env, - memory: Memory::new(tcx, memory_extra), + memory: Memory::new(), recursion_limit: tcx.recursion_limit(), } } @@ -433,49 +432,6 @@ pub fn cur_span(&self) -> Span { .map_or(self.tcx.span, |f| f.current_span()) } - #[inline(always)] - pub fn scalar_to_ptr(&self, scalar: Scalar) -> Pointer> { - self.memory.scalar_to_ptr(scalar) - } - - /// Test if this value might be null. - /// If the machine does not support ptr-to-int casts, this is conservative. - pub fn scalar_may_be_null(&self, scalar: Scalar) -> bool { - match scalar.try_to_int() { - Ok(int) => int.is_null(), - Err(_) => { - // Can only happen during CTFE. - let ptr = self.scalar_to_ptr(scalar); - match self.memory.ptr_try_get_alloc(ptr) { - Ok((alloc_id, offset, _)) => { - let (size, _align) = self - .memory - .get_size_and_align(alloc_id, AllocCheck::MaybeDead) - .expect("alloc info with MaybeDead cannot fail"); - // If the pointer is out-of-bounds, it may be null. - // Note that one-past-the-end (offset == size) is still inbounds, and never null. - offset > size - } - Err(_offset) => bug!("a non-int scalar is always a pointer"), - } - } - } - } - - /// Call this to turn untagged "global" pointers (obtained via `tcx`) into - /// the machine pointer to the allocation. Must never be used - /// for any other pointers, nor for TLS statics. - /// - /// Using the resulting pointer represents a *direct* access to that memory - /// (e.g. by directly using a `static`), - /// as opposed to access through a pointer that was created by the program. - /// - /// This function can fail only if `ptr` points to an `extern static`. - #[inline(always)] - pub fn global_base_pointer(&self, ptr: Pointer) -> InterpResult<'tcx, Pointer> { - self.memory.global_base_pointer(ptr) - } - #[inline(always)] pub(crate) fn stack(&self) -> &[Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>] { M::stack(self) @@ -949,9 +905,9 @@ fn deallocate_local(&mut self, local: LocalValue) -> InterpResult trace!( "deallocating local {:?}: {:?}", local, - self.memory.dump_alloc(ptr.provenance.unwrap().get_alloc_id()) + self.dump_alloc(ptr.provenance.unwrap().get_alloc_id()) ); - self.memory.deallocate(ptr, None, MemoryKind::Stack)?; + self.deallocate_ptr(ptr, None, MemoryKind::Stack)?; }; Ok(()) } @@ -1057,7 +1013,7 @@ fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { } } - write!(fmt, ": {:?}", self.ecx.memory.dump_allocs(allocs)) + write!(fmt, ": {:?}", self.ecx.dump_allocs(allocs)) } Place::Ptr(mplace) => match mplace.ptr.provenance.map(Provenance::get_alloc_id) { Some(alloc_id) => write!( @@ -1065,7 +1021,7 @@ fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { "by align({}) ref {:?}: {:?}", mplace.align.bytes(), mplace.ptr, - self.ecx.memory.dump_alloc(alloc_id) + self.ecx.dump_alloc(alloc_id) ), ptr => write!(fmt, " integral by ref: {:?}", ptr), }, diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index c80d7d71787..a956a478cef 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -318,7 +318,7 @@ pub fn emulate_intrinsic( // exception from the exception.) // This is the dual to the special exception for offset-by-0 // in the inbounds pointer offset operation (see `ptr_offset_inbounds` below). - match (self.memory.ptr_try_get_alloc(a), self.memory.ptr_try_get_alloc(b)) { + match (self.ptr_try_get_alloc_id(a), self.ptr_try_get_alloc_id(b)) { (Err(a), Err(b)) if a == b && a != 0 => { // Both are the same non-null integer. self.write_scalar(Scalar::from_machine_isize(0, self), dest)?; @@ -335,13 +335,13 @@ pub fn emulate_intrinsic( ); } // And they must both be valid for zero-sized accesses ("in-bounds or one past the end"). - self.memory.check_ptr_access_align( + self.check_ptr_access_align( a, Size::ZERO, Align::ONE, CheckInAllocMsg::OffsetFromTest, )?; - self.memory.check_ptr_access_align( + self.check_ptr_access_align( b, Size::ZERO, Align::ONE, @@ -545,7 +545,7 @@ pub fn ptr_offset_inbounds( let min_ptr = if offset_bytes >= 0 { ptr } else { offset_ptr }; let size = offset_bytes.unsigned_abs(); // This call handles checking for integer/null pointers. - self.memory.check_ptr_access_align( + self.check_ptr_access_align( min_ptr, Size::from_bytes(size), Align::ONE, @@ -577,7 +577,7 @@ pub(crate) fn copy_intrinsic( let src = self.read_pointer(&src)?; let dst = self.read_pointer(&dst)?; - self.memory.copy(src, align, dst, align, size, nonoverlapping) + self.mem_copy(src, align, dst, align, size, nonoverlapping) } pub(crate) fn write_bytes_intrinsic( @@ -600,7 +600,7 @@ pub(crate) fn write_bytes_intrinsic( .ok_or_else(|| err_ub_format!("overflow computing total size of `write_bytes`"))?; let bytes = std::iter::repeat(byte).take(len.bytes_usize()); - self.memory.write_bytes(dst, bytes) + self.write_bytes_ptr(dst, bytes) } pub(crate) fn raw_eq_intrinsic( @@ -613,8 +613,8 @@ pub(crate) fn raw_eq_intrinsic( let lhs = self.read_pointer(lhs)?; let rhs = self.read_pointer(rhs)?; - let lhs_bytes = self.memory.read_bytes(lhs, layout.size)?; - let rhs_bytes = self.memory.read_bytes(rhs, layout.size)?; + let lhs_bytes = self.read_bytes_ptr(lhs, layout.size)?; + let rhs_bytes = self.read_bytes_ptr(rhs, layout.size)?; Ok(Scalar::from_bool(lhs_bytes == rhs_bytes)) } } diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index 7d75c84d108..39aec33819c 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -14,7 +14,7 @@ use super::{ AllocId, AllocRange, Allocation, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult, - LocalValue, MemPlace, Memory, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar, + LocalValue, MemPlace, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar, StackPopUnwind, }; @@ -96,11 +96,6 @@ pub trait Machine<'mir, 'tcx>: Sized { /// Extra data stored in every call frame. type FrameExtra; - /// Extra data stored in memory. A reference to this is available when `AllocExtra` - /// gets initialized, so you can e.g., have an `Rc` here if there is global state you - /// need access to in the `AllocExtra` hooks. - type MemoryExtra; - /// Extra data stored in every allocation. type AllocExtra: Debug + Clone + 'static; @@ -123,11 +118,11 @@ pub trait Machine<'mir, 'tcx>: Sized { const PANIC_ON_ALLOC_FAIL: bool; /// Whether memory accesses should be alignment-checked. - fn enforce_alignment(memory_extra: &Self::MemoryExtra) -> bool; + fn enforce_alignment(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; /// Whether, when checking alignment, we should `force_int` and thus support /// custom alignment logic based on whatever the integer address happens to be. - fn force_int_for_alignment_check(memory_extra: &Self::MemoryExtra) -> bool; + fn force_int_for_alignment_check(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; /// Whether to enforce the validity invariant fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; @@ -251,7 +246,7 @@ fn before_terminator(_ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx /// `def_id` is `Some` if this is the "lazy" allocation of a static. #[inline] fn before_access_global( - _memory_extra: &Self::MemoryExtra, + _machine: &Self, _alloc_id: AllocId, _allocation: ConstAllocation<'tcx>, _static_def_id: Option, @@ -270,7 +265,7 @@ fn thread_local_static_base_pointer( /// Return the root pointer for the given `extern static`. fn extern_static_base_pointer( - mem: &Memory<'mir, 'tcx, Self>, + ecx: &InterpCx<'mir, 'tcx, Self>, def_id: DefId, ) -> InterpResult<'tcx, Pointer>; @@ -279,19 +274,19 @@ fn extern_static_base_pointer( /// /// Not called on `extern` or thread-local statics (those use the methods above). fn tag_alloc_base_pointer( - mem: &Memory<'mir, 'tcx, Self>, + ecx: &InterpCx<'mir, 'tcx, Self>, ptr: Pointer, ) -> Pointer; /// "Int-to-pointer cast" fn ptr_from_addr( - mem: &Memory<'mir, 'tcx, Self>, + ecx: &InterpCx<'mir, 'tcx, Self>, addr: u64, ) -> Pointer>; /// Convert a pointer with provenance into an allocation-offset pair. fn ptr_get_alloc( - mem: &Memory<'mir, 'tcx, Self>, + ecx: &InterpCx<'mir, 'tcx, Self>, ptr: Pointer, ) -> (AllocId, Size); @@ -309,7 +304,7 @@ fn ptr_get_alloc( /// cache the result. (This relies on `AllocMap::get_or` being able to add the /// owned allocation to the map even when the map is shared.) fn init_allocation_extra<'b>( - mem: &Memory<'mir, 'tcx, Self>, + ecx: &InterpCx<'mir, 'tcx, Self>, id: AllocId, alloc: Cow<'b, Allocation>, kind: Option>, @@ -322,7 +317,7 @@ fn init_allocation_extra<'b>( /// need to mutate. #[inline(always)] fn memory_read( - _memory_extra: &Self::MemoryExtra, + _machine: &Self, _alloc_extra: &Self::AllocExtra, _tag: Self::PointerTag, _range: AllocRange, @@ -333,7 +328,7 @@ fn memory_read( /// Hook for performing extra checks on a memory write access. #[inline(always)] fn memory_written( - _memory_extra: &mut Self::MemoryExtra, + _machine: &mut Self, _alloc_extra: &mut Self::AllocExtra, _tag: Self::PointerTag, _range: AllocRange, @@ -344,7 +339,7 @@ fn memory_written( /// Hook for performing extra operations on a memory deallocation. #[inline(always)] fn memory_deallocated( - _memory_extra: &mut Self::MemoryExtra, + _machine: &mut Self, _alloc_extra: &mut Self::AllocExtra, _tag: Self::PointerTag, _range: AllocRange, @@ -408,14 +403,14 @@ fn after_stack_pop( type FrameExtra = (); #[inline(always)] - fn enforce_alignment(_memory_extra: &Self::MemoryExtra) -> bool { + fn enforce_alignment(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool { // We do not check for alignment to avoid having to carry an `Align` // in `ConstValue::ByRef`. false } #[inline(always)] - fn force_int_for_alignment_check(_memory_extra: &Self::MemoryExtra) -> bool { + fn force_int_for_alignment_check(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool { // We do not support `force_int`. false } @@ -444,7 +439,7 @@ fn call_extra_fn( #[inline(always)] fn init_allocation_extra<'b>( - _mem: &Memory<$mir, $tcx, Self>, + _ecx: &InterpCx<$mir, $tcx, Self>, _id: AllocId, alloc: Cow<'b, Allocation>, _kind: Option>, @@ -454,28 +449,28 @@ fn init_allocation_extra<'b>( } fn extern_static_base_pointer( - mem: &Memory<$mir, $tcx, Self>, + ecx: &InterpCx<$mir, $tcx, Self>, def_id: DefId, ) -> InterpResult<$tcx, Pointer> { // Use the `AllocId` associated with the `DefId`. Any actual *access* will fail. - Ok(Pointer::new(mem.tcx.create_static_alloc(def_id), Size::ZERO)) + Ok(Pointer::new(ecx.tcx.create_static_alloc(def_id), Size::ZERO)) } #[inline(always)] fn tag_alloc_base_pointer( - _mem: &Memory<$mir, $tcx, Self>, + _ecx: &InterpCx<$mir, $tcx, Self>, ptr: Pointer, ) -> Pointer { ptr } #[inline(always)] - fn ptr_from_addr(_mem: &Memory<$mir, $tcx, Self>, addr: u64) -> Pointer> { + fn ptr_from_addr(_ecx: &InterpCx<$mir, $tcx, Self>, addr: u64) -> Pointer> { Pointer::new(None, Size::from_bytes(addr)) } #[inline(always)] - fn ptr_get_alloc(_mem: &Memory<$mir, $tcx, Self>, ptr: Pointer) -> (AllocId, Size) { + fn ptr_get_alloc(_ecx: &InterpCx<$mir, $tcx, Self>, ptr: Pointer) -> (AllocId, Size) { // We know `offset` is relative to the allocation, so we can use `into_parts`. let (alloc_id, offset) = ptr.into_parts(); (alloc_id, offset) diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index 4a235809119..fcfcd5d8989 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -17,10 +17,10 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_middle::mir::display_allocation; use rustc_middle::ty::{Instance, ParamEnv, TyCtxt}; -use rustc_target::abi::{Align, HasDataLayout, Size, TargetDataLayout}; +use rustc_target::abi::{Align, HasDataLayout, Size}; use super::{ - alloc_range, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg, GlobalAlloc, + alloc_range, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg, GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak, Pointer, PointerArithmetic, Provenance, Scalar, ScalarMaybeUninit, }; @@ -108,19 +108,6 @@ pub struct Memory<'mir, 'tcx, M: Machine<'mir, 'tcx>> { /// that do not exist any more. // FIXME: this should not be public, but interning currently needs access to it pub(super) dead_alloc_map: FxHashMap, - - /// Extra data added by the machine. - pub extra: M::MemoryExtra, - - /// Lets us implement `HasDataLayout`, which is awfully convenient. - pub tcx: TyCtxt<'tcx>, -} - -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> HasDataLayout for Memory<'mir, 'tcx, M> { - #[inline] - fn data_layout(&self) -> &TargetDataLayout { - &self.tcx.data_layout - } } /// A reference to some allocation that was already bounds-checked for the given region @@ -142,16 +129,21 @@ pub struct AllocRefMut<'a, 'tcx, Tag, Extra> { } impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { - pub fn new(tcx: TyCtxt<'tcx>, extra: M::MemoryExtra) -> Self { + pub fn new() -> Self { Memory { alloc_map: M::MemoryMap::default(), extra_fn_ptr_map: FxHashMap::default(), dead_alloc_map: FxHashMap::default(), - extra, - tcx, } } + /// This is used by [priroda](https://github.com/oli-obk/priroda) + pub fn alloc_map(&self) -> &M::MemoryMap { + &self.alloc_map + } +} + +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Call this to turn untagged "global" pointers (obtained via `tcx`) into /// the machine pointer to the allocation. Must never be used /// for any other pointers, nor for TLS statics. @@ -182,7 +174,7 @@ pub fn global_base_pointer( Ok(M::tag_alloc_base_pointer(self, Pointer::new(alloc_id, offset))) } - pub fn create_fn_alloc( + pub fn create_fn_alloc_ptr( &mut self, fn_val: FnVal<'tcx, M::ExtraFnVal>, ) -> Pointer { @@ -191,7 +183,7 @@ pub fn create_fn_alloc( FnVal::Other(extra) => { // FIXME(RalfJung): Should we have a cache here? let id = self.tcx.reserve_alloc_id(); - let old = self.extra_fn_ptr_map.insert(id, extra); + let old = self.memory.extra_fn_ptr_map.insert(id, extra); assert!(old.is_none()); id } @@ -201,17 +193,17 @@ pub fn create_fn_alloc( self.global_base_pointer(Pointer::from(id)).unwrap() } - pub fn allocate( + pub fn allocate_ptr( &mut self, size: Size, align: Align, kind: MemoryKind, ) -> InterpResult<'static, Pointer> { let alloc = Allocation::uninit(size, align, M::PANIC_ON_ALLOC_FAIL)?; - Ok(self.allocate_with(alloc, kind)) + Ok(self.allocate_raw_ptr(alloc, kind)) } - pub fn allocate_bytes( + pub fn allocate_bytes_ptr( &mut self, bytes: &[u8], align: Align, @@ -219,10 +211,10 @@ pub fn allocate_bytes( mutability: Mutability, ) -> Pointer { let alloc = Allocation::from_bytes(bytes, align, mutability); - self.allocate_with(alloc, kind) + self.allocate_raw_ptr(alloc, kind) } - pub fn allocate_with( + pub fn allocate_raw_ptr( &mut self, alloc: Allocation, kind: MemoryKind, @@ -234,11 +226,11 @@ pub fn allocate_with( "dynamically allocating global memory" ); let alloc = M::init_allocation_extra(self, id, Cow::Owned(alloc), Some(kind)); - self.alloc_map.insert(id, (kind, alloc.into_owned())); + self.memory.alloc_map.insert(id, (kind, alloc.into_owned())); M::tag_alloc_base_pointer(self, Pointer::from(id)) } - pub fn reallocate( + pub fn reallocate_ptr( &mut self, ptr: Pointer>, old_size_and_align: Option<(Size, Align)>, @@ -246,7 +238,7 @@ pub fn reallocate( new_align: Align, kind: MemoryKind, ) -> InterpResult<'tcx, Pointer> { - let (alloc_id, offset, ptr) = self.ptr_get_alloc(ptr)?; + let (alloc_id, offset, ptr) = self.ptr_get_alloc_id(ptr)?; if offset.bytes() != 0 { throw_ub_format!( "reallocating {:?} which does not point to the beginning of an object", @@ -256,13 +248,13 @@ pub fn reallocate( // For simplicities' sake, we implement reallocate as "alloc, copy, dealloc". // This happens so rarely, the perf advantage is outweighed by the maintenance cost. - let new_ptr = self.allocate(new_size, new_align, kind)?; + let new_ptr = self.allocate_ptr(new_size, new_align, kind)?; let old_size = match old_size_and_align { Some((size, _align)) => size, - None => self.get_raw(alloc_id)?.size(), + None => self.get_alloc_raw(alloc_id)?.size(), }; // This will also call the access hooks. - self.copy( + self.mem_copy( ptr.into(), Align::ONE, new_ptr.into(), @@ -270,19 +262,19 @@ pub fn reallocate( old_size.min(new_size), /*nonoverlapping*/ true, )?; - self.deallocate(ptr.into(), old_size_and_align, kind)?; + self.deallocate_ptr(ptr.into(), old_size_and_align, kind)?; Ok(new_ptr) } #[instrument(skip(self), level = "debug")] - pub fn deallocate( + pub fn deallocate_ptr( &mut self, ptr: Pointer>, old_size_and_align: Option<(Size, Align)>, kind: MemoryKind, ) -> InterpResult<'tcx> { - let (alloc_id, offset, ptr) = self.ptr_get_alloc(ptr)?; + let (alloc_id, offset, ptr) = self.ptr_get_alloc_id(ptr)?; trace!("deallocating: {}", alloc_id); if offset.bytes() != 0 { @@ -292,7 +284,7 @@ pub fn deallocate( ); } - let Some((alloc_kind, mut alloc)) = self.alloc_map.remove(&alloc_id) else { + let Some((alloc_kind, mut alloc)) = self.memory.alloc_map.remove(&alloc_id) else { // Deallocating global memory -- always an error return Err(match self.tcx.get_global_alloc(alloc_id) { Some(GlobalAlloc::Function(..)) => { @@ -335,14 +327,14 @@ pub fn deallocate( // Let the machine take some extra action let size = alloc.size(); M::memory_deallocated( - &mut self.extra, + &mut self.machine, &mut alloc.extra, ptr.provenance, alloc_range(Size::ZERO, size), )?; // Don't forget to remember size and align of this now-dead allocation - let old = self.dead_alloc_map.insert(alloc_id, (size, alloc.align)); + let old = self.memory.dead_alloc_map.insert(alloc_id, (size, alloc.align)); if old.is_some() { bug!("Nothing can be deallocated twice"); } @@ -358,7 +350,7 @@ fn get_ptr_access( size: Size, align: Align, ) -> InterpResult<'tcx, Option<(AllocId, Size, Pointer)>> { - let align = M::enforce_alignment(&self.extra).then_some(align); + let align = M::enforce_alignment(&self).then_some(align); self.check_and_deref_ptr( ptr, size, @@ -366,7 +358,7 @@ fn get_ptr_access( CheckInAllocMsg::MemoryAccessTest, |alloc_id, offset, ptr| { let (size, align) = - self.get_size_and_align(alloc_id, AllocCheck::Dereferenceable)?; + self.get_alloc_size_and_align(alloc_id, AllocCheck::Dereferenceable)?; Ok((size, align, (alloc_id, offset, ptr))) }, ) @@ -392,7 +384,7 @@ pub fn check_ptr_access_align( | CheckInAllocMsg::OffsetFromTest | CheckInAllocMsg::InboundsTest => AllocCheck::Live, }; - let (size, align) = self.get_size_and_align(alloc_id, check)?; + let (size, align) = self.get_alloc_size_and_align(alloc_id, check)?; Ok((size, align, ())) })?; Ok(()) @@ -427,7 +419,7 @@ fn check_offset_align(offset: u64, align: Align) -> InterpResult<'static> { } } - Ok(match self.ptr_try_get_alloc(ptr) { + Ok(match self.ptr_try_get_alloc_id(ptr) { Err(addr) => { // We couldn't get a proper allocation. This is only okay if the access size is 0, // and the address is not null. @@ -456,7 +448,7 @@ fn check_offset_align(offset: u64, align: Align) -> InterpResult<'static> { // Test align. Check this last; if both bounds and alignment are violated // we want the error to be about the bounds. if let Some(align) = align { - if M::force_int_for_alignment_check(&self.extra) { + if M::force_int_for_alignment_check(self) { let addr = Scalar::from_pointer(ptr, &self.tcx) .to_machine_usize(&self.tcx) .expect("ptr-to-int cast for align check should never fail"); @@ -479,7 +471,7 @@ fn check_offset_align(offset: u64, align: Align) -> InterpResult<'static> { } /// Allocation accessors -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Helper function to obtain a global (tcx) allocation. /// This attempts to return a reference to an existing allocation if /// one can be found in `tcx`. That, however, is only possible if `tcx` and @@ -517,7 +509,7 @@ fn get_global_alloc( (self.tcx.eval_static_initializer(def_id)?, Some(def_id)) } }; - M::before_access_global(&self.extra, id, alloc, def_id, is_write)?; + M::before_access_global(&self.machine, id, alloc, def_id, is_write)?; // We got tcx memory. Let the machine initialize its "extra" stuff. let alloc = M::init_allocation_extra( self, @@ -530,7 +522,7 @@ fn get_global_alloc( /// Gives raw access to the `Allocation`, without bounds or alignment checks. /// The caller is responsible for calling the access hooks! - fn get_raw( + fn get_alloc_raw( &self, id: AllocId, ) -> InterpResult<'tcx, &Allocation> { @@ -538,7 +530,7 @@ fn get_raw( // ways of "erroring": An actual error, or because we got a reference from // `get_global_alloc` that we can actually use directly without inserting anything anywhere. // So the error type is `InterpResult<'tcx, &Allocation>`. - let a = self.alloc_map.get_or(id, || { + let a = self.memory.alloc_map.get_or(id, || { let alloc = self.get_global_alloc(id, /*is_write*/ false).map_err(Err)?; match alloc { Cow::Borrowed(alloc) => { @@ -564,27 +556,27 @@ fn get_raw( } /// "Safe" (bounds and align-checked) allocation access. - pub fn get<'a>( + pub fn get_ptr_alloc<'a>( &'a self, ptr: Pointer>, size: Size, align: Align, ) -> InterpResult<'tcx, Option>> { - let align = M::enforce_alignment(&self.extra).then_some(align); + let align = M::enforce_alignment(self).then_some(align); let ptr_and_alloc = self.check_and_deref_ptr( ptr, size, align, CheckInAllocMsg::MemoryAccessTest, |alloc_id, offset, ptr| { - let alloc = self.get_raw(alloc_id)?; + let alloc = self.get_alloc_raw(alloc_id)?; Ok((alloc.size(), alloc.align, (alloc_id, offset, ptr, alloc))) }, )?; if let Some((alloc_id, offset, ptr, alloc)) = ptr_and_alloc { let range = alloc_range(offset, size); - M::memory_read(&self.extra, &alloc.extra, ptr.provenance, range)?; - Ok(Some(AllocRef { alloc, range, tcx: self.tcx, alloc_id })) + M::memory_read(&self.machine, &alloc.extra, ptr.provenance, range)?; + Ok(Some(AllocRef { alloc, range, tcx: *self.tcx, alloc_id })) } else { // Even in this branch we have to be sure that we actually access the allocation, in // order to ensure that `static FOO: Type = FOO;` causes a cycle error instead of @@ -596,7 +588,7 @@ pub fn get<'a>( /// Return the `extra` field of the given allocation. pub fn get_alloc_extra<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, &'a M::AllocExtra> { - Ok(&self.get_raw(id)?.extra) + Ok(&self.get_alloc_raw(id)?.extra) } /// Gives raw mutable access to the `Allocation`, without bounds or alignment checks. @@ -604,16 +596,15 @@ pub fn get_alloc_extra<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, &'a M::A /// /// Also returns a ptr to `self.extra` so that the caller can use it in parallel with the /// allocation. - fn get_raw_mut( + fn get_alloc_raw_mut( &mut self, id: AllocId, - ) -> InterpResult<'tcx, (&mut Allocation, &mut M::MemoryExtra)> - { + ) -> InterpResult<'tcx, (&mut Allocation, &mut M)> { // We have "NLL problem case #3" here, which cannot be worked around without loss of // efficiency even for the common case where the key is in the map. // // (Cannot use `get_mut_or` since `get_global_alloc` needs `&self`.) - if self.alloc_map.get_mut(id).is_none() { + if self.memory.alloc_map.get_mut(id).is_none() { // Slow path. // Allocation not found locally, go look global. let alloc = self.get_global_alloc(id, /*is_write*/ true)?; @@ -621,18 +612,18 @@ fn get_raw_mut( "I got a global allocation that I have to copy but the machine does \ not expect that to happen", ); - self.alloc_map.insert(id, (MemoryKind::Machine(kind), alloc.into_owned())); + self.memory.alloc_map.insert(id, (MemoryKind::Machine(kind), alloc.into_owned())); } - let (_kind, alloc) = self.alloc_map.get_mut(id).unwrap(); + let (_kind, alloc) = self.memory.alloc_map.get_mut(id).unwrap(); if alloc.mutability == Mutability::Not { throw_ub!(WriteToReadOnly(id)) } - Ok((alloc, &mut self.extra)) + Ok((alloc, &mut self.machine)) } /// "Safe" (bounds and align-checked) allocation access. - pub fn get_mut<'a>( + pub fn get_ptr_alloc_mut<'a>( &'a mut self, ptr: Pointer>, size: Size, @@ -640,12 +631,12 @@ pub fn get_mut<'a>( ) -> InterpResult<'tcx, Option>> { let parts = self.get_ptr_access(ptr, size, align)?; if let Some((alloc_id, offset, ptr)) = parts { - let tcx = self.tcx; + let tcx = *self.tcx; // FIXME: can we somehow avoid looking up the allocation twice here? // We cannot call `get_raw_mut` inside `check_and_deref_ptr` as that would duplicate `&mut self`. - let (alloc, extra) = self.get_raw_mut(alloc_id)?; + let (alloc, machine) = self.get_alloc_raw_mut(alloc_id)?; let range = alloc_range(offset, size); - M::memory_written(extra, &mut alloc.extra, ptr.provenance, range)?; + M::memory_written(machine, &mut alloc.extra, ptr.provenance, range)?; Ok(Some(AllocRefMut { alloc, range, tcx, alloc_id })) } else { Ok(None) @@ -656,16 +647,16 @@ pub fn get_mut<'a>( pub fn get_alloc_extra_mut<'a>( &'a mut self, id: AllocId, - ) -> InterpResult<'tcx, (&'a mut M::AllocExtra, &'a mut M::MemoryExtra)> { - let (alloc, memory_extra) = self.get_raw_mut(id)?; - Ok((&mut alloc.extra, memory_extra)) + ) -> InterpResult<'tcx, (&'a mut M::AllocExtra, &'a mut M)> { + let (alloc, machine) = self.get_alloc_raw_mut(id)?; + Ok((&mut alloc.extra, machine)) } /// Obtain the size and alignment of an allocation, even if that allocation has /// been deallocated. /// /// If `liveness` is `AllocCheck::MaybeDead`, this function always returns `Ok`. - pub fn get_size_and_align( + pub fn get_alloc_size_and_align( &self, id: AllocId, liveness: AllocCheck, @@ -674,7 +665,7 @@ pub fn get_size_and_align( // Don't use `self.get_raw` here as that will // a) cause cycles in case `id` refers to a static // b) duplicate a global's allocation in miri - if let Some((_, alloc)) = self.alloc_map.get(id) { + if let Some((_, alloc)) = self.memory.alloc_map.get(id) { return Ok((alloc.size(), alloc.align)); } @@ -713,6 +704,7 @@ pub fn get_size_and_align( // Deallocated pointers are allowed, we should be able to find // them in the map. Ok(*self + .memory .dead_alloc_map .get(&id) .expect("deallocated pointers should all be recorded in `dead_alloc_map`")) @@ -724,7 +716,7 @@ pub fn get_size_and_align( } fn get_fn_alloc(&self, id: AllocId) -> Option> { - if let Some(extra) = self.extra_fn_ptr_map.get(&id) { + if let Some(extra) = self.memory.extra_fn_ptr_map.get(&id) { Some(FnVal::Other(*extra)) } else { match self.tcx.get_global_alloc(id) { @@ -734,12 +726,12 @@ fn get_fn_alloc(&self, id: AllocId) -> Option> { } } - pub fn get_fn( + pub fn get_ptr_fn( &self, ptr: Pointer>, ) -> InterpResult<'tcx, FnVal<'tcx, M::ExtraFnVal>> { trace!("get_fn({:?})", ptr); - let (alloc_id, offset, _ptr) = self.ptr_get_alloc(ptr)?; + let (alloc_id, offset, _ptr) = self.ptr_get_alloc_id(ptr)?; if offset.bytes() != 0 { throw_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset))) } @@ -747,8 +739,8 @@ pub fn get_fn( .ok_or_else(|| err_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset))).into()) } - pub fn mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx> { - self.get_raw_mut(id)?.0.mutability = Mutability::Not; + pub fn alloc_mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx> { + self.get_alloc_raw_mut(id)?.0.mutability = Mutability::Not; Ok(()) } @@ -765,7 +757,7 @@ pub fn dump_alloc<'a>(&'a self, id: AllocId) -> DumpAllocs<'a, 'mir, 'tcx, M> { pub fn dump_allocs<'a>(&'a self, mut allocs: Vec) -> DumpAllocs<'a, 'mir, 'tcx, M> { allocs.sort(); allocs.dedup(); - DumpAllocs { mem: self, allocs } + DumpAllocs { ecx: self, allocs } } /// Print leaked memory. Allocations reachable from `static_roots` or a `Global` allocation @@ -775,14 +767,15 @@ pub fn leak_report(&self, static_roots: &[AllocId]) -> usize { let reachable = { let mut reachable = FxHashSet::default(); let global_kind = M::GLOBAL_KIND.map(MemoryKind::Machine); - let mut todo: Vec<_> = self.alloc_map.filter_map_collect(move |&id, &(kind, _)| { - if Some(kind) == global_kind { Some(id) } else { None } - }); + let mut todo: Vec<_> = + self.memory.alloc_map.filter_map_collect(move |&id, &(kind, _)| { + if Some(kind) == global_kind { Some(id) } else { None } + }); todo.extend(static_roots); while let Some(id) = todo.pop() { if reachable.insert(id) { // This is a new allocation, add its relocations to `todo`. - if let Some((_, alloc)) = self.alloc_map.get(id) { + if let Some((_, alloc)) = self.memory.alloc_map.get(id) { todo.extend(alloc.relocations().values().map(|tag| tag.get_alloc_id())); } } @@ -791,7 +784,7 @@ pub fn leak_report(&self, static_roots: &[AllocId]) -> usize { }; // All allocations that are *not* `reachable` and *not* `may_leak` are considered leaking. - let leaks: Vec<_> = self.alloc_map.filter_map_collect(|&id, &(kind, _)| { + let leaks: Vec<_> = self.memory.alloc_map.filter_map_collect(|&id, &(kind, _)| { if kind.may_leak() || reachable.contains(&id) { None } else { Some(id) } }); let n = leaks.len(); @@ -800,17 +793,12 @@ pub fn leak_report(&self, static_roots: &[AllocId]) -> usize { } n } - - /// This is used by [priroda](https://github.com/oli-obk/priroda) - pub fn alloc_map(&self) -> &M::MemoryMap { - &self.alloc_map - } } #[doc(hidden)] /// There's no way to use this directly, it's just a helper struct for the `dump_alloc(s)` methods. pub struct DumpAllocs<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> { - mem: &'a Memory<'mir, 'tcx, M>, + ecx: &'a InterpCx<'mir, 'tcx, M>, allocs: Vec, } @@ -840,25 +828,25 @@ fn write_allocation_track_relocs<'tcx, Tag: Provenance, Extra>( } write!(fmt, "{}", id)?; - match self.mem.alloc_map.get(id) { + match self.ecx.memory.alloc_map.get(id) { Some(&(kind, ref alloc)) => { // normal alloc write!(fmt, " ({}, ", kind)?; write_allocation_track_relocs( &mut *fmt, - self.mem.tcx, + *self.ecx.tcx, &mut allocs_to_print, alloc, )?; } None => { // global alloc - match self.mem.tcx.get_global_alloc(id) { + match self.ecx.tcx.get_global_alloc(id) { Some(GlobalAlloc::Memory(alloc)) => { write!(fmt, " (unchanged global, ")?; write_allocation_track_relocs( &mut *fmt, - self.mem.tcx, + *self.ecx.tcx, &mut allocs_to_print, alloc.inner(), )?; @@ -867,7 +855,7 @@ fn write_allocation_track_relocs<'tcx, Tag: Provenance, Extra>( write!(fmt, " (fn: {})", func)?; } Some(GlobalAlloc::Static(did)) => { - write!(fmt, " (static: {})", self.mem.tcx.def_path_str(did))?; + write!(fmt, " (static: {})", self.ecx.tcx.def_path_str(did))?; } None => { write!(fmt, " (deallocated)")?; @@ -923,16 +911,16 @@ pub fn check_bytes(&self, range: AllocRange, allow_uninit_and_ptr: bool) -> Inte } } -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Reads the given number of bytes from memory. Returns them as a slice. /// /// Performs appropriate bounds checks. - pub fn read_bytes( + pub fn read_bytes_ptr( &self, ptr: Pointer>, size: Size, ) -> InterpResult<'tcx, &[u8]> { - let Some(alloc_ref) = self.get(ptr, size, Align::ONE)? else { + let Some(alloc_ref) = self.get_ptr_alloc(ptr, size, Align::ONE)? else { // zero-sized access return Ok(&[]); }; @@ -947,7 +935,7 @@ pub fn read_bytes( /// Writes the given stream of bytes into memory. /// /// Performs appropriate bounds checks. - pub fn write_bytes( + pub fn write_bytes_ptr( &mut self, ptr: Pointer>, src: impl IntoIterator, @@ -958,7 +946,7 @@ pub fn write_bytes( assert_eq!(lower, len, "can only write iterators with a precise length"); let size = Size::from_bytes(len); - let Some(alloc_ref) = self.get_mut(ptr, size, Align::ONE)? else { + let Some(alloc_ref) = self.get_ptr_alloc_mut(ptr, size, Align::ONE)? else { // zero-sized access assert_matches!( src.next(), @@ -984,7 +972,7 @@ pub fn write_bytes( Ok(()) } - pub fn copy( + pub fn mem_copy( &mut self, src: Pointer>, src_align: Align, @@ -993,10 +981,10 @@ pub fn copy( size: Size, nonoverlapping: bool, ) -> InterpResult<'tcx> { - self.copy_repeatedly(src, src_align, dest, dest_align, size, 1, nonoverlapping) + self.mem_copy_repeatedly(src, src_align, dest, dest_align, size, 1, nonoverlapping) } - pub fn copy_repeatedly( + pub fn mem_copy_repeatedly( &mut self, src: Pointer>, src_align: Align, @@ -1019,9 +1007,9 @@ pub fn copy_repeatedly( // Zero-sized *source*, that means dst is also zero-sized and we have nothing to do. return Ok(()); }; - let src_alloc = self.get_raw(src_alloc_id)?; + let src_alloc = self.get_alloc_raw(src_alloc_id)?; let src_range = alloc_range(src_offset, size); - M::memory_read(&self.extra, &src_alloc.extra, src.provenance, src_range)?; + M::memory_read(&self.machine, &src_alloc.extra, src.provenance, src_range)?; // We need the `dest` ptr for the next operation, so we get it now. // We already did the source checks and called the hooks so we are good to return early. let Some((dest_alloc_id, dest_offset, dest)) = dest_parts else { @@ -1044,7 +1032,7 @@ pub fn copy_repeatedly( let compressed = src_alloc.compress_uninit_range(src_range); // Destination alloc preparations and access hooks. - let (dest_alloc, extra) = self.get_raw_mut(dest_alloc_id)?; + let (dest_alloc, extra) = self.get_alloc_raw_mut(dest_alloc_id)?; let dest_range = alloc_range(dest_offset, size * num_copies); M::memory_written(extra, &mut dest_alloc.extra, dest.provenance, dest_range)?; let dest_bytes = dest_alloc @@ -1112,7 +1100,7 @@ pub fn copy_repeatedly( } /// Machine pointer introspection. -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { pub fn scalar_to_ptr(&self, scalar: Scalar) -> Pointer> { // We use `to_bits_or_ptr_internal` since we are just implementing the method people need to // call to force getting out a pointer. @@ -1129,9 +1117,32 @@ pub fn scalar_to_ptr(&self, scalar: Scalar) -> Pointer) -> bool { + match scalar.try_to_int() { + Ok(int) => int.is_null(), + Err(_) => { + // Can only happen during CTFE. + let ptr = self.scalar_to_ptr(scalar); + match self.ptr_try_get_alloc_id(ptr) { + Ok((alloc_id, offset, _)) => { + let (size, _align) = self + .get_alloc_size_and_align(alloc_id, AllocCheck::MaybeDead) + .expect("alloc info with MaybeDead cannot fail"); + // If the pointer is out-of-bounds, it may be null. + // Note that one-past-the-end (offset == size) is still inbounds, and never null. + offset > size + } + Err(_offset) => bug!("a non-int scalar is always a pointer"), + } + } + } + } + /// Turning a "maybe pointer" into a proper pointer (and some information /// about where it points), or an absolute address. - pub fn ptr_try_get_alloc( + pub fn ptr_try_get_alloc_id( &self, ptr: Pointer>, ) -> Result<(AllocId, Size, Pointer), u64> { @@ -1146,11 +1157,11 @@ pub fn ptr_try_get_alloc( /// Turning a "maybe pointer" into a proper pointer (and some information about where it points). #[inline(always)] - pub fn ptr_get_alloc( + pub fn ptr_get_alloc_id( &self, ptr: Pointer>, ) -> InterpResult<'tcx, (AllocId, Size, Pointer)> { - self.ptr_try_get_alloc(ptr).map_err(|offset| { + self.ptr_try_get_alloc_id(ptr).map_err(|offset| { err_ub!(DanglingIntPointer(offset, CheckInAllocMsg::InboundsTest)).into() }) } diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 9000567558b..3012f763b80 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -257,7 +257,7 @@ fn try_read_immediate_from_mplace( return Ok(None); } - let Some(alloc) = self.get_alloc(mplace)? else { + let Some(alloc) = self.get_place_alloc(mplace)? else { return Ok(Some(ImmTy { // zero-sized type imm: Scalar::ZST.into(), @@ -340,7 +340,7 @@ pub fn read_pointer( // Turn the wide MPlace into a string (must already be dereferenced!) pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::PointerTag>) -> InterpResult<'tcx, &str> { let len = mplace.len(self)?; - let bytes = self.memory.read_bytes(mplace.ptr, Size::from_bytes(len))?; + let bytes = self.read_bytes_ptr(mplace.ptr, Size::from_bytes(len))?; let str = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?; Ok(str) } diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index 5b8d74b4307..c37973bd166 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -306,25 +306,25 @@ pub fn deref_operand( } #[inline] - pub(super) fn get_alloc( + pub(super) fn get_place_alloc( &self, place: &MPlaceTy<'tcx, M::PointerTag>, ) -> InterpResult<'tcx, Option>> { assert!(!place.layout.is_unsized()); assert!(!place.meta.has_meta()); let size = place.layout.size; - self.memory.get(place.ptr, size, place.align) + self.get_ptr_alloc(place.ptr, size, place.align) } #[inline] - pub(super) fn get_alloc_mut( + pub(super) fn get_place_alloc_mut( &mut self, place: &MPlaceTy<'tcx, M::PointerTag>, ) -> InterpResult<'tcx, Option>> { assert!(!place.layout.is_unsized()); assert!(!place.meta.has_meta()); let size = place.layout.size; - self.memory.get_mut(place.ptr, size, place.align) + self.get_ptr_alloc_mut(place.ptr, size, place.align) } /// Check if this mplace is dereferenceable and sufficiently aligned. @@ -337,8 +337,8 @@ fn check_mplace_access( .size_and_align_of_mplace(&mplace)? .unwrap_or((mplace.layout.size, mplace.layout.align.abi)); assert!(mplace.mplace.align <= align, "dynamic alignment less strict than static one?"); - let align = M::enforce_alignment(&self.memory.extra).then_some(align); - self.memory.check_ptr_access_align(mplace.ptr, size, align.unwrap_or(Align::ONE), msg)?; + let align = M::enforce_alignment(self).then_some(align); + self.check_ptr_access_align(mplace.ptr, size, align.unwrap_or(Align::ONE), msg)?; Ok(()) } @@ -748,7 +748,7 @@ fn write_immediate_to_mplace_no_validate( // Invalid places are a thing: the return place of a diverging function let tcx = *self.tcx; - let Some(mut alloc) = self.get_alloc_mut(dest)? else { + let Some(mut alloc) = self.get_place_alloc_mut(dest)? else { // zero-sized access return Ok(()); }; @@ -857,8 +857,7 @@ fn copy_op_no_validate( }); assert_eq!(src.meta, dest.meta, "Can only copy between equally-sized instances"); - self.memory - .copy(src.ptr, src.align, dest.ptr, dest.align, size, /*nonoverlapping*/ true) + self.mem_copy(src.ptr, src.align, dest.ptr, dest.align, size, /*nonoverlapping*/ true) } /// Copies the data from an operand to a place. The layouts may disagree, but they must @@ -942,7 +941,7 @@ pub fn force_allocation_maybe_sized( let (size, align) = self .size_and_align_of(&meta, &local_layout)? .expect("Cannot allocate for non-dyn-sized type"); - let ptr = self.memory.allocate(size, align, MemoryKind::Stack)?; + let ptr = self.allocate_ptr(size, align, MemoryKind::Stack)?; let mplace = MemPlace { ptr: ptr.into(), align, meta }; if let LocalValue::Live(Operand::Immediate(value)) = local_val { // Preserve old value. @@ -979,7 +978,7 @@ pub fn allocate( layout: TyAndLayout<'tcx>, kind: MemoryKind, ) -> InterpResult<'static, MPlaceTy<'tcx, M::PointerTag>> { - let ptr = self.memory.allocate(layout.size, layout.align.abi, kind)?; + let ptr = self.allocate_ptr(layout.size, layout.align.abi, kind)?; Ok(MPlaceTy::from_aligned_ptr(ptr.into(), layout)) } @@ -990,7 +989,7 @@ pub fn allocate_str( kind: MemoryKind, mutbl: Mutability, ) -> MPlaceTy<'tcx, M::PointerTag> { - let ptr = self.memory.allocate_bytes(str.as_bytes(), Align::ONE, kind, mutbl); + let ptr = self.allocate_bytes_ptr(str.as_bytes(), Align::ONE, kind, mutbl); let meta = Scalar::from_machine_usize(u64::try_from(str.len()).unwrap(), self); let mplace = MemPlace { ptr: ptr.into(), align: Align::ONE, meta: MemPlaceMeta::Meta(meta) }; diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 4272bfd5d6c..84563daa088 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -225,7 +225,7 @@ pub fn eval_rvalue_into_place( if length == 0 { // Nothing to copy... but let's still make sure that `dest` as a place is valid. - self.get_alloc_mut(&dest)?; + self.get_place_alloc_mut(&dest)?; } else { // Write the src to the first element. let first = self.mplace_field(&dest, 0)?; @@ -241,7 +241,7 @@ pub fn eval_rvalue_into_place( // that place might be more aligned than its type mandates (a `u8` array could // be 4-aligned if it sits at the right spot in a struct). Instead we use // `first.layout.align`, i.e., the alignment given by the type. - self.memory.copy_repeatedly( + self.mem_copy_repeatedly( first_ptr, first.align, rest_ptr, diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index d33358499e2..a7587652114 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -72,7 +72,7 @@ pub(super) fn eval_terminator( let (fn_val, fn_abi, with_caller_location) = match *func.layout.ty.kind() { ty::FnPtr(_sig) => { let fn_ptr = self.read_pointer(&func)?; - let fn_val = self.memory.get_fn(fn_ptr)?; + let fn_val = self.get_ptr_fn(fn_ptr)?; (fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false) } ty::FnDef(def_id, substs) => { diff --git a/compiler/rustc_const_eval/src/interpret/traits.rs b/compiler/rustc_const_eval/src/interpret/traits.rs index fc60a40e2ad..5cf3807faaa 100644 --- a/compiler/rustc_const_eval/src/interpret/traits.rs +++ b/compiler/rustc_const_eval/src/interpret/traits.rs @@ -32,7 +32,7 @@ pub fn get_vtable( let vtable_allocation = self.tcx.vtable_allocation((ty, poly_trait_ref)); - let vtable_ptr = self.memory.global_base_pointer(Pointer::from(vtable_allocation))?; + let vtable_ptr = self.global_base_pointer(Pointer::from(vtable_allocation))?; Ok(vtable_ptr.into()) } @@ -48,11 +48,10 @@ pub fn get_vtable_slot( let ptr_size = self.pointer_size(); let vtable_slot = vtable.offset(ptr_size * idx, self)?; let vtable_slot = self - .memory - .get(vtable_slot, ptr_size, self.tcx.data_layout.pointer_align.abi)? + .get_ptr_alloc(vtable_slot, ptr_size, self.tcx.data_layout.pointer_align.abi)? .expect("cannot be a ZST"); let fn_ptr = self.scalar_to_ptr(vtable_slot.read_ptr_sized(Size::ZERO)?.check_init()?); - self.memory.get_fn(fn_ptr) + self.get_ptr_fn(fn_ptr) } /// Returns the drop fn instance as well as the actual dynamic type. @@ -63,8 +62,7 @@ pub fn read_drop_type_from_vtable( let pointer_size = self.pointer_size(); // We don't care about the pointee type; we just want a pointer. let vtable = self - .memory - .get( + .get_ptr_alloc( vtable, pointer_size * u64::try_from(COMMON_VTABLE_ENTRIES.len()).unwrap(), self.tcx.data_layout.pointer_align.abi, @@ -77,7 +75,7 @@ pub fn read_drop_type_from_vtable( .check_init()?; // We *need* an instance here, no other kind of function value, to be able // to determine the type. - let drop_instance = self.memory.get_fn(self.scalar_to_ptr(drop_fn))?.as_instance()?; + let drop_instance = self.get_ptr_fn(self.scalar_to_ptr(drop_fn))?.as_instance()?; trace!("Found drop fn: {:?}", drop_instance); let fn_sig = drop_instance.ty(*self.tcx, self.param_env).fn_sig(*self.tcx); let fn_sig = self.tcx.normalize_erasing_late_bound_regions(self.param_env, fn_sig); @@ -99,8 +97,7 @@ pub fn read_size_and_align_from_vtable( // We check for `size = 3 * ptr_size`, which covers the drop fn (unused here), // the size, and the align (which we read below). let vtable = self - .memory - .get( + .get_ptr_alloc( vtable, pointer_size * u64::try_from(COMMON_VTABLE_ENTRIES.len()).unwrap(), self.tcx.data_layout.pointer_align.abi, @@ -132,8 +129,7 @@ pub fn read_new_vtable_after_trait_upcasting_from_vtable( let vtable_slot = vtable.offset(pointer_size * idx, self)?; let new_vtable = self - .memory - .get(vtable_slot, pointer_size, self.tcx.data_layout.pointer_align.abi)? + .get_ptr_alloc(vtable_slot, pointer_size, self.tcx.data_layout.pointer_align.abi)? .expect("cannot be a ZST"); let new_vtable = self.scalar_to_ptr(new_vtable.read_ptr_sized(Size::ZERO)?.check_init()?); diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index 9da7f5e30cb..42ff7b19459 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -315,7 +315,7 @@ fn check_wide_ptr_meta( let vtable = self.ecx.scalar_to_ptr(meta.unwrap_meta()); // Direct call to `check_ptr_access_align` checks alignment even on CTFE machines. try_validation!( - self.ecx.memory.check_ptr_access_align( + self.ecx.check_ptr_access_align( vtable, 3 * self.ecx.tcx.data_layout.pointer_size, // drop, size, align self.ecx.tcx.data_layout.pointer_align.abi, @@ -403,7 +403,7 @@ fn check_safe_pointer( .unwrap_or_else(|| (place.layout.size, place.layout.align.abi)); // Direct call to `check_ptr_access_align` checks alignment even on CTFE machines. try_validation!( - self.ecx.memory.check_ptr_access_align( + self.ecx.check_ptr_access_align( place.ptr, size, align, @@ -432,7 +432,7 @@ fn check_safe_pointer( if let Some(ref mut ref_tracking) = self.ref_tracking { // Proceed recursively even for ZST, no reason to skip them! // `!` is a ZST and we want to validate it. - if let Ok((alloc_id, _offset, _ptr)) = self.ecx.memory.ptr_try_get_alloc(place.ptr) { + if let Ok((alloc_id, _offset, _ptr)) = self.ecx.ptr_try_get_alloc_id(place.ptr) { // Special handling for pointers to statics (irrespective of their type). let alloc_kind = self.ecx.tcx.get_global_alloc(alloc_id); if let Some(GlobalAlloc::Static(did)) = alloc_kind { @@ -579,7 +579,7 @@ fn try_visit_primitive( if let Some(_) = self.ref_tracking { let ptr = self.ecx.scalar_to_ptr(value); let _fn = try_validation!( - self.ecx.memory.get_fn(ptr), + self.ecx.get_ptr_fn(ptr), self.path, err_ub!(DanglingIntPointer(0, _)) => { "a null function pointer" }, @@ -825,7 +825,7 @@ fn visit_aggregate( let mplace = op.assert_mem_place(); // strings are never immediate let len = mplace.len(self.ecx)?; try_validation!( - self.ecx.memory.read_bytes(mplace.ptr, Size::from_bytes(len)), + self.ecx.read_bytes_ptr(mplace.ptr, Size::from_bytes(len)), self.path, err_ub!(InvalidUninitBytes(..)) => { "uninitialized data in `str`" }, err_unsup!(ReadPointerAsBytes) => { "a pointer in `str`" }, @@ -861,7 +861,7 @@ fn visit_aggregate( // to reject those pointers, we just do not have the machinery to // talk about parts of a pointer. // We also accept uninit, for consistency with the slow path. - let Some(alloc) = self.ecx.memory.get(mplace.ptr, size, mplace.align)? else { + let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, mplace.align)? else { // Size 0, nothing more to check. return Ok(()); }; diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index e1dbf90b5b9..49ebdd30254 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -184,8 +184,6 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> type MemoryKind = !; - type MemoryExtra = (); - fn load_mir( _ecx: &InterpCx<'mir, 'tcx, Self>, _instance: ty::InstanceDef<'tcx>, @@ -267,7 +265,7 @@ fn access_local_mut<'a>( } fn before_access_global( - _memory_extra: &(), + _machine: &Self, _alloc_id: AllocId, alloc: ConstAllocation<'tcx, Self::PointerTag, Self::AllocExtra>, _static_def_id: Option, @@ -377,7 +375,6 @@ fn new( span, param_env, ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop), - (), ); let ret = ecx diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index 5be745bc1de..602aa14b9e6 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -180,8 +180,6 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> type MemoryKind = !; - type MemoryExtra = (); - fn load_mir( _ecx: &InterpCx<'mir, 'tcx, Self>, _instance: ty::InstanceDef<'tcx>, @@ -263,7 +261,7 @@ fn access_local_mut<'a>( } fn before_access_global( - _memory_extra: &(), + _machine: &Self, _alloc_id: AllocId, alloc: ConstAllocation<'tcx, Self::PointerTag, Self::AllocExtra>, _static_def_id: Option, @@ -374,7 +372,6 @@ fn new( span, param_env, ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop), - (), ); let ret = ecx