mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rollup merge of #95620 - RalfJung:memory-no-extras, r=oli-obk

Browse Source 
interpret: remove MemoryExtra in favor of giving access to the Machine

The Miri PR for this is upcoming.

r? ``@oli-obk``
This commit is contained in:
Dylan DPC 2022-04-05 01:53:33 +02:00 committed by GitHub
commit 78f81f0d10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 201 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
compiler/rustc_const_eval/src/const_eval/eval_queries.rs
View File

@ -1,4 +1,4 @@
use super::{CompileTimeEvalContext, CompileTimeInterpreter, ConstEvalErr, MemoryExtra};
use super::{CompileTimeEvalContext, CompileTimeInterpreter, ConstEvalErr};
use crate::interpret::eval_nullary_intrinsic;
use crate::interpret::{
intern_const_alloc_recursive, Allocation, ConstAlloc, ConstValue, CtfeValidationMode, GlobalId,
@ -100,8 +100,7 @@ pub(super) fn mk_eval_cx<'mir, 'tcx>(
tcx,
root_span,
param_env,
CompileTimeInterpreter::new(tcx.const_eval_limit()),
MemoryExtra { can_access_statics },
CompileTimeInterpreter::new(tcx.const_eval_limit(), can_access_statics),
)
}
@ -285,10 +284,9 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
tcx,
tcx.def_span(def.did),
key.param_env,
CompileTimeInterpreter::new(tcx.const_eval_limit()),
// Statics (and promoteds inside statics) may access other statics, because unlike consts
// they do not have to behave "as if" they were evaluated at runtime.
MemoryExtra { can_access_statics: is_static },
CompileTimeInterpreter::new(tcx.const_eval_limit(), /*can_access_statics:*/ is_static),
);
let res = ecx.load_mir(cid.instance.def, cid.promoted);

23
compiler/rustc_const_eval/src/const_eval/machine.rs
View File

@ -93,10 +93,7 @@ pub struct CompileTimeInterpreter<'mir, 'tcx> {
/// The virtual call stack.
pub(crate) stack: Vec<Frame<'mir, 'tcx, AllocId, ()>>,
}
#[derive(Copy, Clone, Debug)]
pub struct MemoryExtra {
/// We need to make sure consts never point to anything mutable, even recursively. That is
/// relied on for pattern matching on consts with references.
/// To achieve this, two pieces have to work together:
@ -107,8 +104,12 @@ pub struct MemoryExtra {
}
impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> {
pub(super) fn new(const_eval_limit: Limit) -> Self {
CompileTimeInterpreter { steps_remaining: const_eval_limit.0, stack: Vec::new() }
pub(super) fn new(const_eval_limit: Limit, can_access_statics: bool) -> Self {
CompileTimeInterpreter {
steps_remaining: const_eval_limit.0,
stack: Vec::new(),
can_access_statics,
}
}
}
@ -233,8 +234,6 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
type MemoryKind = MemoryKind;
type MemoryExtra = MemoryExtra;
const PANIC_ON_ALLOC_FAIL: bool = false; // will be raised as a proper error
fn load_mir(
@ -345,7 +344,7 @@ fn call_intrinsic(
Err(err) => throw_ub_format!("align has to be a power of 2, {}", err),
};
let ptr = ecx.memory.allocate(
let ptr = ecx.allocate_ptr(
Size::from_bytes(size as u64),
align,
interpret::MemoryKind::Machine(MemoryKind::Heap),
@ -365,14 +364,14 @@ fn call_intrinsic(
// If an allocation is created in an another const,
// we don't deallocate it.
let (alloc_id, _, _) = ecx.memory.ptr_get_alloc(ptr)?;
let (alloc_id, _, _) = ecx.ptr_get_alloc_id(ptr)?;
let is_allocated_in_another_const = matches!(
ecx.tcx.get_global_alloc(alloc_id),
Some(interpret::GlobalAlloc::Memory(_))
);
if !is_allocated_in_another_const {
ecx.memory.deallocate(
ecx.deallocate_ptr(
ptr,
Some((size, align)),
interpret::MemoryKind::Machine(MemoryKind::Heap),
@ -472,7 +471,7 @@ fn stack_mut<'a>(
}
fn before_access_global(
memory_extra: &MemoryExtra,
machine: &Self,
alloc_id: AllocId,
alloc: ConstAllocation<'tcx>,
static_def_id: Option<DefId>,
@ -488,7 +487,7 @@ fn before_access_global(
}
} else {
// Read access. These are usually allowed, with some exceptions.
if memory_extra.can_access_statics {
if machine.can_access_statics {
// Machine configuration allows us read from anything (e.g., `static` initializer).
Ok(())
} else if static_def_id.is_some() {

8
compiler/rustc_const_eval/src/interpret/cast.rs
View File

@ -56,7 +56,7 @@ pub fn cast(
)
.ok_or_else(|| err_inval!(TooGeneric))?;
let fn_ptr = self.memory.create_fn_alloc(FnVal::Instance(instance));
let fn_ptr = self.create_fn_alloc_ptr(FnVal::Instance(instance));
self.write_pointer(fn_ptr, dest)?;
}
_ => span_bug!(self.cur_span(), "reify fn pointer on {:?}", src.layout.ty),
@ -87,7 +87,7 @@ pub fn cast(
substs,
ty::ClosureKind::FnOnce,
);
let fn_ptr = self.memory.create_fn_alloc(FnVal::Instance(instance));
let fn_ptr = self.create_fn_alloc_ptr(FnVal::Instance(instance));
self.write_pointer(fn_ptr, dest)?;
}
_ => span_bug!(self.cur_span(), "closure fn pointer on {:?}", src.layout.ty),
@ -153,8 +153,8 @@ pub fn misc_cast(
return Ok(**src);
} else {
// Casting the metadata away from a fat ptr.
assert_eq!(src.layout.size, 2 * self.memory.pointer_size());
assert_eq!(dest_layout.size, self.memory.pointer_size());
assert_eq!(src.layout.size, 2 * self.pointer_size());
assert_eq!(dest_layout.size, self.pointer_size());
assert!(src.layout.ty.is_unsafe_ptr());
return match **src {
Immediate::ScalarPair(data, _) => Ok(data.into()),

60
compiler/rustc_const_eval/src/interpret/eval_context.rs
View File

@ -22,9 +22,9 @@
use rustc_target::abi::{call::FnAbi, Align, HasDataLayout, Size, TargetDataLayout};
use super::{
AllocCheck, AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine,
MemPlace, MemPlaceMeta, Memory, MemoryKind, Operand, Place, PlaceTy, Pointer,
PointerArithmetic, Provenance, Scalar, ScalarMaybeUninit, StackPopJump,
AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace,
MemPlaceMeta, Memory, MemoryKind, Operand, Place, PlaceTy, PointerArithmetic, Provenance,
Scalar, ScalarMaybeUninit, StackPopJump,
};
use crate::transform::validate::equal_up_to_regions;
@ -413,13 +413,12 @@ pub fn new(
root_span: Span,
param_env: ty::ParamEnv<'tcx>,
machine: M,
memory_extra: M::MemoryExtra,
) -> Self {
InterpCx {
machine,
tcx: tcx.at(root_span),
param_env,
memory: Memory::new(tcx, memory_extra),
memory: Memory::new(),
recursion_limit: tcx.recursion_limit(),
}
}
@ -433,49 +432,6 @@ pub fn cur_span(&self) -> Span {
.map_or(self.tcx.span, |f| f.current_span())
}
#[inline(always)]
pub fn scalar_to_ptr(&self, scalar: Scalar<M::PointerTag>) -> Pointer<Option<M::PointerTag>> {
self.memory.scalar_to_ptr(scalar)
}
/// Test if this value might be null.
/// If the machine does not support ptr-to-int casts, this is conservative.
pub fn scalar_may_be_null(&self, scalar: Scalar<M::PointerTag>) -> bool {
match scalar.try_to_int() {
Ok(int) => int.is_null(),
Err(_) => {
// Can only happen during CTFE.
let ptr = self.scalar_to_ptr(scalar);
match self.memory.ptr_try_get_alloc(ptr) {
Ok((alloc_id, offset, _)) => {
let (size, _align) = self
.memory
.get_size_and_align(alloc_id, AllocCheck::MaybeDead)
.expect("alloc info with MaybeDead cannot fail");
// If the pointer is out-of-bounds, it may be null.
// Note that one-past-the-end (offset == size) is still inbounds, and never null.
offset > size
}
Err(_offset) => bug!("a non-int scalar is always a pointer"),
}
}
}
}
/// Call this to turn untagged "global" pointers (obtained via `tcx`) into
/// the machine pointer to the allocation. Must never be used
/// for any other pointers, nor for TLS statics.
///
/// Using the resulting pointer represents a *direct* access to that memory
/// (e.g. by directly using a `static`),
/// as opposed to access through a pointer that was created by the program.
///
/// This function can fail only if `ptr` points to an `extern static`.
#[inline(always)]
pub fn global_base_pointer(&self, ptr: Pointer) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
self.memory.global_base_pointer(ptr)
}
#[inline(always)]
pub(crate) fn stack(&self) -> &[Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>] {
M::stack(self)
@ -949,9 +905,9 @@ fn deallocate_local(&mut self, local: LocalValue<M::PointerTag>) -> InterpResult
trace!(
"deallocating local {:?}: {:?}",
local,
self.memory.dump_alloc(ptr.provenance.unwrap().get_alloc_id())
self.dump_alloc(ptr.provenance.unwrap().get_alloc_id())
);
self.memory.deallocate(ptr, None, MemoryKind::Stack)?;
self.deallocate_ptr(ptr, None, MemoryKind::Stack)?;
};
Ok(())
}
@ -1057,7 +1013,7 @@ fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
}
}
write!(fmt, ": {:?}", self.ecx.memory.dump_allocs(allocs))
write!(fmt, ": {:?}", self.ecx.dump_allocs(allocs))
}
Place::Ptr(mplace) => match mplace.ptr.provenance.map(Provenance::get_alloc_id) {
Some(alloc_id) => write!(
@ -1065,7 +1021,7 @@ fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
"by align({}) ref {:?}: {:?}",
mplace.align.bytes(),
mplace.ptr,
self.ecx.memory.dump_alloc(alloc_id)
self.ecx.dump_alloc(alloc_id)
),
ptr => write!(fmt, " integral by ref: {:?}", ptr),
},

16
compiler/rustc_const_eval/src/interpret/intrinsics.rs
View File

@ -318,7 +318,7 @@ pub fn emulate_intrinsic(
// exception from the exception.)
// This is the dual to the special exception for offset-by-0
// in the inbounds pointer offset operation (see `ptr_offset_inbounds` below).
match (self.memory.ptr_try_get_alloc(a), self.memory.ptr_try_get_alloc(b)) {
match (self.ptr_try_get_alloc_id(a), self.ptr_try_get_alloc_id(b)) {
(Err(a), Err(b)) if a == b && a != 0 => {
// Both are the same non-null integer.
self.write_scalar(Scalar::from_machine_isize(0, self), dest)?;
@ -335,13 +335,13 @@ pub fn emulate_intrinsic(
);
}
// And they must both be valid for zero-sized accesses ("in-bounds or one past the end").
self.memory.check_ptr_access_align(
self.check_ptr_access_align(
a,
Size::ZERO,
Align::ONE,
CheckInAllocMsg::OffsetFromTest,
)?;
self.memory.check_ptr_access_align(
self.check_ptr_access_align(
b,
Size::ZERO,
Align::ONE,
@ -545,7 +545,7 @@ pub fn ptr_offset_inbounds(
let min_ptr = if offset_bytes >= 0 { ptr } else { offset_ptr };
let size = offset_bytes.unsigned_abs();
// This call handles checking for integer/null pointers.
self.memory.check_ptr_access_align(
self.check_ptr_access_align(
min_ptr,
Size::from_bytes(size),
Align::ONE,
@ -577,7 +577,7 @@ pub(crate) fn copy_intrinsic(
let src = self.read_pointer(&src)?;
let dst = self.read_pointer(&dst)?;
self.memory.copy(src, align, dst, align, size, nonoverlapping)
self.mem_copy(src, align, dst, align, size, nonoverlapping)
}
pub(crate) fn write_bytes_intrinsic(
@ -600,7 +600,7 @@ pub(crate) fn write_bytes_intrinsic(
.ok_or_else(|| err_ub_format!("overflow computing total size of `write_bytes`"))?;
let bytes = std::iter::repeat(byte).take(len.bytes_usize());
self.memory.write_bytes(dst, bytes)
self.write_bytes_ptr(dst, bytes)
}
pub(crate) fn raw_eq_intrinsic(
@ -613,8 +613,8 @@ pub(crate) fn raw_eq_intrinsic(
let lhs = self.read_pointer(lhs)?;
let rhs = self.read_pointer(rhs)?;
let lhs_bytes = self.memory.read_bytes(lhs, layout.size)?;
let rhs_bytes = self.memory.read_bytes(rhs, layout.size)?;
let lhs_bytes = self.read_bytes_ptr(lhs, layout.size)?;
let rhs_bytes = self.read_bytes_ptr(rhs, layout.size)?;
Ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
}
}

45
compiler/rustc_const_eval/src/interpret/machine.rs
View File

@ -14,7 +14,7 @@
use super::{
AllocId, AllocRange, Allocation, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult,
LocalValue, MemPlace, Memory, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar,
LocalValue, MemPlace, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar,
StackPopUnwind,
};
@ -96,11 +96,6 @@ pub trait Machine<'mir, 'tcx>: Sized {
/// Extra data stored in every call frame.
type FrameExtra;
/// Extra data stored in memory. A reference to this is available when `AllocExtra`
/// gets initialized, so you can e.g., have an `Rc` here if there is global state you
/// need access to in the `AllocExtra` hooks.
type MemoryExtra;
/// Extra data stored in every allocation.
type AllocExtra: Debug + Clone + 'static;
@ -123,11 +118,11 @@ pub trait Machine<'mir, 'tcx>: Sized {
const PANIC_ON_ALLOC_FAIL: bool;
/// Whether memory accesses should be alignment-checked.
fn enforce_alignment(memory_extra: &Self::MemoryExtra) -> bool;
fn enforce_alignment(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
/// Whether, when checking alignment, we should `force_int` and thus support
/// custom alignment logic based on whatever the integer address happens to be.
fn force_int_for_alignment_check(memory_extra: &Self::MemoryExtra) -> bool;
fn force_int_for_alignment_check(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
/// Whether to enforce the validity invariant
fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
@ -251,7 +246,7 @@ fn before_terminator(_ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx
/// `def_id` is `Some` if this is the "lazy" allocation of a static.
#[inline]
fn before_access_global(
_memory_extra: &Self::MemoryExtra,
_machine: &Self,
_alloc_id: AllocId,
_allocation: ConstAllocation<'tcx>,
_static_def_id: Option<DefId>,
@ -270,7 +265,7 @@ fn thread_local_static_base_pointer(
/// Return the root pointer for the given `extern static`.
fn extern_static_base_pointer(
mem: &Memory<'mir, 'tcx, Self>,
ecx: &InterpCx<'mir, 'tcx, Self>,
def_id: DefId,
) -> InterpResult<'tcx, Pointer<Self::PointerTag>>;
@ -279,19 +274,19 @@ fn extern_static_base_pointer(
///
/// Not called on `extern` or thread-local statics (those use the methods above).
fn tag_alloc_base_pointer(
mem: &Memory<'mir, 'tcx, Self>,
ecx: &InterpCx<'mir, 'tcx, Self>,
ptr: Pointer,
) -> Pointer<Self::PointerTag>;
/// "Int-to-pointer cast"
fn ptr_from_addr(
mem: &Memory<'mir, 'tcx, Self>,
ecx: &InterpCx<'mir, 'tcx, Self>,
addr: u64,
) -> Pointer<Option<Self::PointerTag>>;
/// Convert a pointer with provenance into an allocation-offset pair.
fn ptr_get_alloc(
mem: &Memory<'mir, 'tcx, Self>,
ecx: &InterpCx<'mir, 'tcx, Self>,
ptr: Pointer<Self::PointerTag>,
) -> (AllocId, Size);
@ -309,7 +304,7 @@ fn ptr_get_alloc(
/// cache the result. (This relies on `AllocMap::get_or` being able to add the
/// owned allocation to the map even when the map is shared.)
fn init_allocation_extra<'b>(
mem: &Memory<'mir, 'tcx, Self>,
ecx: &InterpCx<'mir, 'tcx, Self>,
id: AllocId,
alloc: Cow<'b, Allocation>,
kind: Option<MemoryKind<Self::MemoryKind>>,
@ -322,7 +317,7 @@ fn init_allocation_extra<'b>(
/// need to mutate.
#[inline(always)]
fn memory_read(
_memory_extra: &Self::MemoryExtra,
_machine: &Self,
_alloc_extra: &Self::AllocExtra,
_tag: Self::PointerTag,
_range: AllocRange,
@ -333,7 +328,7 @@ fn memory_read(
/// Hook for performing extra checks on a memory write access.
#[inline(always)]
fn memory_written(
_memory_extra: &mut Self::MemoryExtra,
_machine: &mut Self,
_alloc_extra: &mut Self::AllocExtra,
_tag: Self::PointerTag,
_range: AllocRange,
@ -344,7 +339,7 @@ fn memory_written(
/// Hook for performing extra operations on a memory deallocation.
#[inline(always)]
fn memory_deallocated(
_memory_extra: &mut Self::MemoryExtra,
_machine: &mut Self,
_alloc_extra: &mut Self::AllocExtra,
_tag: Self::PointerTag,
_range: AllocRange,
@ -408,14 +403,14 @@ fn after_stack_pop(
type FrameExtra = ();
#[inline(always)]
fn enforce_alignment(_memory_extra: &Self::MemoryExtra) -> bool {
fn enforce_alignment(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
// We do not check for alignment to avoid having to carry an `Align`
// in `ConstValue::ByRef`.
false
}
#[inline(always)]
fn force_int_for_alignment_check(_memory_extra: &Self::MemoryExtra) -> bool {
fn force_int_for_alignment_check(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
// We do not support `force_int`.
false
}
@ -444,7 +439,7 @@ fn call_extra_fn(
#[inline(always)]
fn init_allocation_extra<'b>(
_mem: &Memory<$mir, $tcx, Self>,
_ecx: &InterpCx<$mir, $tcx, Self>,
_id: AllocId,
alloc: Cow<'b, Allocation>,
_kind: Option<MemoryKind<Self::MemoryKind>>,
@ -454,28 +449,28 @@ fn init_allocation_extra<'b>(
}
fn extern_static_base_pointer(
mem: &Memory<$mir, $tcx, Self>,
ecx: &InterpCx<$mir, $tcx, Self>,
def_id: DefId,
) -> InterpResult<$tcx, Pointer> {
// Use the `AllocId` associated with the `DefId`. Any actual *access* will fail.
Ok(Pointer::new(mem.tcx.create_static_alloc(def_id), Size::ZERO))
Ok(Pointer::new(ecx.tcx.create_static_alloc(def_id), Size::ZERO))
}
#[inline(always)]
fn tag_alloc_base_pointer(
_mem: &Memory<$mir, $tcx, Self>,
_ecx: &InterpCx<$mir, $tcx, Self>,
ptr: Pointer<AllocId>,
) -> Pointer<AllocId> {
ptr
}
#[inline(always)]
fn ptr_from_addr(_mem: &Memory<$mir, $tcx, Self>, addr: u64) -> Pointer<Option<AllocId>> {
fn ptr_from_addr(_ecx: &InterpCx<$mir, $tcx, Self>, addr: u64) -> Pointer<Option<AllocId>> {
Pointer::new(None, Size::from_bytes(addr))
}
#[inline(always)]
fn ptr_get_alloc(_mem: &Memory<$mir, $tcx, Self>, ptr: Pointer<AllocId>) -> (AllocId, Size) {
fn ptr_get_alloc(_ecx: &InterpCx<$mir, $tcx, Self>, ptr: Pointer<AllocId>) -> (AllocId, Size) {
// We know `offset` is relative to the allocation, so we can use `into_parts`.
let (alloc_id, offset) = ptr.into_parts();
(alloc_id, offset)

221
compiler/rustc_const_eval/src/interpret/memory.rs
View File

@ -17,10 +17,10 @@
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_middle::mir::display_allocation;
use rustc_middle::ty::{Instance, ParamEnv, TyCtxt};
use rustc_target::abi::{Align, HasDataLayout, Size, TargetDataLayout};
use rustc_target::abi::{Align, HasDataLayout, Size};
use super::{
alloc_range, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg, GlobalAlloc,
alloc_range, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg, GlobalAlloc, InterpCx,
InterpResult, Machine, MayLeak, Pointer, PointerArithmetic, Provenance, Scalar,
ScalarMaybeUninit,
};
@ -108,19 +108,6 @@ pub struct Memory<'mir, 'tcx, M: Machine<'mir, 'tcx>> {
/// that do not exist any more.
// FIXME: this should not be public, but interning currently needs access to it
pub(super) dead_alloc_map: FxHashMap<AllocId, (Size, Align)>,
/// Extra data added by the machine.
pub extra: M::MemoryExtra,
/// Lets us implement `HasDataLayout`, which is awfully convenient.
pub tcx: TyCtxt<'tcx>,
}
impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> HasDataLayout for Memory<'mir, 'tcx, M> {
#[inline]
fn data_layout(&self) -> &TargetDataLayout {
&self.tcx.data_layout
}
}
/// A reference to some allocation that was already bounds-checked for the given region
@ -142,16 +129,21 @@ pub struct AllocRefMut<'a, 'tcx, Tag, Extra> {
}
impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
pub fn new(tcx: TyCtxt<'tcx>, extra: M::MemoryExtra) -> Self {
pub fn new() -> Self {
Memory {
alloc_map: M::MemoryMap::default(),
extra_fn_ptr_map: FxHashMap::default(),
dead_alloc_map: FxHashMap::default(),
extra,
tcx,
}
}
/// This is used by [priroda](https://github.com/oli-obk/priroda)
pub fn alloc_map(&self) -> &M::MemoryMap {
&self.alloc_map
}
}
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Call this to turn untagged "global" pointers (obtained via `tcx`) into
/// the machine pointer to the allocation. Must never be used
/// for any other pointers, nor for TLS statics.
@ -182,7 +174,7 @@ pub fn global_base_pointer(
Ok(M::tag_alloc_base_pointer(self, Pointer::new(alloc_id, offset)))
}
pub fn create_fn_alloc(
pub fn create_fn_alloc_ptr(
&mut self,
fn_val: FnVal<'tcx, M::ExtraFnVal>,
) -> Pointer<M::PointerTag> {
@ -191,7 +183,7 @@ pub fn create_fn_alloc(
FnVal::Other(extra) => {
// FIXME(RalfJung): Should we have a cache here?
let id = self.tcx.reserve_alloc_id();
let old = self.extra_fn_ptr_map.insert(id, extra);
let old = self.memory.extra_fn_ptr_map.insert(id, extra);
assert!(old.is_none());
id
}
@ -201,17 +193,17 @@ pub fn create_fn_alloc(
self.global_base_pointer(Pointer::from(id)).unwrap()
}
pub fn allocate(
pub fn allocate_ptr(
&mut self,
size: Size,
align: Align,
kind: MemoryKind<M::MemoryKind>,
) -> InterpResult<'static, Pointer<M::PointerTag>> {
let alloc = Allocation::uninit(size, align, M::PANIC_ON_ALLOC_FAIL)?;
Ok(self.allocate_with(alloc, kind))
Ok(self.allocate_raw_ptr(alloc, kind))
}
pub fn allocate_bytes(
pub fn allocate_bytes_ptr(
&mut self,
bytes: &[u8],
align: Align,
@ -219,10 +211,10 @@ pub fn allocate_bytes(
mutability: Mutability,
) -> Pointer<M::PointerTag> {
let alloc = Allocation::from_bytes(bytes, align, mutability);
self.allocate_with(alloc, kind)
self.allocate_raw_ptr(alloc, kind)
}
pub fn allocate_with(
pub fn allocate_raw_ptr(
&mut self,
alloc: Allocation,
kind: MemoryKind<M::MemoryKind>,
@ -234,11 +226,11 @@ pub fn allocate_with(
"dynamically allocating global memory"
);
let alloc = M::init_allocation_extra(self, id, Cow::Owned(alloc), Some(kind));
self.alloc_map.insert(id, (kind, alloc.into_owned()));
self.memory.alloc_map.insert(id, (kind, alloc.into_owned()));
M::tag_alloc_base_pointer(self, Pointer::from(id))
}
pub fn reallocate(
pub fn reallocate_ptr(
&mut self,
ptr: Pointer<Option<M::PointerTag>>,
old_size_and_align: Option<(Size, Align)>,
@ -246,7 +238,7 @@ pub fn reallocate(
new_align: Align,
kind: MemoryKind<M::MemoryKind>,
) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
let (alloc_id, offset, ptr) = self.ptr_get_alloc(ptr)?;
let (alloc_id, offset, ptr) = self.ptr_get_alloc_id(ptr)?;
if offset.bytes() != 0 {
throw_ub_format!(
"reallocating {:?} which does not point to the beginning of an object",
@ -256,13 +248,13 @@ pub fn reallocate(
// For simplicities' sake, we implement reallocate as "alloc, copy, dealloc".
// This happens so rarely, the perf advantage is outweighed by the maintenance cost.
let new_ptr = self.allocate(new_size, new_align, kind)?;
let new_ptr = self.allocate_ptr(new_size, new_align, kind)?;
let old_size = match old_size_and_align {
Some((size, _align)) => size,
None => self.get_raw(alloc_id)?.size(),
None => self.get_alloc_raw(alloc_id)?.size(),
};
// This will also call the access hooks.
self.copy(
self.mem_copy(
ptr.into(),
Align::ONE,
new_ptr.into(),
@ -270,19 +262,19 @@ pub fn reallocate(
old_size.min(new_size),
/*nonoverlapping*/ true,
)?;
self.deallocate(ptr.into(), old_size_and_align, kind)?;
self.deallocate_ptr(ptr.into(), old_size_and_align, kind)?;
Ok(new_ptr)
}
#[instrument(skip(self), level = "debug")]
pub fn deallocate(
pub fn deallocate_ptr(
&mut self,
ptr: Pointer<Option<M::PointerTag>>,
old_size_and_align: Option<(Size, Align)>,
kind: MemoryKind<M::MemoryKind>,
) -> InterpResult<'tcx> {
let (alloc_id, offset, ptr) = self.ptr_get_alloc(ptr)?;
let (alloc_id, offset, ptr) = self.ptr_get_alloc_id(ptr)?;
trace!("deallocating: {}", alloc_id);
if offset.bytes() != 0 {
@ -292,7 +284,7 @@ pub fn deallocate(
);
}
let Some((alloc_kind, mut alloc)) = self.alloc_map.remove(&alloc_id) else {
let Some((alloc_kind, mut alloc)) = self.memory.alloc_map.remove(&alloc_id) else {
// Deallocating global memory -- always an error
return Err(match self.tcx.get_global_alloc(alloc_id) {
Some(GlobalAlloc::Function(..)) => {
@ -335,14 +327,14 @@ pub fn deallocate(
// Let the machine take some extra action
let size = alloc.size();
M::memory_deallocated(
&mut self.extra,
&mut self.machine,
&mut alloc.extra,
ptr.provenance,
alloc_range(Size::ZERO, size),
)?;
// Don't forget to remember size and align of this now-dead allocation
let old = self.dead_alloc_map.insert(alloc_id, (size, alloc.align));
let old = self.memory.dead_alloc_map.insert(alloc_id, (size, alloc.align));
if old.is_some() {
bug!("Nothing can be deallocated twice");
}
@ -358,7 +350,7 @@ fn get_ptr_access(
size: Size,
align: Align,
) -> InterpResult<'tcx, Option<(AllocId, Size, Pointer<M::PointerTag>)>> {
let align = M::enforce_alignment(&self.extra).then_some(align);
let align = M::enforce_alignment(&self).then_some(align);
self.check_and_deref_ptr(
ptr,
size,
@ -366,7 +358,7 @@ fn get_ptr_access(
CheckInAllocMsg::MemoryAccessTest,
|alloc_id, offset, ptr| {
let (size, align) =
self.get_size_and_align(alloc_id, AllocCheck::Dereferenceable)?;
self.get_alloc_size_and_align(alloc_id, AllocCheck::Dereferenceable)?;
Ok((size, align, (alloc_id, offset, ptr)))
},
)
@ -392,7 +384,7 @@ pub fn check_ptr_access_align(
| CheckInAllocMsg::OffsetFromTest
| CheckInAllocMsg::InboundsTest => AllocCheck::Live,
};
let (size, align) = self.get_size_and_align(alloc_id, check)?;
let (size, align) = self.get_alloc_size_and_align(alloc_id, check)?;
Ok((size, align, ()))
})?;
Ok(())
@ -427,7 +419,7 @@ fn check_offset_align(offset: u64, align: Align) -> InterpResult<'static> {
}
}
Ok(match self.ptr_try_get_alloc(ptr) {
Ok(match self.ptr_try_get_alloc_id(ptr) {
Err(addr) => {
// We couldn't get a proper allocation. This is only okay if the access size is 0,
// and the address is not null.
@ -456,7 +448,7 @@ fn check_offset_align(offset: u64, align: Align) -> InterpResult<'static> {
// Test align. Check this last; if both bounds and alignment are violated
// we want the error to be about the bounds.
if let Some(align) = align {
if M::force_int_for_alignment_check(&self.extra) {
if M::force_int_for_alignment_check(self) {
let addr = Scalar::from_pointer(ptr, &self.tcx)
.to_machine_usize(&self.tcx)
.expect("ptr-to-int cast for align check should never fail");
@ -479,7 +471,7 @@ fn check_offset_align(offset: u64, align: Align) -> InterpResult<'static> {
}
/// Allocation accessors
impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Helper function to obtain a global (tcx) allocation.
/// This attempts to return a reference to an existing allocation if
/// one can be found in `tcx`. That, however, is only possible if `tcx` and
@ -517,7 +509,7 @@ fn get_global_alloc(
(self.tcx.eval_static_initializer(def_id)?, Some(def_id))
}
};
M::before_access_global(&self.extra, id, alloc, def_id, is_write)?;
M::before_access_global(&self.machine, id, alloc, def_id, is_write)?;
// We got tcx memory. Let the machine initialize its "extra" stuff.
let alloc = M::init_allocation_extra(
self,
@ -530,7 +522,7 @@ fn get_global_alloc(
/// Gives raw access to the `Allocation`, without bounds or alignment checks.
/// The caller is responsible for calling the access hooks!
fn get_raw(
fn get_alloc_raw(
&self,
id: AllocId,
) -> InterpResult<'tcx, &Allocation<M::PointerTag, M::AllocExtra>> {
@ -538,7 +530,7 @@ fn get_raw(
// ways of "erroring": An actual error, or because we got a reference from
// `get_global_alloc` that we can actually use directly without inserting anything anywhere.
// So the error type is `InterpResult<'tcx, &Allocation<M::PointerTag>>`.
let a = self.alloc_map.get_or(id, || {
let a = self.memory.alloc_map.get_or(id, || {
let alloc = self.get_global_alloc(id, /*is_write*/ false).map_err(Err)?;
match alloc {
Cow::Borrowed(alloc) => {
@ -564,27 +556,27 @@ fn get_raw(
}
/// "Safe" (bounds and align-checked) allocation access.
pub fn get<'a>(
pub fn get_ptr_alloc<'a>(
&'a self,
ptr: Pointer<Option<M::PointerTag>>,
size: Size,
align: Align,
) -> InterpResult<'tcx, Option<AllocRef<'a, 'tcx, M::PointerTag, M::AllocExtra>>> {
let align = M::enforce_alignment(&self.extra).then_some(align);
let align = M::enforce_alignment(self).then_some(align);
let ptr_and_alloc = self.check_and_deref_ptr(
ptr,
size,
align,
CheckInAllocMsg::MemoryAccessTest,
|alloc_id, offset, ptr| {
let alloc = self.get_raw(alloc_id)?;
let alloc = self.get_alloc_raw(alloc_id)?;
Ok((alloc.size(), alloc.align, (alloc_id, offset, ptr, alloc)))
},
)?;
if let Some((alloc_id, offset, ptr, alloc)) = ptr_and_alloc {
let range = alloc_range(offset, size);
M::memory_read(&self.extra, &alloc.extra, ptr.provenance, range)?;
Ok(Some(AllocRef { alloc, range, tcx: self.tcx, alloc_id }))
M::memory_read(&self.machine, &alloc.extra, ptr.provenance, range)?;
Ok(Some(AllocRef { alloc, range, tcx: *self.tcx, alloc_id }))
} else {
// Even in this branch we have to be sure that we actually access the allocation, in
// order to ensure that `static FOO: Type = FOO;` causes a cycle error instead of
@ -596,7 +588,7 @@ pub fn get<'a>(
/// Return the `extra` field of the given allocation.
pub fn get_alloc_extra<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, &'a M::AllocExtra> {
Ok(&self.get_raw(id)?.extra)
Ok(&self.get_alloc_raw(id)?.extra)
}
/// Gives raw mutable access to the `Allocation`, without bounds or alignment checks.
@ -604,16 +596,15 @@ pub fn get_alloc_extra<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, &'a M::A
///
/// Also returns a ptr to `self.extra` so that the caller can use it in parallel with the
/// allocation.
fn get_raw_mut(
fn get_alloc_raw_mut(
&mut self,
id: AllocId,
) -> InterpResult<'tcx, (&mut Allocation<M::PointerTag, M::AllocExtra>, &mut M::MemoryExtra)>
{
) -> InterpResult<'tcx, (&mut Allocation<M::PointerTag, M::AllocExtra>, &mut M)> {
// We have "NLL problem case #3" here, which cannot be worked around without loss of
// efficiency even for the common case where the key is in the map.
// <https://rust-lang.github.io/rfcs/2094-nll.html#problem-case-3-conditional-control-flow-across-functions>
// (Cannot use `get_mut_or` since `get_global_alloc` needs `&self`.)
if self.alloc_map.get_mut(id).is_none() {
if self.memory.alloc_map.get_mut(id).is_none() {
// Slow path.
// Allocation not found locally, go look global.
let alloc = self.get_global_alloc(id, /*is_write*/ true)?;
@ -621,18 +612,18 @@ fn get_raw_mut(
"I got a global allocation that I have to copy but the machine does \
not expect that to happen",
);
self.alloc_map.insert(id, (MemoryKind::Machine(kind), alloc.into_owned()));
self.memory.alloc_map.insert(id, (MemoryKind::Machine(kind), alloc.into_owned()));
}
let (_kind, alloc) = self.alloc_map.get_mut(id).unwrap();
let (_kind, alloc) = self.memory.alloc_map.get_mut(id).unwrap();
if alloc.mutability == Mutability::Not {
throw_ub!(WriteToReadOnly(id))
}
Ok((alloc, &mut self.extra))
Ok((alloc, &mut self.machine))
}
/// "Safe" (bounds and align-checked) allocation access.
pub fn get_mut<'a>(
pub fn get_ptr_alloc_mut<'a>(
&'a mut self,
ptr: Pointer<Option<M::PointerTag>>,
size: Size,
@ -640,12 +631,12 @@ pub fn get_mut<'a>(
) -> InterpResult<'tcx, Option<AllocRefMut<'a, 'tcx, M::PointerTag, M::AllocExtra>>> {
let parts = self.get_ptr_access(ptr, size, align)?;
if let Some((alloc_id, offset, ptr)) = parts {
let tcx = self.tcx;
let tcx = *self.tcx;
// FIXME: can we somehow avoid looking up the allocation twice here?
// We cannot call `get_raw_mut` inside `check_and_deref_ptr` as that would duplicate `&mut self`.
let (alloc, extra) = self.get_raw_mut(alloc_id)?;
let (alloc, machine) = self.get_alloc_raw_mut(alloc_id)?;
let range = alloc_range(offset, size);
M::memory_written(extra, &mut alloc.extra, ptr.provenance, range)?;
M::memory_written(machine, &mut alloc.extra, ptr.provenance, range)?;
Ok(Some(AllocRefMut { alloc, range, tcx, alloc_id }))
} else {
Ok(None)
@ -656,16 +647,16 @@ pub fn get_mut<'a>(
pub fn get_alloc_extra_mut<'a>(
&'a mut self,
id: AllocId,
) -> InterpResult<'tcx, (&'a mut M::AllocExtra, &'a mut M::MemoryExtra)> {
let (alloc, memory_extra) = self.get_raw_mut(id)?;
Ok((&mut alloc.extra, memory_extra))
) -> InterpResult<'tcx, (&'a mut M::AllocExtra, &'a mut M)> {
let (alloc, machine) = self.get_alloc_raw_mut(id)?;
Ok((&mut alloc.extra, machine))
}
/// Obtain the size and alignment of an allocation, even if that allocation has
/// been deallocated.
///
/// If `liveness` is `AllocCheck::MaybeDead`, this function always returns `Ok`.
pub fn get_size_and_align(
pub fn get_alloc_size_and_align(
&self,
id: AllocId,
liveness: AllocCheck,
@ -674,7 +665,7 @@ pub fn get_size_and_align(
// Don't use `self.get_raw` here as that will
// a) cause cycles in case `id` refers to a static
// b) duplicate a global's allocation in miri
if let Some((_, alloc)) = self.alloc_map.get(id) {
if let Some((_, alloc)) = self.memory.alloc_map.get(id) {
return Ok((alloc.size(), alloc.align));
}
@ -713,6 +704,7 @@ pub fn get_size_and_align(
// Deallocated pointers are allowed, we should be able to find
// them in the map.
Ok(*self
.memory
.dead_alloc_map
.get(&id)
.expect("deallocated pointers should all be recorded in `dead_alloc_map`"))
@ -724,7 +716,7 @@ pub fn get_size_and_align(
}
fn get_fn_alloc(&self, id: AllocId) -> Option<FnVal<'tcx, M::ExtraFnVal>> {
if let Some(extra) = self.extra_fn_ptr_map.get(&id) {
if let Some(extra) = self.memory.extra_fn_ptr_map.get(&id) {
Some(FnVal::Other(*extra))
} else {
match self.tcx.get_global_alloc(id) {
@ -734,12 +726,12 @@ fn get_fn_alloc(&self, id: AllocId) -> Option<FnVal<'tcx, M::ExtraFnVal>> {
}
}
pub fn get_fn(
pub fn get_ptr_fn(
&self,
ptr: Pointer<Option<M::PointerTag>>,
) -> InterpResult<'tcx, FnVal<'tcx, M::ExtraFnVal>> {
trace!("get_fn({:?})", ptr);
let (alloc_id, offset, _ptr) = self.ptr_get_alloc(ptr)?;
let (alloc_id, offset, _ptr) = self.ptr_get_alloc_id(ptr)?;
if offset.bytes() != 0 {
throw_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset)))
}
@ -747,8 +739,8 @@ pub fn get_fn(
.ok_or_else(|| err_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset))).into())
}
pub fn mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx> {
self.get_raw_mut(id)?.0.mutability = Mutability::Not;
pub fn alloc_mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx> {
self.get_alloc_raw_mut(id)?.0.mutability = Mutability::Not;
Ok(())
}
@ -765,7 +757,7 @@ pub fn dump_alloc<'a>(&'a self, id: AllocId) -> DumpAllocs<'a, 'mir, 'tcx, M> {
pub fn dump_allocs<'a>(&'a self, mut allocs: Vec<AllocId>) -> DumpAllocs<'a, 'mir, 'tcx, M> {
allocs.sort();
allocs.dedup();
DumpAllocs { mem: self, allocs }
DumpAllocs { ecx: self, allocs }
}
/// Print leaked memory. Allocations reachable from `static_roots` or a `Global` allocation
@ -775,14 +767,15 @@ pub fn leak_report(&self, static_roots: &[AllocId]) -> usize {
let reachable = {
let mut reachable = FxHashSet::default();
let global_kind = M::GLOBAL_KIND.map(MemoryKind::Machine);
let mut todo: Vec<_> = self.alloc_map.filter_map_collect(move |&id, &(kind, _)| {
if Some(kind) == global_kind { Some(id) } else { None }
});
let mut todo: Vec<_> =
self.memory.alloc_map.filter_map_collect(move |&id, &(kind, _)| {
if Some(kind) == global_kind { Some(id) } else { None }
});
todo.extend(static_roots);
while let Some(id) = todo.pop() {
if reachable.insert(id) {
// This is a new allocation, add its relocations to `todo`.
if let Some((_, alloc)) = self.alloc_map.get(id) {
if let Some((_, alloc)) = self.memory.alloc_map.get(id) {
todo.extend(alloc.relocations().values().map(|tag| tag.get_alloc_id()));
}
}
@ -791,7 +784,7 @@ pub fn leak_report(&self, static_roots: &[AllocId]) -> usize {
};
// All allocations that are *not* `reachable` and *not* `may_leak` are considered leaking.
let leaks: Vec<_> = self.alloc_map.filter_map_collect(|&id, &(kind, _)| {
let leaks: Vec<_> = self.memory.alloc_map.filter_map_collect(|&id, &(kind, _)| {
if kind.may_leak() || reachable.contains(&id) { None } else { Some(id) }
});
let n = leaks.len();
@ -800,17 +793,12 @@ pub fn leak_report(&self, static_roots: &[AllocId]) -> usize {
}
n
}
/// This is used by [priroda](https://github.com/oli-obk/priroda)
pub fn alloc_map(&self) -> &M::MemoryMap {
&self.alloc_map
}
}
#[doc(hidden)]
/// There's no way to use this directly, it's just a helper struct for the `dump_alloc(s)` methods.
pub struct DumpAllocs<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> {
mem: &'a Memory<'mir, 'tcx, M>,
ecx: &'a InterpCx<'mir, 'tcx, M>,
allocs: Vec<AllocId>,
}
@ -840,25 +828,25 @@ fn write_allocation_track_relocs<'tcx, Tag: Provenance, Extra>(
}
write!(fmt, "{}", id)?;
match self.mem.alloc_map.get(id) {
match self.ecx.memory.alloc_map.get(id) {
Some(&(kind, ref alloc)) => {
// normal alloc
write!(fmt, " ({}, ", kind)?;
write_allocation_track_relocs(
&mut *fmt,
self.mem.tcx,
*self.ecx.tcx,
&mut allocs_to_print,
alloc,
)?;
}
None => {
// global alloc
match self.mem.tcx.get_global_alloc(id) {
match self.ecx.tcx.get_global_alloc(id) {
Some(GlobalAlloc::Memory(alloc)) => {
write!(fmt, " (unchanged global, ")?;
write_allocation_track_relocs(
&mut *fmt,
self.mem.tcx,
*self.ecx.tcx,
&mut allocs_to_print,
alloc.inner(),
)?;
@ -867,7 +855,7 @@ fn write_allocation_track_relocs<'tcx, Tag: Provenance, Extra>(
write!(fmt, " (fn: {})", func)?;
}
Some(GlobalAlloc::Static(did)) => {
write!(fmt, " (static: {})", self.mem.tcx.def_path_str(did))?;
write!(fmt, " (static: {})", self.ecx.tcx.def_path_str(did))?;
}
None => {
write!(fmt, " (deallocated)")?;
@ -923,16 +911,16 @@ pub fn check_bytes(&self, range: AllocRange, allow_uninit_and_ptr: bool) -> Inte
}
}
impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Reads the given number of bytes from memory. Returns them as a slice.
///
/// Performs appropriate bounds checks.
pub fn read_bytes(
pub fn read_bytes_ptr(
&self,
ptr: Pointer<Option<M::PointerTag>>,
size: Size,
) -> InterpResult<'tcx, &[u8]> {
let Some(alloc_ref) = self.get(ptr, size, Align::ONE)? else {
let Some(alloc_ref) = self.get_ptr_alloc(ptr, size, Align::ONE)? else {
// zero-sized access
return Ok(&[]);
};
@ -947,7 +935,7 @@ pub fn read_bytes(
/// Writes the given stream of bytes into memory.
///
/// Performs appropriate bounds checks.
pub fn write_bytes(
pub fn write_bytes_ptr(
&mut self,
ptr: Pointer<Option<M::PointerTag>>,
src: impl IntoIterator<Item = u8>,
@ -958,7 +946,7 @@ pub fn write_bytes(
assert_eq!(lower, len, "can only write iterators with a precise length");
let size = Size::from_bytes(len);
let Some(alloc_ref) = self.get_mut(ptr, size, Align::ONE)? else {
let Some(alloc_ref) = self.get_ptr_alloc_mut(ptr, size, Align::ONE)? else {
// zero-sized access
assert_matches!(
src.next(),
@ -984,7 +972,7 @@ pub fn write_bytes(
Ok(())
}
pub fn copy(
pub fn mem_copy(
&mut self,
src: Pointer<Option<M::PointerTag>>,
src_align: Align,
@ -993,10 +981,10 @@ pub fn copy(
size: Size,
nonoverlapping: bool,
) -> InterpResult<'tcx> {
self.copy_repeatedly(src, src_align, dest, dest_align, size, 1, nonoverlapping)
self.mem_copy_repeatedly(src, src_align, dest, dest_align, size, 1, nonoverlapping)
}
pub fn copy_repeatedly(
pub fn mem_copy_repeatedly(
&mut self,
src: Pointer<Option<M::PointerTag>>,
src_align: Align,
@ -1019,9 +1007,9 @@ pub fn copy_repeatedly(
// Zero-sized *source*, that means dst is also zero-sized and we have nothing to do.
return Ok(());
};
let src_alloc = self.get_raw(src_alloc_id)?;
let src_alloc = self.get_alloc_raw(src_alloc_id)?;
let src_range = alloc_range(src_offset, size);
M::memory_read(&self.extra, &src_alloc.extra, src.provenance, src_range)?;
M::memory_read(&self.machine, &src_alloc.extra, src.provenance, src_range)?;
// We need the `dest` ptr for the next operation, so we get it now.
// We already did the source checks and called the hooks so we are good to return early.
let Some((dest_alloc_id, dest_offset, dest)) = dest_parts else {
@ -1044,7 +1032,7 @@ pub fn copy_repeatedly(
let compressed = src_alloc.compress_uninit_range(src_range);
// Destination alloc preparations and access hooks.
let (dest_alloc, extra) = self.get_raw_mut(dest_alloc_id)?;
let (dest_alloc, extra) = self.get_alloc_raw_mut(dest_alloc_id)?;
let dest_range = alloc_range(dest_offset, size * num_copies);
M::memory_written(extra, &mut dest_alloc.extra, dest.provenance, dest_range)?;
let dest_bytes = dest_alloc
@ -1112,7 +1100,7 @@ pub fn copy_repeatedly(
}
/// Machine pointer introspection.
impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub fn scalar_to_ptr(&self, scalar: Scalar<M::PointerTag>) -> Pointer<Option<M::PointerTag>> {
// We use `to_bits_or_ptr_internal` since we are just implementing the method people need to
// call to force getting out a pointer.
@ -1129,9 +1117,32 @@ pub fn scalar_to_ptr(&self, scalar: Scalar<M::PointerTag>) -> Pointer<Option<M::
}
}
/// Test if this value might be null.
/// If the machine does not support ptr-to-int casts, this is conservative.
pub fn scalar_may_be_null(&self, scalar: Scalar<M::PointerTag>) -> bool {
match scalar.try_to_int() {
Ok(int) => int.is_null(),
Err(_) => {
// Can only happen during CTFE.
let ptr = self.scalar_to_ptr(scalar);
match self.ptr_try_get_alloc_id(ptr) {
Ok((alloc_id, offset, _)) => {
let (size, _align) = self
.get_alloc_size_and_align(alloc_id, AllocCheck::MaybeDead)
.expect("alloc info with MaybeDead cannot fail");
// If the pointer is out-of-bounds, it may be null.
// Note that one-past-the-end (offset == size) is still inbounds, and never null.
offset > size
}
Err(_offset) => bug!("a non-int scalar is always a pointer"),
}
}
}
}
/// Turning a "maybe pointer" into a proper pointer (and some information
/// about where it points), or an absolute address.
pub fn ptr_try_get_alloc(
pub fn ptr_try_get_alloc_id(
&self,
ptr: Pointer<Option<M::PointerTag>>,
) -> Result<(AllocId, Size, Pointer<M::PointerTag>), u64> {
@ -1146,11 +1157,11 @@ pub fn ptr_try_get_alloc(
/// Turning a "maybe pointer" into a proper pointer (and some information about where it points).
#[inline(always)]
pub fn ptr_get_alloc(
pub fn ptr_get_alloc_id(
&self,
ptr: Pointer<Option<M::PointerTag>>,
) -> InterpResult<'tcx, (AllocId, Size, Pointer<M::PointerTag>)> {
self.ptr_try_get_alloc(ptr).map_err(|offset| {
self.ptr_try_get_alloc_id(ptr).map_err(|offset| {
err_ub!(DanglingIntPointer(offset, CheckInAllocMsg::InboundsTest)).into()
})
}

4
compiler/rustc_const_eval/src/interpret/operand.rs
View File

@ -257,7 +257,7 @@ fn try_read_immediate_from_mplace(
return Ok(None);
}
let Some(alloc) = self.get_alloc(mplace)? else {
let Some(alloc) = self.get_place_alloc(mplace)? else {
return Ok(Some(ImmTy {
// zero-sized type
imm: Scalar::ZST.into(),
@ -340,7 +340,7 @@ pub fn read_pointer(
// Turn the wide MPlace into a string (must already be dereferenced!)
pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::PointerTag>) -> InterpResult<'tcx, &str> {
let len = mplace.len(self)?;
let bytes = self.memory.read_bytes(mplace.ptr, Size::from_bytes(len))?;
let bytes = self.read_bytes_ptr(mplace.ptr, Size::from_bytes(len))?;
let str = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?;
Ok(str)
}

23
compiler/rustc_const_eval/src/interpret/place.rs
View File

@ -306,25 +306,25 @@ pub fn deref_operand(
}
#[inline]
pub(super) fn get_alloc(
pub(super) fn get_place_alloc(
&self,
place: &MPlaceTy<'tcx, M::PointerTag>,
) -> InterpResult<'tcx, Option<AllocRef<'_, 'tcx, M::PointerTag, M::AllocExtra>>> {
assert!(!place.layout.is_unsized());
assert!(!place.meta.has_meta());
let size = place.layout.size;
self.memory.get(place.ptr, size, place.align)
self.get_ptr_alloc(place.ptr, size, place.align)
}
#[inline]
pub(super) fn get_alloc_mut(
pub(super) fn get_place_alloc_mut(
&mut self,
place: &MPlaceTy<'tcx, M::PointerTag>,
) -> InterpResult<'tcx, Option<AllocRefMut<'_, 'tcx, M::PointerTag, M::AllocExtra>>> {
assert!(!place.layout.is_unsized());
assert!(!place.meta.has_meta());
let size = place.layout.size;
self.memory.get_mut(place.ptr, size, place.align)
self.get_ptr_alloc_mut(place.ptr, size, place.align)
}
/// Check if this mplace is dereferenceable and sufficiently aligned.
@ -337,8 +337,8 @@ fn check_mplace_access(
.size_and_align_of_mplace(&mplace)?
.unwrap_or((mplace.layout.size, mplace.layout.align.abi));
assert!(mplace.mplace.align <= align, "dynamic alignment less strict than static one?");
let align = M::enforce_alignment(&self.memory.extra).then_some(align);
self.memory.check_ptr_access_align(mplace.ptr, size, align.unwrap_or(Align::ONE), msg)?;
let align = M::enforce_alignment(self).then_some(align);
self.check_ptr_access_align(mplace.ptr, size, align.unwrap_or(Align::ONE), msg)?;
Ok(())
}
@ -748,7 +748,7 @@ fn write_immediate_to_mplace_no_validate(
// Invalid places are a thing: the return place of a diverging function
let tcx = *self.tcx;
let Some(mut alloc) = self.get_alloc_mut(dest)? else {
let Some(mut alloc) = self.get_place_alloc_mut(dest)? else {
// zero-sized access
return Ok(());
};
@ -857,8 +857,7 @@ fn copy_op_no_validate(
});
assert_eq!(src.meta, dest.meta, "Can only copy between equally-sized instances");
self.memory
.copy(src.ptr, src.align, dest.ptr, dest.align, size, /*nonoverlapping*/ true)
self.mem_copy(src.ptr, src.align, dest.ptr, dest.align, size, /*nonoverlapping*/ true)
}
/// Copies the data from an operand to a place. The layouts may disagree, but they must
@ -942,7 +941,7 @@ pub fn force_allocation_maybe_sized(
let (size, align) = self
.size_and_align_of(&meta, &local_layout)?
.expect("Cannot allocate for non-dyn-sized type");
let ptr = self.memory.allocate(size, align, MemoryKind::Stack)?;
let ptr = self.allocate_ptr(size, align, MemoryKind::Stack)?;
let mplace = MemPlace { ptr: ptr.into(), align, meta };
if let LocalValue::Live(Operand::Immediate(value)) = local_val {
// Preserve old value.
@ -979,7 +978,7 @@ pub fn allocate(
layout: TyAndLayout<'tcx>,
kind: MemoryKind<M::MemoryKind>,
) -> InterpResult<'static, MPlaceTy<'tcx, M::PointerTag>> {
let ptr = self.memory.allocate(layout.size, layout.align.abi, kind)?;
let ptr = self.allocate_ptr(layout.size, layout.align.abi, kind)?;
Ok(MPlaceTy::from_aligned_ptr(ptr.into(), layout))
}
@ -990,7 +989,7 @@ pub fn allocate_str(
kind: MemoryKind<M::MemoryKind>,
mutbl: Mutability,
) -> MPlaceTy<'tcx, M::PointerTag> {
let ptr = self.memory.allocate_bytes(str.as_bytes(), Align::ONE, kind, mutbl);
let ptr = self.allocate_bytes_ptr(str.as_bytes(), Align::ONE, kind, mutbl);
let meta = Scalar::from_machine_usize(u64::try_from(str.len()).unwrap(), self);
let mplace =
MemPlace { ptr: ptr.into(), align: Align::ONE, meta: MemPlaceMeta::Meta(meta) };

4
compiler/rustc_const_eval/src/interpret/step.rs
View File

@ -225,7 +225,7 @@ pub fn eval_rvalue_into_place(
if length == 0 {
// Nothing to copy... but let's still make sure that `dest` as a place is valid.
self.get_alloc_mut(&dest)?;
self.get_place_alloc_mut(&dest)?;
} else {
// Write the src to the first element.
let first = self.mplace_field(&dest, 0)?;
@ -241,7 +241,7 @@ pub fn eval_rvalue_into_place(
// that place might be more aligned than its type mandates (a `u8` array could
// be 4-aligned if it sits at the right spot in a struct). Instead we use
// `first.layout.align`, i.e., the alignment given by the type.
self.memory.copy_repeatedly(
self.mem_copy_repeatedly(
first_ptr,
first.align,
rest_ptr,

2
compiler/rustc_const_eval/src/interpret/terminator.rs
View File

@ -72,7 +72,7 @@ pub(super) fn eval_terminator(
let (fn_val, fn_abi, with_caller_location) = match *func.layout.ty.kind() {
ty::FnPtr(_sig) => {
let fn_ptr = self.read_pointer(&func)?;
let fn_val = self.memory.get_fn(fn_ptr)?;
let fn_val = self.get_ptr_fn(fn_ptr)?;
(fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false)
}
ty::FnDef(def_id, substs) => {

18
compiler/rustc_const_eval/src/interpret/traits.rs
View File

@ -32,7 +32,7 @@ pub fn get_vtable(
let vtable_allocation = self.tcx.vtable_allocation((ty, poly_trait_ref));
let vtable_ptr = self.memory.global_base_pointer(Pointer::from(vtable_allocation))?;
let vtable_ptr = self.global_base_pointer(Pointer::from(vtable_allocation))?;
Ok(vtable_ptr.into())
}
@ -48,11 +48,10 @@ pub fn get_vtable_slot(
let ptr_size = self.pointer_size();
let vtable_slot = vtable.offset(ptr_size * idx, self)?;
let vtable_slot = self
.memory
.get(vtable_slot, ptr_size, self.tcx.data_layout.pointer_align.abi)?
.get_ptr_alloc(vtable_slot, ptr_size, self.tcx.data_layout.pointer_align.abi)?
.expect("cannot be a ZST");
let fn_ptr = self.scalar_to_ptr(vtable_slot.read_ptr_sized(Size::ZERO)?.check_init()?);
self.memory.get_fn(fn_ptr)
self.get_ptr_fn(fn_ptr)
}
/// Returns the drop fn instance as well as the actual dynamic type.
@ -63,8 +62,7 @@ pub fn read_drop_type_from_vtable(
let pointer_size = self.pointer_size();
// We don't care about the pointee type; we just want a pointer.
let vtable = self
.memory
.get(
.get_ptr_alloc(
vtable,
pointer_size * u64::try_from(COMMON_VTABLE_ENTRIES.len()).unwrap(),
self.tcx.data_layout.pointer_align.abi,
@ -77,7 +75,7 @@ pub fn read_drop_type_from_vtable(
.check_init()?;
// We *need* an instance here, no other kind of function value, to be able
// to determine the type.
let drop_instance = self.memory.get_fn(self.scalar_to_ptr(drop_fn))?.as_instance()?;
let drop_instance = self.get_ptr_fn(self.scalar_to_ptr(drop_fn))?.as_instance()?;
trace!("Found drop fn: {:?}", drop_instance);
let fn_sig = drop_instance.ty(*self.tcx, self.param_env).fn_sig(*self.tcx);
let fn_sig = self.tcx.normalize_erasing_late_bound_regions(self.param_env, fn_sig);
@ -99,8 +97,7 @@ pub fn read_size_and_align_from_vtable(
// We check for `size = 3 * ptr_size`, which covers the drop fn (unused here),
// the size, and the align (which we read below).
let vtable = self
.memory
.get(
.get_ptr_alloc(
vtable,
pointer_size * u64::try_from(COMMON_VTABLE_ENTRIES.len()).unwrap(),
self.tcx.data_layout.pointer_align.abi,
@ -132,8 +129,7 @@ pub fn read_new_vtable_after_trait_upcasting_from_vtable(
let vtable_slot = vtable.offset(pointer_size * idx, self)?;
let new_vtable = self
.memory
.get(vtable_slot, pointer_size, self.tcx.data_layout.pointer_align.abi)?
.get_ptr_alloc(vtable_slot, pointer_size, self.tcx.data_layout.pointer_align.abi)?
.expect("cannot be a ZST");
let new_vtable = self.scalar_to_ptr(new_vtable.read_ptr_sized(Size::ZERO)?.check_init()?);

12
compiler/rustc_const_eval/src/interpret/validity.rs
View File

@ -315,7 +315,7 @@ fn check_wide_ptr_meta(
let vtable = self.ecx.scalar_to_ptr(meta.unwrap_meta());
// Direct call to `check_ptr_access_align` checks alignment even on CTFE machines.
try_validation!(
self.ecx.memory.check_ptr_access_align(
self.ecx.check_ptr_access_align(
vtable,
3 * self.ecx.tcx.data_layout.pointer_size, // drop, size, align
self.ecx.tcx.data_layout.pointer_align.abi,
@ -403,7 +403,7 @@ fn check_safe_pointer(
.unwrap_or_else(|| (place.layout.size, place.layout.align.abi));
// Direct call to `check_ptr_access_align` checks alignment even on CTFE machines.
try_validation!(
self.ecx.memory.check_ptr_access_align(
self.ecx.check_ptr_access_align(
place.ptr,
size,
align,
@ -432,7 +432,7 @@ fn check_safe_pointer(
if let Some(ref mut ref_tracking) = self.ref_tracking {
// Proceed recursively even for ZST, no reason to skip them!
// `!` is a ZST and we want to validate it.
if let Ok((alloc_id, _offset, _ptr)) = self.ecx.memory.ptr_try_get_alloc(place.ptr) {
if let Ok((alloc_id, _offset, _ptr)) = self.ecx.ptr_try_get_alloc_id(place.ptr) {
// Special handling for pointers to statics (irrespective of their type).
let alloc_kind = self.ecx.tcx.get_global_alloc(alloc_id);
if let Some(GlobalAlloc::Static(did)) = alloc_kind {
@ -579,7 +579,7 @@ fn try_visit_primitive(
if let Some(_) = self.ref_tracking {
let ptr = self.ecx.scalar_to_ptr(value);
let _fn = try_validation!(
self.ecx.memory.get_fn(ptr),
self.ecx.get_ptr_fn(ptr),
self.path,
err_ub!(DanglingIntPointer(0, _)) =>
{ "a null function pointer" },
@ -825,7 +825,7 @@ fn visit_aggregate(
let mplace = op.assert_mem_place(); // strings are never immediate
let len = mplace.len(self.ecx)?;
try_validation!(
self.ecx.memory.read_bytes(mplace.ptr, Size::from_bytes(len)),
self.ecx.read_bytes_ptr(mplace.ptr, Size::from_bytes(len)),
self.path,
err_ub!(InvalidUninitBytes(..)) => { "uninitialized data in `str`" },
err_unsup!(ReadPointerAsBytes) => { "a pointer in `str`" },
@ -861,7 +861,7 @@ fn visit_aggregate(
// to reject those pointers, we just do not have the machinery to
// talk about parts of a pointer.
// We also accept uninit, for consistency with the slow path.
let Some(alloc) = self.ecx.memory.get(mplace.ptr, size, mplace.align)? else {
let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, mplace.align)? else {
// Size 0, nothing more to check.
return Ok(());
};

5
compiler/rustc_mir_transform/src/const_prop.rs
View File

@ -184,8 +184,6 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
type MemoryKind = !;
type MemoryExtra = ();
fn load_mir(
_ecx: &InterpCx<'mir, 'tcx, Self>,
_instance: ty::InstanceDef<'tcx>,
@ -267,7 +265,7 @@ fn access_local_mut<'a>(
}
fn before_access_global(
_memory_extra: &(),
_machine: &Self,
_alloc_id: AllocId,
alloc: ConstAllocation<'tcx, Self::PointerTag, Self::AllocExtra>,
_static_def_id: Option<DefId>,
@ -377,7 +375,6 @@ fn new(
span,
param_env,
ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop),
(),
);
let ret = ecx

5
compiler/rustc_mir_transform/src/const_prop_lint.rs
View File

@ -180,8 +180,6 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
type MemoryKind = !;
type MemoryExtra = ();
fn load_mir(
_ecx: &InterpCx<'mir, 'tcx, Self>,
_instance: ty::InstanceDef<'tcx>,
@ -263,7 +261,7 @@ fn access_local_mut<'a>(
}
fn before_access_global(
_memory_extra: &(),
_machine: &Self,
_alloc_id: AllocId,
alloc: ConstAllocation<'tcx, Self::PointerTag, Self::AllocExtra>,
_static_def_id: Option<DefId>,
@ -374,7 +372,6 @@ fn new(
span,
param_env,
ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop),
(),
);
let ret = ecx