Auto merge of #115568 - matthiaskrgr:rollup-2igo8rl, r=matthiaskrgr

Rollup of 7 pull requests

Successful merges:

 - #113510 (Document soundness of Integer -> Pointer -> Integer conversions in `const` contexts.)
 - #114412 (document our assumptions about symbols provided by the libc)
 - #114813 (explain why we can mutate the FPU control word)
 - #115523 (improve `AttrTokenStream`)
 - #115536 (interpret: make MemPlace, Place, Operand types private to the interpreter)
 - #115540 (Support debuginfo for custom MIR.)
 - #115563 (llvm-wrapper: adapt for LLVM API change)

r? `@ghost`
`@rustbot` modify labels: rollup

compiler/rustc_ast/src/tokenstream.rs

@@ -213,14 +213,10 @@ pub fn to_tokenstream(&self) -> TokenStream {
                         .into_iter()
                 }
                 AttrTokenTree::Attributes(data) => {
-                    let mut outer_attrs = Vec::new();
+                    let idx = data
-                    let mut inner_attrs = Vec::new();
+                        .attrs
-                    for attr in &data.attrs {
+                        .partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer));
-                        match attr.style {
+                    let (outer_attrs, inner_attrs) = data.attrs.split_at(idx);
-                            crate::AttrStyle::Outer => outer_attrs.push(attr),
-                            crate::AttrStyle::Inner => inner_attrs.push(attr),
-                        }
-                    }
 
                     let mut target_tokens: Vec<_> = data
                         .tokens
@@ -265,10 +261,10 @@ pub fn to_tokenstream(&self) -> TokenStream {
                             "Failed to find trailing delimited group in: {target_tokens:?}"
                         );
                     }
-                    let mut flat: SmallVec<[_; 1]> = SmallVec::new();
+                    let mut flat: SmallVec<[_; 1]> =
+                        SmallVec::with_capacity(target_tokens.len() + outer_attrs.len());
                     for attr in outer_attrs {
-                        // FIXME: Make this more efficient
+                        flat.extend(attr.tokens().0.iter().cloned());
-                        flat.extend(attr.tokens().0.clone().iter().cloned());
                     }
                     flat.extend(target_tokens);
                     flat.into_iter()

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -79,7 +79,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
     intern_const_alloc_recursive(ecx, intern_kind, &ret)?;
     // we leave alignment checks off, since this `ecx` will not be used for further evaluation anyway
 
-    debug!("eval_body_using_ecx done: {:?}", *ret);
+    debug!("eval_body_using_ecx done: {:?}", ret);
     Ok(ret)
 }
 
@@ -147,7 +147,7 @@ pub(super) fn op_to_const<'tcx>(
     // We know `offset` is relative to the allocation, so we can use `into_parts`.
     let to_const_value = |mplace: &MPlaceTy<'_>| {
         debug!("to_const_value(mplace: {:?})", mplace);
-        match mplace.ptr.into_parts() {
+        match mplace.ptr().into_parts() {
             (Some(alloc_id), offset) => {
                 let alloc = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
                 ConstValue::ByRef { alloc, offset }
@@ -370,7 +370,7 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
                     inner = true;
                 }
             };
-            let alloc_id = mplace.ptr.provenance.unwrap();
+            let alloc_id = mplace.ptr().provenance.unwrap();
 
             // Validation failed, report an error. This is always a hard error.
             if let Err(error) = validation {

compiler/rustc_const_eval/src/const_eval/mod.rs

@@ -30,7 +30,7 @@ pub(crate) fn const_caller_location(
     if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &loc_place).is_err() {
         bug!("intern_const_alloc_recursive should not error in this case")
     }
-    ConstValue::Scalar(Scalar::from_maybe_pointer(loc_place.ptr, &tcx))
+    ConstValue::Scalar(Scalar::from_maybe_pointer(loc_place.ptr(), &tcx))
 }
 
 // We forbid type-level constants that contain more than `VALTREE_MAX_NODES` nodes.

compiler/rustc_const_eval/src/const_eval/valtrees.rs

@@ -5,7 +5,7 @@
 use crate::interpret::MPlaceTy;
 use crate::interpret::{
     intern_const_alloc_recursive, ConstValue, ImmTy, Immediate, InternKind, MemPlaceMeta,
-    MemoryKind, Place, Projectable, Scalar,
+    MemoryKind, PlaceTy, Projectable, Scalar,
 };
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt};
@@ -318,7 +318,7 @@ fn valtree_into_mplace<'tcx>(
                     let len_scalar = Scalar::from_target_usize(len as u64, &tcx);
 
                     Immediate::ScalarPair(
-                        Scalar::from_maybe_pointer((*pointee_place).ptr, &tcx),
+                        Scalar::from_maybe_pointer(pointee_place.ptr(), &tcx),
                         len_scalar,
                     )
                 }
@@ -383,5 +383,5 @@ fn valtree_into_mplace<'tcx>(
 }
 
 fn dump_place<'tcx>(ecx: &CompileTimeEvalContext<'tcx, 'tcx>, place: &MPlaceTy<'tcx>) {
-    trace!("{:?}", ecx.dump_place(Place::Ptr(**place)));
+    trace!("{:?}", ecx.dump_place(&PlaceTy::from(place.clone())));
 }

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -21,8 +21,8 @@
 
 use super::{
     AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace,
-    MemPlaceMeta, Memory, MemoryKind, Operand, Place, PlaceTy, PointerArithmetic, Provenance,
+    MemPlaceMeta, Memory, MemoryKind, Operand, Place, PlaceTy, Pointer, PointerArithmetic,
-    Scalar, StackPopJump,
+    Projectable, Provenance, Scalar, StackPopJump,
 };
 use crate::errors::{self, ErroneousConstUsed};
 use crate::util;
@@ -155,17 +155,26 @@ pub enum StackPopCleanup {
 }
 
 /// State of a local variable including a memoized layout
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct LocalState<'tcx, Prov: Provenance = AllocId> {
-    pub value: LocalValue<Prov>,
+    value: LocalValue<Prov>,
     /// Don't modify if `Some`, this is only used to prevent computing the layout twice.
     /// Avoids computing the layout of locals that are never actually initialized.
-    pub layout: Cell<Option<TyAndLayout<'tcx>>>,
+    layout: Cell<Option<TyAndLayout<'tcx>>>,
+}
+
+impl<Prov: Provenance> std::fmt::Debug for LocalState<'_, Prov> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LocalState")
+            .field("value", &self.value)
+            .field("ty", &self.layout.get().map(|l| l.ty))
+            .finish()
+    }
 }
 
 /// Current value of a local variable
 #[derive(Copy, Clone, Debug)] // Miri debug-prints these
-pub enum LocalValue<Prov: Provenance = AllocId> {
+pub(super) enum LocalValue<Prov: Provenance = AllocId> {
     /// This local is not currently alive, and cannot be used at all.
     Dead,
     /// A normal, live local.
@@ -176,10 +185,27 @@ pub enum LocalValue<Prov: Provenance = AllocId> {
     Live(Operand<Prov>),
 }
 
-impl<'tcx, Prov: Provenance + 'static> LocalState<'tcx, Prov> {
+impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> {
+    pub fn make_live_uninit(&mut self) {
+        self.value = LocalValue::Live(Operand::Immediate(Immediate::Uninit));
+    }
+
+    /// This is a hack because Miri needs a way to visit all the provenance in a `LocalState`
+    /// without having a layout or `TyCtxt` available, and we want to keep the `Operand` type
+    /// private.
+    pub fn as_mplace_or_imm(
+        &self,
+    ) -> Option<Either<(Pointer<Option<Prov>>, MemPlaceMeta<Prov>), Immediate<Prov>>> {
+        match self.value {
+            LocalValue::Dead => None,
+            LocalValue::Live(Operand::Indirect(mplace)) => Some(Left((mplace.ptr, mplace.meta))),
+            LocalValue::Live(Operand::Immediate(imm)) => Some(Right(imm)),
+        }
+    }
+
     /// Read the local's value or error if the local is not yet live or not live anymore.
     #[inline(always)]
-    pub fn access(&self) -> InterpResult<'tcx, &Operand<Prov>> {
+    pub(super) fn access(&self) -> InterpResult<'tcx, &Operand<Prov>> {
         match &self.value {
             LocalValue::Dead => throw_ub!(DeadLocal), // could even be "invalid program"?
             LocalValue::Live(val) => Ok(val),
@@ -189,10 +215,10 @@ pub fn access(&self) -> InterpResult<'tcx, &Operand<Prov>> {
     /// Overwrite the local. If the local can be overwritten in place, return a reference
     /// to do so; otherwise return the `MemPlace` to consult instead.
     ///
-    /// Note: This may only be invoked from the `Machine::access_local_mut` hook and not from
+    /// Note: Before calling this, call the `before_access_local_mut` machine hook! You may be
-    /// anywhere else. You may be invalidating machine invariants if you do!
+    /// invalidating machine invariants otherwise!
     #[inline(always)]
-    pub fn access_mut(&mut self) -> InterpResult<'tcx, &mut Operand<Prov>> {
+    pub(super) fn access_mut(&mut self) -> InterpResult<'tcx, &mut Operand<Prov>> {
         match &mut self.value {
             LocalValue::Dead => throw_ub!(DeadLocal), // could even be "invalid program"?
             LocalValue::Live(val) => Ok(val),
@@ -694,7 +720,7 @@ pub fn size_and_align_of_mplace(
         &self,
         mplace: &MPlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, Option<(Size, Align)>> {
-        self.size_and_align_of(&mplace.meta, &mplace.layout)
+        self.size_and_align_of(&mplace.meta(), &mplace.layout)
     }
 
     #[instrument(skip(self, body, return_place, return_to_block), level = "debug")]
@@ -826,7 +852,7 @@ pub(super) fn pop_stack_frame(&mut self, unwinding: bool) -> InterpResult<'tcx>
                 .expect("return place should always be live");
             let dest = self.frame().return_place.clone();
             let err = self.copy_op(&op, &dest, /*allow_transmute*/ true);
-            trace!("return value: {:?}", self.dump_place(*dest));
+            trace!("return value: {:?}", self.dump_place(&dest));
             // We delay actually short-circuiting on this error until *after* the stack frame is
             // popped, since we want this error to be attributed to the caller, whose type defines
             // this transmute.
@@ -974,7 +1000,7 @@ fn is_very_trivially_sized(ty: Ty<'_>) -> bool {
             }
             // Need to allocate some memory, since `Immediate::Uninit` cannot be unsized.
             let dest_place = self.allocate_dyn(layout, MemoryKind::Stack, meta)?;
-            Operand::Indirect(*dest_place)
+            Operand::Indirect(*dest_place.mplace())
         } else {
             assert!(!meta.has_meta()); // we're dropping the metadata
             // Just make this an efficient immediate.
@@ -1068,8 +1094,11 @@ pub fn eval_global(
     }
 
     #[must_use]
-    pub fn dump_place(&self, place: Place<M::Provenance>) -> PlacePrinter<'_, 'mir, 'tcx, M> {
+    pub fn dump_place(
-        PlacePrinter { ecx: self, place }
+        &self,
+        place: &PlaceTy<'tcx, M::Provenance>,
+    ) -> PlacePrinter<'_, 'mir, 'tcx, M> {
+        PlacePrinter { ecx: self, place: *place.place() }
     }
 
     #[must_use]

compiler/rustc_const_eval/src/interpret/intern.rs

@@ -25,7 +25,7 @@
 
 use super::{
     AllocId, Allocation, ConstAllocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy,
-    ValueVisitor,
+    Projectable, ValueVisitor,
 };
 use crate::const_eval;
 use crate::errors::{DanglingPtrInFinal, UnsupportedUntypedPointer};
@@ -177,7 +177,7 @@ fn visit_value(&mut self, mplace: &MPlaceTy<'tcx>) -> InterpResult<'tcx> {
             if let ty::Dynamic(_, _, ty::Dyn) =
                 tcx.struct_tail_erasing_lifetimes(referenced_ty, self.ecx.param_env).kind()
             {
-                let ptr = mplace.meta.unwrap_meta().to_pointer(&tcx)?;
+                let ptr = mplace.meta().unwrap_meta().to_pointer(&tcx)?;
                 if let Some(alloc_id) = ptr.provenance {
                     // Explicitly choose const mode here, since vtables are immutable, even
                     // if the reference of the fat pointer is mutable.
@@ -191,7 +191,7 @@ fn visit_value(&mut self, mplace: &MPlaceTy<'tcx>) -> InterpResult<'tcx> {
             }
             // Check if we have encountered this pointer+layout combination before.
             // Only recurse for allocation-backed pointers.
-            if let Some(alloc_id) = mplace.ptr.provenance {
+            if let Some(alloc_id) = mplace.ptr().provenance {
                 // Compute the mode with which we intern this. Our goal here is to make as many
                 // statics as we can immutable so they can be placed in read-only memory by LLVM.
                 let ref_mode = match self.mode {
@@ -267,7 +267,7 @@ fn visit_value(&mut self, mplace: &MPlaceTy<'tcx>) -> InterpResult<'tcx> {
 
                     // If there is no provenance in this allocation, it does not contain references
                     // that point to another allocation, and we can avoid the interning walk.
-                    if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, align)? {
+                    if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr(), size, align)? {
                         if !alloc.has_provenance() {
                             return Ok(false);
                         }
@@ -353,7 +353,7 @@ pub fn intern_const_alloc_recursive<
         leftover_allocations,
         // The outermost allocation must exist, because we allocated it with
         // `Memory::allocate`.
-        ret.ptr.provenance.unwrap(),
+        ret.ptr().provenance.unwrap(),
         base_intern_mode,
         Some(ret.layout.ty),
     );
@@ -466,7 +466,7 @@ pub fn intern_with_temp_alloc(
     ) -> InterpResult<'tcx, ConstAllocation<'tcx>> {
         let dest = self.allocate(layout, MemoryKind::Stack)?;
         f(self, &dest.clone().into())?;
-        let mut alloc = self.memory.alloc_map.remove(&dest.ptr.provenance.unwrap()).unwrap().1;
+        let mut alloc = self.memory.alloc_map.remove(&dest.ptr().provenance.unwrap()).unwrap().1;
         alloc.mutability = Mutability::Not;
         Ok(self.tcx.mk_const_alloc(alloc))
     }

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -466,7 +466,7 @@ pub fn emulate_intrinsic(
             _ => return Ok(false),
         }
 
-        trace!("{:?}", self.dump_place(**dest));
+        trace!("{:?}", self.dump_place(dest));
         self.go_to_block(ret);
         Ok(true)
     }

compiler/rustc_const_eval/src/interpret/machine.rs

@@ -18,7 +18,7 @@
 
 use super::{
     AllocBytes, AllocId, AllocRange, Allocation, ConstAllocation, FnArg, Frame, ImmTy, InterpCx,
-    InterpResult, MPlaceTy, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar,
+    InterpResult, MPlaceTy, MemoryKind, OpTy, PlaceTy, Pointer, Provenance, Scalar,
 };
 
 /// Data returned by Machine::stack_pop,
@@ -237,22 +237,22 @@ fn binary_ptr_op(
         right: &ImmTy<'tcx, Self::Provenance>,
     ) -> InterpResult<'tcx, (Scalar<Self::Provenance>, bool, Ty<'tcx>)>;
 
-    /// Called to write the specified `local` from the `frame`.
+    /// Called before writing the specified `local` of the `frame`.
     /// Since writing a ZST is not actually accessing memory or locals, this is never invoked
     /// for ZST reads.
     ///
     /// Due to borrow checker trouble, we indicate the `frame` as an index rather than an `&mut
     /// Frame`.
-    #[inline]
+    #[inline(always)]
-    fn access_local_mut<'a>(
+    fn before_access_local_mut<'a>(
-        ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
+        _ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
-        frame: usize,
+        _frame: usize,
-        local: mir::Local,
+        _local: mir::Local,
-    ) -> InterpResult<'tcx, &'a mut Operand<Self::Provenance>>
+    ) -> InterpResult<'tcx>
     where
         'tcx: 'mir,
     {
-        ecx.stack_mut()[frame].locals[local].access_mut()
+        Ok(())
     }
 
     /// Called before a basic block terminator is executed.

compiler/rustc_const_eval/src/interpret/mod.rs

@@ -20,16 +20,21 @@
 
 pub use rustc_middle::mir::interpret::*; // have all the `interpret` symbols in one place: here
 
-pub use self::eval_context::{Frame, FrameInfo, InterpCx, LocalState, LocalValue, StackPopCleanup};
+pub use self::eval_context::{Frame, FrameInfo, InterpCx, StackPopCleanup};
 pub use self::intern::{intern_const_alloc_recursive, InternKind};
 pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackPopJump};
 pub use self::memory::{AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind};
-pub use self::operand::{ImmTy, Immediate, OpTy, Operand, Readable};
+pub use self::operand::{ImmTy, Immediate, OpTy, Readable};
-pub use self::place::{MPlaceTy, MemPlace, MemPlaceMeta, Place, PlaceTy, Writeable};
+pub use self::place::{MPlaceTy, MemPlaceMeta, PlaceTy, Writeable};
 pub use self::projection::Projectable;
 pub use self::terminator::FnArg;
 pub use self::validity::{CtfeValidationMode, RefTracking};
 pub use self::visitor::ValueVisitor;
 
+use self::{
+    operand::Operand,
+    place::{MemPlace, Place},
+};
+
 pub(crate) use self::intrinsics::eval_nullary_intrinsic;
 use eval_context::{from_known_layout, mir_assign_valid_types};

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -88,7 +88,7 @@ pub fn to_scalar_pair(self) -> (Scalar<Prov>, Scalar<Prov>) {
 
 // ScalarPair needs a type to interpret, so we often have an immediate and a type together
 // as input for binary and cast operations.
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct ImmTy<'tcx, Prov: Provenance = AllocId> {
     imm: Immediate<Prov>,
     pub layout: TyAndLayout<'tcx>,
@@ -134,6 +134,12 @@ fn p<'a, 'tcx, Prov: Provenance>(
     }
 }
 
+impl<Prov: Provenance> std::fmt::Debug for ImmTy<'_, Prov> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ImmTy").field("imm", &self.imm).field("ty", &self.layout.ty).finish()
+    }
+}
+
 impl<'tcx, Prov: Provenance> std::ops::Deref for ImmTy<'tcx, Prov> {
     type Target = Immediate<Prov>;
     #[inline(always)]
@@ -142,51 +148,6 @@ fn deref(&self) -> &Immediate<Prov> {
     }
 }
 
-/// An `Operand` is the result of computing a `mir::Operand`. It can be immediate,
-/// or still in memory. The latter is an optimization, to delay reading that chunk of
-/// memory and to avoid having to store arbitrary-sized data here.
-#[derive(Copy, Clone, Debug)]
-pub enum Operand<Prov: Provenance = AllocId> {
-    Immediate(Immediate<Prov>),
-    Indirect(MemPlace<Prov>),
-}
-
-#[derive(Clone, Debug)]
-pub struct OpTy<'tcx, Prov: Provenance = AllocId> {
-    op: Operand<Prov>, // Keep this private; it helps enforce invariants.
-    pub layout: TyAndLayout<'tcx>,
-    /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct:
-    /// it needs to have a different alignment than the field type would usually have.
-    /// So we represent this here with a separate field that "overwrites" `layout.align`.
-    /// This means `layout.align` should never be used for an `OpTy`!
-    /// `None` means "alignment does not matter since this is a by-value operand"
-    /// (`Operand::Immediate`); this field is only relevant for `Operand::Indirect`.
-    /// Also CTFE ignores alignment anyway, so this is for Miri only.
-    pub align: Option<Align>,
-}
-
-impl<'tcx, Prov: Provenance> std::ops::Deref for OpTy<'tcx, Prov> {
-    type Target = Operand<Prov>;
-    #[inline(always)]
-    fn deref(&self) -> &Operand<Prov> {
-        &self.op
-    }
-}
-
-impl<'tcx, Prov: Provenance> From<MPlaceTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
-    #[inline(always)]
-    fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self {
-        OpTy { op: Operand::Indirect(*mplace), layout: mplace.layout, align: Some(mplace.align) }
-    }
-}
-
-impl<'tcx, Prov: Provenance> From<ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
-    #[inline(always)]
-    fn from(val: ImmTy<'tcx, Prov>) -> Self {
-        OpTy { op: Operand::Immediate(val.imm), layout: val.layout, align: None }
-    }
-}
-
 impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
     #[inline]
     pub fn from_scalar(val: Scalar<Prov>, layout: TyAndLayout<'tcx>) -> Self {
@@ -319,7 +280,61 @@ fn to_op<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
     }
 }
 
-impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> {
+/// An `Operand` is the result of computing a `mir::Operand`. It can be immediate,
+/// or still in memory. The latter is an optimization, to delay reading that chunk of
+/// memory and to avoid having to store arbitrary-sized data here.
+#[derive(Copy, Clone, Debug)]
+pub(super) enum Operand<Prov: Provenance = AllocId> {
+    Immediate(Immediate<Prov>),
+    Indirect(MemPlace<Prov>),
+}
+
+#[derive(Clone)]
+pub struct OpTy<'tcx, Prov: Provenance = AllocId> {
+    op: Operand<Prov>, // Keep this private; it helps enforce invariants.
+    pub layout: TyAndLayout<'tcx>,
+    /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct:
+    /// it needs to have a different alignment than the field type would usually have.
+    /// So we represent this here with a separate field that "overwrites" `layout.align`.
+    /// This means `layout.align` should never be used for an `OpTy`!
+    /// `None` means "alignment does not matter since this is a by-value operand"
+    /// (`Operand::Immediate`); this field is only relevant for `Operand::Indirect`.
+    /// Also CTFE ignores alignment anyway, so this is for Miri only.
+    pub align: Option<Align>,
+}
+
+impl<Prov: Provenance> std::fmt::Debug for OpTy<'_, Prov> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("OpTy").field("op", &self.op).field("ty", &self.layout.ty).finish()
+    }
+}
+
+impl<'tcx, Prov: Provenance> From<ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
+    #[inline(always)]
+    fn from(val: ImmTy<'tcx, Prov>) -> Self {
+        OpTy { op: Operand::Immediate(val.imm), layout: val.layout, align: None }
+    }
+}
+
+impl<'tcx, Prov: Provenance> From<MPlaceTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
+    #[inline(always)]
+    fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self {
+        OpTy {
+            op: Operand::Indirect(*mplace.mplace()),
+            layout: mplace.layout,
+            align: Some(mplace.align),
+        }
+    }
+}
+
+impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
+    #[inline(always)]
+    pub(super) fn op(&self) -> &Operand<Prov> {
+        &self.op
+    }
+}
+
+impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> {
     #[inline(always)]
     fn layout(&self) -> TyAndLayout<'tcx> {
         self.layout
@@ -328,7 +343,7 @@ fn layout(&self) -> TyAndLayout<'tcx> {
     #[inline]
     fn meta(&self) -> MemPlaceMeta<Prov> {
         match self.as_mplace_or_imm() {
-            Left(mplace) => mplace.meta,
+            Left(mplace) => mplace.meta(),
             Right(_) => {
                 debug_assert!(self.layout.is_sized(), "unsized immediates are not a thing");
                 MemPlaceMeta::None
@@ -362,18 +377,19 @@ fn to_op<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
     }
 }
 
+/// The `Readable` trait describes interpreter values that one can read from.
 pub trait Readable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> {
     fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>>;
 }
 
-impl<'tcx, Prov: Provenance + 'static> Readable<'tcx, Prov> for OpTy<'tcx, Prov> {
+impl<'tcx, Prov: Provenance> Readable<'tcx, Prov> for OpTy<'tcx, Prov> {
     #[inline(always)]
     fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
         self.as_mplace_or_imm()
     }
 }
 
-impl<'tcx, Prov: Provenance + 'static> Readable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
+impl<'tcx, Prov: Provenance> Readable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
     #[inline(always)]
     fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
         Left(self.clone())
@@ -535,7 +551,7 @@ pub fn read_target_isize(
     /// Turn the wide MPlace into a string (must already be dereferenced!)
     pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &str> {
         let len = mplace.len(self)?;
-        let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr, Size::from_bytes(len))?;
+        let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len))?;
         let str = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?;
         Ok(str)
     }
@@ -630,7 +646,7 @@ pub fn eval_place_to_op(
             op = self.project(&op, elem)?
         }
 
-        trace!("eval_place_to_op: got {:?}", *op);
+        trace!("eval_place_to_op: got {:?}", op);
         // Sanity-check the type we ended up with.
         debug_assert!(
             mir_assign_valid_types(
@@ -673,7 +689,7 @@ pub fn eval_operand(
                 self.eval_mir_constant(&c, Some(constant.span), layout)?
             }
         };
-        trace!("{:?}: {:?}", mir_op, *op);
+        trace!("{:?}: {:?}", mir_op, op);
         Ok(op)
     }
 

compiler/rustc_const_eval/src/interpret/place.rs

@@ -51,7 +51,7 @@ pub fn has_meta(self) -> bool {
 }
 
 #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
-pub struct MemPlace<Prov: Provenance = AllocId> {
+pub(super) struct MemPlace<Prov: Provenance = AllocId> {
     /// The pointer can be a pure integer, with the `None` provenance.
     pub ptr: Pointer<Option<Prov>>,
     /// Metadata for unsized places. Interpretation is up to the type.
@@ -60,68 +60,6 @@ pub struct MemPlace<Prov: Provenance = AllocId> {
     pub meta: MemPlaceMeta<Prov>,
 }
 
-/// A MemPlace with its layout. Constructing it is only possible in this module.
-#[derive(Clone, Hash, Eq, PartialEq, Debug)]
-pub struct MPlaceTy<'tcx, Prov: Provenance = AllocId> {
-    mplace: MemPlace<Prov>,
-    pub layout: TyAndLayout<'tcx>,
-    /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct:
-    /// it needs to have a different alignment than the field type would usually have.
-    /// So we represent this here with a separate field that "overwrites" `layout.align`.
-    /// This means `layout.align` should never be used for a `MPlaceTy`!
-    pub align: Align,
-}
-
-impl<'tcx, Prov: Provenance> std::ops::Deref for MPlaceTy<'tcx, Prov> {
-    type Target = MemPlace<Prov>;
-    #[inline(always)]
-    fn deref(&self) -> &MemPlace<Prov> {
-        &self.mplace
-    }
-}
-
-#[derive(Copy, Clone, Debug)]
-pub enum Place<Prov: Provenance = AllocId> {
-    /// A place referring to a value allocated in the `Memory` system.
-    Ptr(MemPlace<Prov>),
-
-    /// To support alloc-free locals, we are able to write directly to a local. The offset indicates
-    /// where in the local this place is located; if it is `None`, no projection has been applied.
-    /// Such projections are meaningful even if the offset is 0, since they can change layouts.
-    /// (Without that optimization, we'd just always be a `MemPlace`.)
-    /// Note that this only stores the frame index, not the thread this frame belongs to -- that is
-    /// implicit. This means a `Place` must never be moved across interpreter thread boundaries!
-    ///
-    /// This variant shall not be used for unsized types -- those must always live in memory.
-    Local { frame: usize, local: mir::Local, offset: Option<Size> },
-}
-
-#[derive(Clone, Debug)]
-pub struct PlaceTy<'tcx, Prov: Provenance = AllocId> {
-    place: Place<Prov>, // Keep this private; it helps enforce invariants.
-    pub layout: TyAndLayout<'tcx>,
-    /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct:
-    /// it needs to have a different alignment than the field type would usually have.
-    /// So we represent this here with a separate field that "overwrites" `layout.align`.
-    /// This means `layout.align` should never be used for a `PlaceTy`!
-    pub align: Align,
-}
-
-impl<'tcx, Prov: Provenance> std::ops::Deref for PlaceTy<'tcx, Prov> {
-    type Target = Place<Prov>;
-    #[inline(always)]
-    fn deref(&self) -> &Place<Prov> {
-        &self.place
-    }
-}
-
-impl<'tcx, Prov: Provenance> From<MPlaceTy<'tcx, Prov>> for PlaceTy<'tcx, Prov> {
-    #[inline(always)]
-    fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self {
-        PlaceTy { place: Place::Ptr(*mplace), layout: mplace.layout, align: mplace.align }
-    }
-}
-
 impl<Prov: Provenance> MemPlace<Prov> {
     #[inline(always)]
     pub fn from_ptr(ptr: Pointer<Option<Prov>>) -> Self {
@@ -165,6 +103,27 @@ fn offset_with_meta_<'tcx>(
     }
 }
 
+/// A MemPlace with its layout. Constructing it is only possible in this module.
+#[derive(Clone, Hash, Eq, PartialEq)]
+pub struct MPlaceTy<'tcx, Prov: Provenance = AllocId> {
+    mplace: MemPlace<Prov>,
+    pub layout: TyAndLayout<'tcx>,
+    /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct:
+    /// it needs to have a different alignment than the field type would usually have.
+    /// So we represent this here with a separate field that "overwrites" `layout.align`.
+    /// This means `layout.align` should never be used for a `MPlaceTy`!
+    pub align: Align,
+}
+
+impl<Prov: Provenance> std::fmt::Debug for MPlaceTy<'_, Prov> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("MPlaceTy")
+            .field("mplace", &self.mplace)
+            .field("ty", &self.layout.ty)
+            .finish()
+    }
+}
+
 impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
     /// Produces a MemPlace that works for ZST but nothing else.
     /// Conceptually this is a new allocation, but it doesn't actually create an allocation so you
@@ -194,9 +153,29 @@ pub fn from_aligned_ptr_with_meta(
             align: layout.align.abi,
         }
     }
+
+    /// Adjust the provenance of the main pointer (metadata is unaffected).
+    pub fn map_provenance(self, f: impl FnOnce(Option<Prov>) -> Option<Prov>) -> Self {
+        MPlaceTy { mplace: self.mplace.map_provenance(f), ..self }
+    }
+
+    #[inline(always)]
+    pub(super) fn mplace(&self) -> &MemPlace<Prov> {
+        &self.mplace
+    }
+
+    #[inline(always)]
+    pub fn ptr(&self) -> Pointer<Option<Prov>> {
+        self.mplace.ptr
+    }
+
+    #[inline(always)]
+    pub fn to_ref(&self, cx: &impl HasDataLayout) -> Immediate<Prov> {
+        self.mplace.to_ref(cx)
+    }
 }
 
-impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
+impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
     #[inline(always)]
     fn layout(&self) -> TyAndLayout<'tcx> {
         self.layout
@@ -204,7 +183,7 @@ fn layout(&self) -> TyAndLayout<'tcx> {
 
     #[inline(always)]
     fn meta(&self) -> MemPlaceMeta<Prov> {
-        self.meta
+        self.mplace.meta
     }
 
     fn offset_with_meta<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
@@ -229,7 +208,76 @@ fn to_op<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
     }
 }
 
-impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
+#[derive(Copy, Clone, Debug)]
+pub(super) enum Place<Prov: Provenance = AllocId> {
+    /// A place referring to a value allocated in the `Memory` system.
+    Ptr(MemPlace<Prov>),
+
+    /// To support alloc-free locals, we are able to write directly to a local. The offset indicates
+    /// where in the local this place is located; if it is `None`, no projection has been applied.
+    /// Such projections are meaningful even if the offset is 0, since they can change layouts.
+    /// (Without that optimization, we'd just always be a `MemPlace`.)
+    /// Note that this only stores the frame index, not the thread this frame belongs to -- that is
+    /// implicit. This means a `Place` must never be moved across interpreter thread boundaries!
+    ///
+    /// This variant shall not be used for unsized types -- those must always live in memory.
+    Local { frame: usize, local: mir::Local, offset: Option<Size> },
+}
+
+#[derive(Clone)]
+pub struct PlaceTy<'tcx, Prov: Provenance = AllocId> {
+    place: Place<Prov>, // Keep this private; it helps enforce invariants.
+    pub layout: TyAndLayout<'tcx>,
+    /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct:
+    /// it needs to have a different alignment than the field type would usually have.
+    /// So we represent this here with a separate field that "overwrites" `layout.align`.
+    /// This means `layout.align` should never be used for a `PlaceTy`!
+    pub align: Align,
+}
+
+impl<Prov: Provenance> std::fmt::Debug for PlaceTy<'_, Prov> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PlaceTy").field("place", &self.place).field("ty", &self.layout.ty).finish()
+    }
+}
+
+impl<'tcx, Prov: Provenance> From<MPlaceTy<'tcx, Prov>> for PlaceTy<'tcx, Prov> {
+    #[inline(always)]
+    fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self {
+        PlaceTy { place: Place::Ptr(mplace.mplace), layout: mplace.layout, align: mplace.align }
+    }
+}
+
+impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
+    #[inline(always)]
+    pub(super) fn place(&self) -> &Place<Prov> {
+        &self.place
+    }
+
+    /// A place is either an mplace or some local.
+    #[inline(always)]
+    pub fn as_mplace_or_local(
+        &self,
+    ) -> Either<MPlaceTy<'tcx, Prov>, (usize, mir::Local, Option<Size>)> {
+        match self.place {
+            Place::Ptr(mplace) => Left(MPlaceTy { mplace, layout: self.layout, align: self.align }),
+            Place::Local { frame, local, offset } => Right((frame, local, offset)),
+        }
+    }
+
+    #[inline(always)]
+    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
+    pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
+        self.as_mplace_or_local().left().unwrap_or_else(|| {
+            bug!(
+                "PlaceTy of type {} was a local when it was expected to be an MPlace",
+                self.layout.ty
+            )
+        })
+    }
+}
+
+impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
     #[inline(always)]
     fn layout(&self) -> TyAndLayout<'tcx> {
         self.layout
@@ -238,7 +286,7 @@ fn layout(&self) -> TyAndLayout<'tcx> {
     #[inline]
     fn meta(&self) -> MemPlaceMeta<Prov> {
         match self.as_mplace_or_local() {
-            Left(mplace) => mplace.meta,
+            Left(mplace) => mplace.meta(),
             Right(_) => {
                 debug_assert!(self.layout.is_sized(), "unsized locals should live in memory");
                 MemPlaceMeta::None
@@ -286,11 +334,11 @@ fn to_op<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
 impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
     #[inline(always)]
     pub fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
-        match **self {
+        match self.op() {
             Operand::Indirect(mplace) => {
-                Left(MPlaceTy { mplace, layout: self.layout, align: self.align.unwrap() })
+                Left(MPlaceTy { mplace: *mplace, layout: self.layout, align: self.align.unwrap() })
             }
-            Operand::Immediate(imm) => Right(ImmTy::from_immediate(imm, self.layout)),
+            Operand::Immediate(imm) => Right(ImmTy::from_immediate(*imm, self.layout)),
         }
     }
 
@@ -306,30 +354,7 @@ pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
     }
 }
 
-impl<'tcx, Prov: Provenance + 'static> PlaceTy<'tcx, Prov> {
+/// The `Weiteable` trait describes interpreter values that can be written to.
-    /// A place is either an mplace or some local.
-    #[inline(always)]
-    pub fn as_mplace_or_local(
-        &self,
-    ) -> Either<MPlaceTy<'tcx, Prov>, (usize, mir::Local, Option<Size>)> {
-        match **self {
-            Place::Ptr(mplace) => Left(MPlaceTy { mplace, layout: self.layout, align: self.align }),
-            Place::Local { frame, local, offset } => Right((frame, local, offset)),
-        }
-    }
-
-    #[inline(always)]
-    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
-    pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
-        self.as_mplace_or_local().left().unwrap_or_else(|| {
-            bug!(
-                "PlaceTy of type {} was a local when it was expected to be an MPlace",
-                self.layout.ty
-            )
-        })
-    }
-}
-
 pub trait Writeable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> {
     fn as_mplace_or_local(
         &self,
@@ -341,7 +366,7 @@ fn force_mplace<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
     ) -> InterpResult<'tcx, MPlaceTy<'tcx, Prov>>;
 }
 
-impl<'tcx, Prov: Provenance + 'static> Writeable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
+impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
     #[inline(always)]
     fn as_mplace_or_local(
         &self,
@@ -360,7 +385,7 @@ fn force_mplace<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
     }
 }
 
-impl<'tcx, Prov: Provenance + 'static> Writeable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
+impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
     #[inline(always)]
     fn as_mplace_or_local(
         &self,
@@ -381,7 +406,7 @@ fn force_mplace<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
 // FIXME: Working around https://github.com/rust-lang/rust/issues/54385
 impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M>
 where
-    Prov: Provenance + 'static,
+    Prov: Provenance,
     M: Machine<'mir, 'tcx, Provenance = Prov>,
 {
     /// Take a value, which represents a (thin or wide) reference, and make it a place.
@@ -415,7 +440,7 @@ pub fn mplace_to_ref(
         &self,
         mplace: &MPlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
-        let imm = mplace.to_ref(self);
+        let imm = mplace.mplace.to_ref(self);
         let layout = self.layout_of(Ty::new_mut_ptr(self.tcx.tcx, mplace.layout.ty))?;
         Ok(ImmTy::from_immediate(imm, layout))
     }
@@ -449,7 +474,7 @@ pub(super) fn get_place_alloc(
             .size_and_align_of_mplace(&mplace)?
             .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
         // Due to packed places, only `mplace.align` matters.
-        self.get_ptr_alloc(mplace.ptr, size, mplace.align)
+        self.get_ptr_alloc(mplace.ptr(), size, mplace.align)
     }
 
     #[inline]
@@ -462,7 +487,7 @@ pub(super) fn get_place_alloc_mut(
             .size_and_align_of_mplace(&mplace)?
             .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
         // Due to packed places, only `mplace.align` matters.
-        self.get_ptr_alloc_mut(mplace.ptr, size, mplace.align)
+        self.get_ptr_alloc_mut(mplace.ptr(), size, mplace.align)
     }
 
     /// Check if this mplace is dereferenceable and sufficiently aligned.
@@ -473,7 +498,7 @@ pub fn check_mplace(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResu
         // Due to packed places, only `mplace.align` matters.
         let align =
             if M::enforce_alignment(self).should_check() { mplace.align } else { Align::ONE };
-        self.check_ptr_access_align(mplace.ptr, size, align, CheckInAllocMsg::DerefTest)?;
+        self.check_ptr_access_align(mplace.ptr(), size, align, CheckInAllocMsg::DerefTest)?;
         Ok(())
     }
 
@@ -542,7 +567,7 @@ pub fn eval_place(
             place = self.project(&place, elem)?
         }
 
-        trace!("{:?}", self.dump_place(place.place));
+        trace!("{:?}", self.dump_place(&place));
         // Sanity-check the type we ended up with.
         debug_assert!(
             mir_assign_valid_types(
@@ -618,7 +643,8 @@ fn write_immediate_no_validate(
                     // just fall back to the indirect path.
                     dest.force_mplace(self)?
                 } else {
-                    match M::access_local_mut(self, frame, local)? {
+                    M::before_access_local_mut(self, frame, local)?;
+                    match self.stack_mut()[frame].locals[local].access_mut()? {
                         Operand::Immediate(local_val) => {
                             // Local can be updated in-place.
                             *local_val = src;
@@ -738,7 +764,8 @@ pub fn write_uninit(
                     // FIXME: share the logic with `write_immediate_no_validate`.
                     dest.force_mplace(self)?
                 } else {
-                    match M::access_local_mut(self, frame, local)? {
+                    M::before_access_local_mut(self, frame, local)?;
+                    match self.stack_mut()[frame].locals[local].access_mut()? {
                         Operand::Immediate(local) => {
                             *local = Immediate::Uninit;
                             return Ok(());
@@ -832,7 +859,7 @@ fn copy_op_no_validate(
                         *src_val,
                         src.layout(),
                         dest_mem.align,
-                        *dest_mem,
+                        dest_mem.mplace,
                     )
                 };
             }
@@ -859,7 +886,12 @@ fn copy_op_no_validate(
         // (Or as the `Assign` docs put it, assignments "not producing primitives" must be
         // non-overlapping.)
         self.mem_copy(
-            src.ptr, src.align, dest.ptr, dest.align, dest_size, /*nonoverlapping*/ true,
+            src.ptr(),
+            src.align,
+            dest.ptr(),
+            dest.align,
+            dest_size,
+            /*nonoverlapping*/ true,
         )
     }
 
@@ -874,7 +906,8 @@ pub fn force_allocation(
     ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
         let mplace = match place.place {
             Place::Local { frame, local, offset } => {
-                let whole_local = match M::access_local_mut(self, frame, local)? {
+                M::before_access_local_mut(self, frame, local)?;
+                let whole_local = match self.stack_mut()[frame].locals[local].access_mut()? {
                     &mut Operand::Immediate(local_val) => {
                         // We need to make an allocation.
 
@@ -894,16 +927,16 @@ pub fn force_allocation(
                                 local_val,
                                 local_layout,
                                 local_layout.align.abi,
-                                *mplace,
+                                mplace.mplace,
                             )?;
                         }
                         M::after_local_allocated(self, frame, local, &mplace)?;
                         // Now we can call `access_mut` again, asserting it goes well, and actually
                         // overwrite things. This points to the entire allocation, not just the part
                         // the place refers to, i.e. we do this before we apply `offset`.
-                        *M::access_local_mut(self, frame, local).unwrap() =
+                        *self.stack_mut()[frame].locals[local].access_mut().unwrap() =
-                            Operand::Indirect(*mplace);
+                            Operand::Indirect(mplace.mplace);
-                        *mplace
+                        mplace.mplace
                     }
                     &mut Operand::Indirect(mplace) => mplace, // this already was an indirect local
                 };
@@ -1011,12 +1044,12 @@ pub(super) fn unpack_dyn_trait(
             matches!(mplace.layout.ty.kind(), ty::Dynamic(_, _, ty::Dyn)),
             "`unpack_dyn_trait` only makes sense on `dyn*` types"
         );
-        let vtable = mplace.meta.unwrap_meta().to_pointer(self)?;
+        let vtable = mplace.meta().unwrap_meta().to_pointer(self)?;
         let (ty, _) = self.get_ptr_vtable(vtable)?;
         let layout = self.layout_of(ty)?;
 
         let mplace = MPlaceTy {
-            mplace: MemPlace { meta: MemPlaceMeta::None, ..**mplace },
+            mplace: MemPlace { meta: MemPlaceMeta::None, ..mplace.mplace },
             layout,
             align: layout.align.abi,
         };

compiler/rustc_const_eval/src/interpret/projection.rs

@@ -89,7 +89,7 @@ fn to_op<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
 }
 
 /// A type representing iteration over the elements of an array.
-pub struct ArrayIterator<'tcx, 'a, Prov: Provenance + 'static, P: Projectable<'tcx, Prov>> {
+pub struct ArrayIterator<'tcx, 'a, Prov: Provenance, P: Projectable<'tcx, Prov>> {
     base: &'a P,
     range: Range<u64>,
     stride: Size,
@@ -97,9 +97,7 @@ pub struct ArrayIterator<'tcx, 'a, Prov: Provenance + 'static, P: Projectable<'t
     _phantom: PhantomData<Prov>, // otherwise it says `Prov` is never used...
 }
 
-impl<'tcx, 'a, Prov: Provenance + 'static, P: Projectable<'tcx, Prov>>
+impl<'tcx, 'a, Prov: Provenance, P: Projectable<'tcx, Prov>> ArrayIterator<'tcx, 'a, Prov, P> {
-    ArrayIterator<'tcx, 'a, Prov, P>
-{
     /// Should be the same `ecx` on each call, and match the one used to create the iterator.
     pub fn next<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
         &mut self,
@@ -113,7 +111,7 @@ pub fn next<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
 // FIXME: Working around https://github.com/rust-lang/rust/issues/54385
 impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M>
 where
-    Prov: Provenance + 'static,
+    Prov: Provenance,
     M: Machine<'mir, 'tcx, Provenance = Prov>,
 {
     /// Offset a pointer to project to a field of a struct/union. Unlike `place_field`, this is

compiler/rustc_const_eval/src/interpret/step.rs

@@ -204,7 +204,7 @@ pub fn eval_rvalue_into_place(
                     // avoid writing each operand individually and instead just make many copies
                     // of the first element.
                     let elem_size = first.layout.size;
-                    let first_ptr = first.ptr;
+                    let first_ptr = first.ptr();
                     let rest_ptr = first_ptr.offset(elem_size, self)?;
                     // For the alignment of `rest_ptr`, we crucially do *not* use `first.align` as
                     // that place might be more aligned than its type mandates (a `u8` array could
@@ -305,7 +305,7 @@ pub fn eval_rvalue_into_place(
             }
         }
 
-        trace!("{:?}", self.dump_place(*dest));
+        trace!("{:?}", self.dump_place(&dest));
 
         Ok(())
     }

compiler/rustc_const_eval/src/interpret/terminator.rs

@@ -793,7 +793,7 @@ pub(crate) fn eval_fn_call(
                         throw_ub_custom!(fluent::const_eval_dyn_star_call_vtable_mismatch);
                     }
 
-                    (vptr, dyn_ty, recv.ptr)
+                    (vptr, dyn_ty, recv.ptr())
                 } else {
                     // Doesn't have to be a `dyn Trait`, but the unsized tail must be `dyn Trait`.
                     // (For that reason we also cannot use `unpack_dyn_trait`.)
@@ -810,7 +810,7 @@ pub(crate) fn eval_fn_call(
                     assert!(receiver_place.layout.is_unsized());
 
                     // Get the required information from the vtable.
-                    let vptr = receiver_place.meta.unwrap_meta().to_pointer(self)?;
+                    let vptr = receiver_place.meta().unwrap_meta().to_pointer(self)?;
                     let (dyn_ty, dyn_trait) = self.get_ptr_vtable(vptr)?;
                     if dyn_trait != data.principal() {
                         throw_ub_custom!(fluent::const_eval_dyn_call_vtable_mismatch);
@@ -819,7 +819,7 @@ pub(crate) fn eval_fn_call(
                     // It might be surprising that we use a pointer as the receiver even if this
                     // is a by-val case; this works because by-val passing of an unsized `dyn
                     // Trait` to a function is actually desugared to a pointer.
-                    (vptr, dyn_ty, receiver_place.ptr)
+                    (vptr, dyn_ty, receiver_place.ptr())
                 };
 
                 // Now determine the actual method to call. We can do that in two different ways and

compiler/rustc_const_eval/src/interpret/validity.rs

@@ -360,7 +360,7 @@ fn check_safe_pointer(
         // Handle wide pointers.
         // Check metadata early, for better diagnostics
         if place.layout.is_unsized() {
-            self.check_wide_ptr_meta(place.meta, place.layout)?;
+            self.check_wide_ptr_meta(place.meta(), place.layout)?;
         }
         // Make sure this is dereferenceable and all.
         let size_and_align = try_validation!(
@@ -379,7 +379,7 @@ fn check_safe_pointer(
         // Direct call to `check_ptr_access_align` checks alignment even on CTFE machines.
         try_validation!(
             self.ecx.check_ptr_access_align(
-                place.ptr,
+                place.ptr(),
                 size,
                 align,
                 CheckInAllocMsg::InboundsTest, // will anyway be replaced by validity message
@@ -414,7 +414,7 @@ fn check_safe_pointer(
         if let Some(ref_tracking) = self.ref_tracking.as_deref_mut() {
             // Proceed recursively even for ZST, no reason to skip them!
             // `!` is a ZST and we want to validate it.
-            if let Ok((alloc_id, _offset, _prov)) = self.ecx.ptr_try_get_alloc_id(place.ptr) {
+            if let Ok((alloc_id, _offset, _prov)) = self.ecx.ptr_try_get_alloc_id(place.ptr()) {
                 // Let's see what kind of memory this points to.
                 let alloc_kind = self.ecx.tcx.try_get_global_alloc(alloc_id);
                 match alloc_kind {
@@ -521,7 +521,7 @@ fn try_visit_primitive(
                 let place =
                     self.ecx.ref_to_mplace(&self.read_immediate(value, ExpectedKind::RawPtr)?)?;
                 if place.layout.is_unsized() {
-                    self.check_wide_ptr_meta(place.meta, place.layout)?;
+                    self.check_wide_ptr_meta(place.meta(), place.layout)?;
                 }
                 Ok(true)
             }
@@ -739,7 +739,7 @@ fn visit_value(&mut self, op: &OpTy<'tcx, M::Provenance>) -> InterpResult<'tcx>
                 let mplace = op.assert_mem_place(); // strings are unsized and hence never immediate
                 let len = mplace.len(self.ecx)?;
                 try_validation!(
-                    self.ecx.read_bytes_ptr_strip_provenance(mplace.ptr, Size::from_bytes(len)),
+                    self.ecx.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len)),
                     self.path,
                     Ub(InvalidUninitBytes(..)) => Uninit { expected: ExpectedKind::Str },
                     Unsup(ReadPointerAsInt(_)) => PointerAsInt { expected: ExpectedKind::Str }
@@ -789,7 +789,7 @@ fn visit_value(&mut self, op: &OpTy<'tcx, M::Provenance>) -> InterpResult<'tcx>
                 // to reject those pointers, we just do not have the machinery to
                 // talk about parts of a pointer.
                 // We also accept uninit, for consistency with the slow path.
-                let alloc = self.ecx.get_ptr_alloc(mplace.ptr, size, mplace.align)?.expect("we already excluded size 0");
+                let alloc = self.ecx.get_ptr_alloc(mplace.ptr(), size, mplace.align)?.expect("we already excluded size 0");
 
                 match alloc.get_bytes_strip_provenance() {
                     // In the happy case, we needn't check anything else.

compiler/rustc_const_eval/src/util/check_validity_requirement.rs

@@ -54,7 +54,7 @@ fn might_permit_raw_init_strict<'tcx>(
 
     if kind == ValidityRequirement::Zero {
         cx.write_bytes_ptr(
-            allocated.ptr,
+            allocated.ptr(),
             std::iter::repeat(0_u8).take(ty.layout.size().bytes_usize()),
         )
         .expect("failed to write bytes for zero valid check");

compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp

@@ -1202,7 +1202,11 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
 
     Ret->ModuleMap[module->identifier] = mem_buffer;
 
+#if LLVM_VERSION_GE(18, 0)
+    if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index)) {
+#else
     if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
+#endif
       LLVMRustSetLastError(toString(std::move(Err)).c_str());
       return nullptr;
     }

compiler/rustc_middle/src/mir/interpret/pointer.rs

@@ -103,7 +103,7 @@ impl<T: HasDataLayout> PointerArithmetic for T {}
 /// mostly opaque; the `Machine` trait extends it with some more operations that also have access to
 /// some global state.
 /// The `Debug` rendering is used to display bare provenance, and for the default impl of `fmt`.
-pub trait Provenance: Copy + fmt::Debug {
+pub trait Provenance: Copy + fmt::Debug + 'static {
     /// Says whether the `offset` field of `Pointer`s with this provenance is the actual physical address.
     /// - If `false`, the offset *must* be relative. This means the bytes representing a pointer are
     ///   different from what the Abstract Machine prescribes, so the interpreter must prevent any

compiler/rustc_mir_build/src/build/custom/parse.rs

@@ -1,5 +1,6 @@
 use rustc_index::IndexSlice;
-use rustc_middle::{mir::*, thir::*, ty::Ty};
+use rustc_middle::ty::{self, Ty};
+use rustc_middle::{mir::*, thir::*};
 use rustc_span::Span;
 
 use super::{PResult, ParseCtxt, ParseError};
@@ -159,6 +160,14 @@ pub fn parse_body(&mut self, expr_id: ExprId) -> PResult<()> {
         );
         self.parse_local_decls(local_decls.iter().copied())?;
 
+        let (debuginfo, rest) = parse_by_kind!(self, rest, _, "body with debuginfo",
+            ExprKind::Block { block } => {
+                let block = &self.thir[*block];
+                (&block.stmts, block.expr.unwrap())
+            },
+        );
+        self.parse_debuginfo(debuginfo.iter().copied())?;
+
         let block_defs = parse_by_kind!(self, rest, _, "body with block defs",
             ExprKind::Block { block } => &self.thir[*block].stmts,
         );
@@ -195,6 +204,52 @@ fn parse_local_decls(&mut self, mut stmts: impl Iterator<Item = StmtId>) -> PRes
         Ok(())
     }
 
+    fn parse_debuginfo(&mut self, stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
+        for stmt in stmts {
+            let stmt = &self.thir[stmt];
+            let expr = match stmt.kind {
+                StmtKind::Let { span, .. } => {
+                    return Err(ParseError {
+                        span,
+                        item_description: format!("{:?}", stmt),
+                        expected: "debuginfo".to_string(),
+                    });
+                }
+                StmtKind::Expr { expr, .. } => expr,
+            };
+            let span = self.thir[expr].span;
+            let (name, operand) = parse_by_kind!(self, expr, _, "debuginfo",
+                @call("mir_debuginfo", args) => {
+                    (args[0], args[1])
+                },
+            );
+            let name = parse_by_kind!(self, name, _, "debuginfo",
+                ExprKind::Literal { lit, neg: false } => lit,
+            );
+            let Some(name) = name.node.str() else {
+                return Err(ParseError {
+                    span,
+                    item_description: format!("{:?}", name),
+                    expected: "string".to_string(),
+                });
+            };
+            let operand = self.parse_operand(operand)?;
+            let value = match operand {
+                Operand::Constant(c) => VarDebugInfoContents::Const(*c),
+                Operand::Copy(p) | Operand::Move(p) => VarDebugInfoContents::Place(p),
+            };
+            let dbginfo = VarDebugInfo {
+                name,
+                source_info: SourceInfo { span, scope: self.source_scope },
+                argument_index: None,
+                value,
+            };
+            self.body.var_debug_info.push(dbginfo);
+        }
+
+        Ok(())
+    }
+
     fn parse_let_statement(&mut self, stmt_id: StmtId) -> PResult<(LocalVarId, Ty<'tcx>, Span)> {
         let pattern = match &self.thir[stmt_id].kind {
             StmtKind::Let { pattern, .. } => pattern,

compiler/rustc_mir_build/src/build/custom/parse/instruction.rs

@@ -204,7 +204,7 @@ fn parse_rvalue(&self, expr_id: ExprId) -> PResult<Rvalue<'tcx>> {
         )
     }
 
-    fn parse_operand(&self, expr_id: ExprId) -> PResult<Operand<'tcx>> {
+    pub fn parse_operand(&self, expr_id: ExprId) -> PResult<Operand<'tcx>> {
         parse_by_kind!(self, expr_id, expr, "operand",
             @call("mir_move", args) => self.parse_place(args[0]).map(Operand::Move),
             @call("mir_static", args) => self.parse_static(args[0]),

compiler/rustc_mir_transform/src/const_prop.rs

@@ -22,8 +22,7 @@
 use crate::MirPass;
 use rustc_const_eval::interpret::{
     self, compile_time_machine, AllocId, ConstAllocation, ConstValue, FnArg, Frame, ImmTy,
-    Immediate, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, PlaceTy, Pointer, Scalar,
+    Immediate, InterpCx, InterpResult, MemoryKind, OpTy, PlaceTy, Pointer, Scalar, StackPopCleanup,
-    StackPopCleanup,
 };
 
 /// The maximum number of bytes that we'll allocate space for a local or the return value.
@@ -225,11 +224,11 @@ fn binary_ptr_op(
         throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp")
     }
 
-    fn access_local_mut<'a>(
+    fn before_access_local_mut<'a>(
         ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
         frame: usize,
         local: Local,
-    ) -> InterpResult<'tcx, &'a mut interpret::Operand<Self::Provenance>> {
+    ) -> InterpResult<'tcx> {
         assert_eq!(frame, 0);
         match ecx.machine.can_const_prop[local] {
             ConstPropMode::NoPropagation => {
@@ -242,7 +241,7 @@ fn access_local_mut<'a>(
             }
             ConstPropMode::FullConstProp => {}
         }
-        ecx.machine.stack[frame].locals[local].access_mut()
+        Ok(())
     }
 
     fn before_access_global(
@@ -382,8 +381,7 @@ fn new(
             // mark those as live... We rely on `local_to_place`/`local_to_op` in the interpreter
             // stopping us before those unsized immediates can cause issues deeper in the
             // interpreter.
-            ecx.frame_mut().locals[local].value =
+            ecx.frame_mut().locals[local].make_live_uninit();
-                LocalValue::Live(interpret::Operand::Immediate(Immediate::Uninit));
         }
 
         ConstPropagator { ecx, tcx, param_env, local_decls: &dummy_body.local_decls }
@@ -392,7 +390,11 @@ fn new(
     fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
         let op = match self.ecx.eval_place_to_op(place, None) {
             Ok(op) => {
-                if matches!(*op, interpret::Operand::Immediate(Immediate::Uninit)) {
+                if op
+                    .as_mplace_or_imm()
+                    .right()
+                    .is_some_and(|imm| matches!(*imm, Immediate::Uninit))
+                {
                     // Make sure nobody accidentally uses this value.
                     return None;
                 }
@@ -415,8 +417,7 @@ fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
     /// Remove `local` from the pool of `Locals`. Allows writing to them,
     /// but not reading from them anymore.
     fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) {
-        ecx.frame_mut().locals[local].value =
+        ecx.frame_mut().locals[local].make_live_uninit();
-            LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit));
         ecx.machine.written_only_inside_own_block_locals.remove(&local);
     }
 
@@ -743,7 +744,8 @@ fn process_projection_elem(
     ) -> Option<PlaceElem<'tcx>> {
         if let PlaceElem::Index(local) = elem
             && let Some(value) = self.get_const(local.into())
-            && let interpret::Operand::Immediate(Immediate::Scalar(scalar)) = *value
+            && let Some(imm) = value.as_mplace_or_imm().right()
+            && let Immediate::Scalar(scalar) = *imm
             && let Ok(offset) = scalar.to_target_usize(&self.tcx)
             && let Some(min_length) = offset.checked_add(1)
         {

compiler/rustc_mir_transform/src/const_prop_lint.rs

@@ -7,7 +7,7 @@
 
 use rustc_const_eval::interpret::Immediate;
 use rustc_const_eval::interpret::{
-    self, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, Scalar, StackPopCleanup,
+    InterpCx, InterpResult, MemoryKind, OpTy, Scalar, StackPopCleanup,
 };
 use rustc_const_eval::ReportErrorExt;
 use rustc_hir::def::DefKind;
@@ -212,8 +212,7 @@ fn new(
             // mark those as live... We rely on `local_to_place`/`local_to_op` in the interpreter
             // stopping us before those unsized immediates can cause issues deeper in the
             // interpreter.
-            ecx.frame_mut().locals[local].value =
+            ecx.frame_mut().locals[local].make_live_uninit();
-                LocalValue::Live(interpret::Operand::Immediate(Immediate::Uninit));
         }
 
         ConstPropagator {
@@ -236,7 +235,11 @@ fn local_decls(&self) -> &'mir LocalDecls<'tcx> {
     fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
         let op = match self.ecx.eval_place_to_op(place, None) {
             Ok(op) => {
-                if matches!(*op, interpret::Operand::Immediate(Immediate::Uninit)) {
+                if op
+                    .as_mplace_or_imm()
+                    .right()
+                    .is_some_and(|imm| matches!(*imm, Immediate::Uninit))
+                {
                     // Make sure nobody accidentally uses this value.
                     return None;
                 }
@@ -259,8 +262,7 @@ fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
     /// Remove `local` from the pool of `Locals`. Allows writing to them,
     /// but not reading from them anymore.
     fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) {
-        ecx.frame_mut().locals[local].value =
+        ecx.frame_mut().locals[local].make_live_uninit();
-            LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit));
         ecx.machine.written_only_inside_own_block_locals.remove(&local);
     }
 
@@ -656,12 +658,12 @@ fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
             }
             StatementKind::StorageLive(local) => {
                 let frame = self.ecx.frame_mut();
-                frame.locals[local].value =
+                frame.locals[local].make_live_uninit();
-                    LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit));
             }
             StatementKind::StorageDead(local) => {
                 let frame = self.ecx.frame_mut();
-                frame.locals[local].value = LocalValue::Dead;
+                // We don't actually track liveness, so the local remains live. But forget its value.
+                frame.locals[local].make_live_uninit();
             }
             _ => {}
         }

library/core/src/intrinsics.rs

@@ -1130,7 +1130,10 @@ pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
     /// may lead to unexpected and unstable compilation results. This makes `transmute` **incredibly
     /// unsafe**. `transmute` should be the absolute last resort.
     ///
-    /// Transmuting pointers to integers in a `const` context is [undefined behavior][ub].
+    /// Transmuting pointers *to* integers in a `const` context is [undefined behavior][ub],
+    /// unless the pointer was originally created *from* an integer.
+    /// (That includes this function specifically, integer-to-pointer casts, and helpers like [`invalid`][crate::ptr::invalid],
+    /// but also semantically-equivalent conversions such as punning through `repr(C)` union fields.)
     /// Any attempt to use the resulting value for integer operations will abort const-evaluation.
     /// (And even outside `const`, such transmutation is touching on many unspecified aspects of the
     /// Rust memory model and should be avoided. See below for alternatives.)

library/core/src/intrinsics/mir.rs

@@ -12,7 +12,8 @@
 //!
 //! Typical usage will look like this:
 //!
-//! ```rust
+#![cfg_attr(bootstrap, doc = "```rust,ignore")]
+#![cfg_attr(not(bootstrap), doc = "```rust")]
 //! #![feature(core_intrinsics, custom_mir)]
 //! #![allow(internal_features)]
 //!
@@ -62,7 +63,8 @@
 //!
 //! # Examples
 //!
-//! ```rust
+#![cfg_attr(bootstrap, doc = "```rust,ignore")]
+#![cfg_attr(not(bootstrap), doc = "```rust")]
 //! #![feature(core_intrinsics, custom_mir)]
 //! #![allow(internal_features)]
 //!
@@ -317,7 +319,8 @@ fn Discriminant<T>(place: T) -> <T as ::core::marker::DiscriminantKind>::Discrim
     ///
     /// # Examples
     ///
-    /// ```rust
+    #[cfg_attr(bootstrap, doc = "```rust,ignore")]
+    #[cfg_attr(not(bootstrap), doc = "```rust")]
     /// #![allow(internal_features)]
     /// #![feature(custom_mir, core_intrinsics)]
     ///
@@ -361,6 +364,11 @@ fn CastTransmute<T, U>(operand: T) -> U
     #[doc(hidden)]
     fn __internal_make_place<T>(place: T) -> *mut T
 );
+define!(
+    "mir_debuginfo",
+    #[doc(hidden)]
+    fn __debuginfo<T>(name: &'static str, s: T)
+);
 
 /// Macro for generating custom MIR.
 ///
@@ -371,6 +379,7 @@ fn __internal_make_place<T>(place: T) -> *mut T
     (
         $(type RET = $ret_ty:ty ;)?
         $(let $local_decl:ident $(: $local_decl_ty:ty)? ;)*
+        $(debug $dbg_name:ident => $dbg_data:expr ;)*
 
         {
             $($entry:tt)*
@@ -394,26 +403,32 @@ fn __internal_make_place<T>(place: T) -> *mut T
             $(
                 let $local_decl $(: $local_decl_ty)? ;
             )*
-
             ::core::intrinsics::mir::__internal_extract_let!($($entry)*);
             $(
                 ::core::intrinsics::mir::__internal_extract_let!($($block)*);
             )*
 
             {
-                // Finally, the contents of the basic blocks
+                // Now debuginfo
-                ::core::intrinsics::mir::__internal_remove_let!({
-                    {}
-                    { $($entry)* }
-                });
                 $(
-                    ::core::intrinsics::mir::__internal_remove_let!({
+                    __debuginfo(stringify!($dbg_name), $dbg_data);
-                        {}
-                        { $($block)* }
-                    });
                 )*
 
-                RET
+                {
+                    // Finally, the contents of the basic blocks
+                    ::core::intrinsics::mir::__internal_remove_let!({
+                        {}
+                        { $($entry)* }
+                    });
+                    $(
+                        ::core::intrinsics::mir::__internal_remove_let!({
+                            {}
+                            { $($block)* }
+                        });
+                    )*
+
+                    RET
+                }
             }
         }
     }}

library/core/src/lib.rs

@@ -20,11 +20,19 @@
 // FIXME: Fill me in with more detail when the interface settles
 //! This library is built on the assumption of a few existing symbols:
 //!
-//! * `memcpy`, `memcmp`, `memset`, `strlen` - These are core memory routines which are
+//! * `memcpy`, `memmove`, `memset`, `memcmp`, `bcmp`, `strlen` - These are core memory routines
-//!   often generated by LLVM. Additionally, this library can make explicit
+//!   which are generated by Rust codegen backends. Additionally, this library can make explicit
-//!   calls to these functions. Their signatures are the same as found in C.
+//!   calls to `strlen`. Their signatures are the same as found in C, but there are extra
-//!   These functions are often provided by the system libc, but can also be
+//!   assumptions about their semantics: For `memcpy`, `memmove`, `memset`, `memcmp`, and `bcmp`, if
-//!   provided by the [compiler-builtins crate](https://crates.io/crates/compiler_builtins).
+//!   the `n` parameter is 0, the function is assumed to not be UB. Furthermore, for `memcpy`, if
+//!   source and target pointer are equal, the function is assumed to not be UB.
+//!   (Note that these are [standard assumptions](https://reviews.llvm.org/D86993) among compilers.)
+//!   These functions are often provided by the system libc, but can also be provided by the
+//!   [compiler-builtins crate](https://crates.io/crates/compiler_builtins).
+//!   Note that the library does not guarantee that it will always make these assumptions, so Rust
+//!   user code directly calling the C functions should follow the C specification! The advice for
+//!   Rust user code is to call the functions provided by this library instead (such as
+//!   `ptr::copy`).
 //!
 //! * `rust_begin_panic` - This function takes four arguments, a
 //!   `fmt::Arguments`, a `&'static str`, and two `u32`'s. These four arguments

library/core/src/num/dec2flt/fpu.rs

@@ -8,6 +8,17 @@
 // round to 80 bits causing double rounding to happen when values are eventually represented as
 // 32/64 bit float values. To overcome this, the FPU control word can be set so that the
 // computations are performed in the desired precision.
+//
+// Note that normally, it is Undefined Behavior to alter the FPU control word while Rust code runs.
+// The compiler assumes that the control word is always in its default state. However, in this
+// particular case the semantics with the altered control word are actually *more faithful*
+// to Rust semantics than the default -- arguably it is all the code that runs *outside* of the scope
+// of a `set_precision` guard that is wrong.
+// In other words, we are only using this to work around <https://github.com/rust-lang/rust/issues/114479>.
+// Sometimes killing UB with UB actually works...
+// (If this is used to set 32bit precision, there is still a risk that the compiler moves some 64bit
+// operation into the scope of the `set_precision` guard. So it's not like this is totally sound.
+// But it's not really any less sound than the default state of 80bit precision...)
 #[cfg(all(target_arch = "x86", not(target_feature = "sse2")))]
 mod fpu_precision {
     use core::arch::asm;

library/core/src/num/dec2flt/number.rs

@@ -51,6 +51,7 @@ fn is_fast_path<F: RawFloat>(&self) -> bool {
     /// There is an exception: disguised fast-path cases, where we can shift
     /// powers-of-10 from the exponent to the significant digits.
     pub fn try_fast_path<F: RawFloat>(&self) -> Option<F> {
+        // Here we need to work around <https://github.com/rust-lang/rust/issues/114479>.
         // The fast path crucially depends on arithmetic being rounded to the correct number of bits
         // without any intermediate rounding. On x86 (without SSE or SSE2) this requires the precision
         // of the x87 FPU stack to be changed so that it directly rounds to 64/32 bit.

src/tools/miri/src/borrow_tracker/stacked_borrows/mod.rs

@@ -615,7 +615,7 @@ fn sb_reborrow(
     ) -> InterpResult<'tcx, Option<Provenance>> {
         let this = self.eval_context_mut();
         // Ensure we bail out if the pointer goes out-of-bounds (see miri#1050).
-        this.check_ptr_access_align(place.ptr, size, Align::ONE, CheckInAllocMsg::InboundsTest)?;
+        this.check_ptr_access_align(place.ptr(), size, Align::ONE, CheckInAllocMsg::InboundsTest)?;
 
         // It is crucial that this gets called on all code paths, to ensure we track tag creation.
         let log_creation = |this: &MiriInterpCx<'mir, 'tcx>,
@@ -682,7 +682,7 @@ fn sb_reborrow(
             trace!(
                 "reborrow of size 0: reference {:?} derived from {:?} (pointee {})",
                 new_tag,
-                place.ptr,
+                place.ptr(),
                 place.layout.ty,
             );
             // Don't update any stacks for a zero-sized access; borrow stacks are per-byte and this
@@ -692,7 +692,7 @@ fn sb_reborrow(
             // attempt to use it for a non-zero-sized access.
             // Dangling slices are a common case here; it's valid to get their length but with raw
             // pointer tagging for example all calls to get_unchecked on them are invalid.
-            if let Ok((alloc_id, base_offset, orig_tag)) = this.ptr_try_get_alloc_id(place.ptr) {
+            if let Ok((alloc_id, base_offset, orig_tag)) = this.ptr_try_get_alloc_id(place.ptr()) {
                 log_creation(this, Some((alloc_id, base_offset, orig_tag)))?;
                 // Still give it the new provenance, it got retagged after all.
                 return Ok(Some(Provenance::Concrete { alloc_id, tag: new_tag }));
@@ -700,11 +700,11 @@ fn sb_reborrow(
                 // This pointer doesn't come with an AllocId. :shrug:
                 log_creation(this, None)?;
                 // Provenance unchanged.
-                return Ok(place.ptr.provenance);
+                return Ok(place.ptr().provenance);
             }
         }
 
-        let (alloc_id, base_offset, orig_tag) = this.ptr_get_alloc_id(place.ptr)?;
+        let (alloc_id, base_offset, orig_tag) = this.ptr_get_alloc_id(place.ptr())?;
         log_creation(this, Some((alloc_id, base_offset, orig_tag)))?;
 
         trace!(

src/tools/miri/src/borrow_tracker/tree_borrows/mod.rs

@@ -172,7 +172,7 @@ fn tb_reborrow(
         let this = self.eval_context_mut();
         // Ensure we bail out if the pointer goes out-of-bounds (see miri#1050).
         this.check_ptr_access_align(
-            place.ptr,
+            place.ptr(),
             ptr_size,
             Align::ONE,
             CheckInAllocMsg::InboundsTest,
@@ -197,7 +197,7 @@ fn tb_reborrow(
         };
 
         trace!("Reborrow of size {:?}", ptr_size);
-        let (alloc_id, base_offset, parent_prov) = match this.ptr_try_get_alloc_id(place.ptr) {
+        let (alloc_id, base_offset, parent_prov) = match this.ptr_try_get_alloc_id(place.ptr()) {
             Ok(data) => {
                 // Unlike SB, we *do* a proper retag for size 0 if can identify the allocation.
                 // After all, the pointer may be lazily initialized outside this initial range.
@@ -210,18 +210,18 @@ fn tb_reborrow(
                 trace!(
                     "reborrow of size 0: reference {:?} derived from {:?} (pointee {})",
                     new_tag,
-                    place.ptr,
+                    place.ptr(),
                     place.layout.ty,
                 );
                 log_creation(this, None)?;
                 // Keep original provenance.
-                return Ok(place.ptr.provenance);
+                return Ok(place.ptr().provenance);
             }
         };
         log_creation(this, Some((alloc_id, base_offset, parent_prov)))?;
 
         let orig_tag = match parent_prov {
-            ProvenanceExtra::Wildcard => return Ok(place.ptr.provenance), // TODO: handle wildcard pointers
+            ProvenanceExtra::Wildcard => return Ok(place.ptr().provenance), // TODO: handle wildcard pointers
             ProvenanceExtra::Concrete(tag) => tag,
         };
 

src/tools/miri/src/concurrency/data_race.rs

@@ -1018,7 +1018,7 @@ fn atomic_access_check(&self, place: &MPlaceTy<'tcx, Provenance>) -> InterpResul
         // be 8-aligned).
         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
         this.check_ptr_access_align(
-            place.ptr,
+            place.ptr(),
             place.layout.size,
             align,
             CheckInAllocMsg::MemoryAccessTest,
@@ -1031,7 +1031,7 @@ fn atomic_access_check(&self, place: &MPlaceTy<'tcx, Provenance>) -> InterpResul
         // We avoid `get_ptr_alloc` since we do *not* want to run the access hooks -- the actual
         // access will happen later.
         let (alloc_id, _offset, _prov) =
-            this.ptr_try_get_alloc_id(place.ptr).expect("there are no zero-sized atomic accesses");
+            this.ptr_try_get_alloc_id(place.ptr()).expect("there are no zero-sized atomic accesses");
         if this.get_alloc_mutability(alloc_id)? == Mutability::Not {
             // FIXME: make this prettier, once these messages have separate title/span/help messages.
             throw_ub_format!(
@@ -1135,7 +1135,7 @@ fn validate_atomic_op<A: Debug + Copy>(
         if let Some(data_race) = &this.machine.data_race {
             if data_race.race_detecting() {
                 let size = place.layout.size;
-                let (alloc_id, base_offset, _prov) = this.ptr_get_alloc_id(place.ptr)?;
+                let (alloc_id, base_offset, _prov) = this.ptr_get_alloc_id(place.ptr())?;
                 // Load and log the atomic operation.
                 // Note that atomic loads are possible even from read-only allocations, so `get_alloc_extra_mut` is not an option.
                 let alloc_meta = this.get_alloc_extra(alloc_id)?.data_race.as_ref().unwrap();
@@ -1143,7 +1143,7 @@ fn validate_atomic_op<A: Debug + Copy>(
                     "Atomic op({}) with ordering {:?} on {:?} (size={})",
                     description,
                     &atomic,
-                    place.ptr,
+                    place.ptr(),
                     size.bytes()
                 );
 
@@ -1186,7 +1186,7 @@ fn validate_atomic_op<A: Debug + Copy>(
                     {
                         log::trace!(
                             "Updated atomic memory({:?}, size={}) to {:#?}",
-                            place.ptr,
+                            place.ptr(),
                             size.bytes(),
                             mem_clocks.atomic_ops
                         );

src/tools/miri/src/concurrency/thread.rs

@@ -8,6 +8,7 @@
 use std::time::{Duration, SystemTime};
 
 use log::trace;
+use either::Either;
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::DefId;
@@ -259,8 +260,15 @@ fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) {
         return_place.visit_tags(visit);
         // Locals.
         for local in locals.iter() {
-            if let LocalValue::Live(value) = &local.value {
+            match local.as_mplace_or_imm() {
-                value.visit_tags(visit);
+                None => {}
+                Some(Either::Left((ptr, meta))) => {
+                    ptr.visit_tags(visit);
+                    meta.visit_tags(visit);
+                }
+                Some(Either::Right(imm)) => {
+                    imm.visit_tags(visit);
+                }
             }
         }
 

src/tools/miri/src/concurrency/weak_memory.rs

@@ -481,7 +481,7 @@ fn validate_overlapping_atomic(
         place: &MPlaceTy<'tcx, Provenance>,
     ) -> InterpResult<'tcx> {
         let this = self.eval_context_ref();
-        let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr)?;
+        let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr())?;
         if let crate::AllocExtra {
             weak_memory: Some(alloc_buffers),
             data_race: Some(alloc_clocks),
@@ -512,7 +512,7 @@ fn buffered_atomic_rmw(
         init: Scalar<Provenance>,
     ) -> InterpResult<'tcx> {
         let this = self.eval_context_mut();
-        let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr)?;
+        let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr())?;
         if let (
             crate::AllocExtra { weak_memory: Some(alloc_buffers), .. },
             crate::MiriMachine { data_race: Some(global), threads, .. },
@@ -539,7 +539,7 @@ fn buffered_atomic_read(
     ) -> InterpResult<'tcx, Scalar<Provenance>> {
         let this = self.eval_context_ref();
         if let Some(global) = &this.machine.data_race {
-            let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr)?;
+            let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr())?;
             if let Some(alloc_buffers) = this.get_alloc_extra(alloc_id)?.weak_memory.as_ref() {
                 if atomic == AtomicReadOrd::SeqCst {
                     global.sc_read(&this.machine.threads);
@@ -577,7 +577,7 @@ fn buffered_atomic_write(
         init: Scalar<Provenance>,
     ) -> InterpResult<'tcx> {
         let this = self.eval_context_mut();
-        let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(dest.ptr)?;
+        let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(dest.ptr())?;
         if let (
             crate::AllocExtra { weak_memory: Some(alloc_buffers), .. },
             crate::MiriMachine { data_race: Some(global), threads, .. },
@@ -627,7 +627,7 @@ fn perform_read_on_buffered_latest(
                 global.sc_read(&this.machine.threads);
             }
             let size = place.layout.size;
-            let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr)?;
+            let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr())?;
             if let Some(alloc_buffers) = this.get_alloc_extra(alloc_id)?.weak_memory.as_ref() {
                 let buffer = alloc_buffers
                     .get_or_create_store_buffer(alloc_range(base_offset, size), init)?;

src/tools/miri/src/diagnostics.rs

@@ -376,9 +376,9 @@ pub fn report_error<'tcx, 'mir>(
     for (i, frame) in ecx.active_thread_stack().iter().enumerate() {
         trace!("-------------------");
         trace!("Frame {}", i);
-        trace!("    return: {:?}", *frame.return_place);
+        trace!("    return: {:?}", frame.return_place);
         for (i, local) in frame.locals.iter().enumerate() {
-            trace!("    local {}: {:?}", i, local.value);
+            trace!("    local {}: {:?}", i, local);
         }
     }
 

src/tools/miri/src/eval.rs

@@ -242,10 +242,7 @@ fn on_main_stack_empty<'tcx>(
                 },
             Done => {
                 // Figure out exit code.
-                let ret_place = MPlaceTy::from_aligned_ptr(
+                let ret_place = this.machine.main_fn_ret_place.clone().unwrap();
-                    this.machine.main_fn_ret_place.unwrap().ptr,
-                    this.machine.layouts.isize,
-                );
                 let exit_code = this.read_target_isize(&ret_place)?;
                 // Need to call this ourselves since we are not going to return to the scheduler
                 // loop, and we want the main thread TLS to not show up as memory leaks.
@@ -308,7 +305,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>(
             let arg_type = Ty::new_array(tcx, tcx.types.u8, size);
             let arg_place =
                 ecx.allocate(ecx.layout_of(arg_type)?, MiriMemoryKind::Machine.into())?;
-            ecx.write_os_str_to_c_str(OsStr::new(arg), arg_place.ptr, size)?;
+            ecx.write_os_str_to_c_str(OsStr::new(arg), arg_place.ptr(), size)?;
             ecx.mark_immutable(&arg_place);
             argvs.push(arg_place.to_ref(&ecx));
         }
@@ -332,7 +329,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>(
                 ecx.allocate(ecx.machine.layouts.isize, MiriMemoryKind::Machine.into())?;
             ecx.write_scalar(argc, &argc_place)?;
             ecx.mark_immutable(&argc_place);
-            ecx.machine.argc = Some(*argc_place);
+            ecx.machine.argc = Some(argc_place.ptr());
 
             let argv_place = ecx.allocate(
                 ecx.layout_of(Ty::new_imm_ptr(tcx, tcx.types.unit))?,
@@ -340,7 +337,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>(
             )?;
             ecx.write_immediate(argv, &argv_place)?;
             ecx.mark_immutable(&argv_place);
-            ecx.machine.argv = Some(*argv_place);
+            ecx.machine.argv = Some(argv_place.ptr());
         }
         // Store command line as UTF-16 for Windows `GetCommandLineW`.
         {
@@ -351,7 +348,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>(
                 Ty::new_array(tcx, tcx.types.u16, u64::try_from(cmd_utf16.len()).unwrap());
             let cmd_place =
                 ecx.allocate(ecx.layout_of(cmd_type)?, MiriMemoryKind::Machine.into())?;
-            ecx.machine.cmd_line = Some(*cmd_place);
+            ecx.machine.cmd_line = Some(cmd_place.ptr());
             // Store the UTF-16 string. We just allocated so we know the bounds are fine.
             for (idx, &c) in cmd_utf16.iter().enumerate() {
                 let place = ecx.project_field(&cmd_place, idx)?;
@@ -364,7 +361,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>(
 
     // Return place (in static memory so that it does not count as leak).
     let ret_place = ecx.allocate(ecx.machine.layouts.isize, MiriMemoryKind::Machine.into())?;
-    ecx.machine.main_fn_ret_place = Some(*ret_place);
+    ecx.machine.main_fn_ret_place = Some(ret_place.clone());
     // Call start function.
 
     match entry_type {

src/tools/miri/src/helpers.rs

@@ -381,7 +381,7 @@ fn visit_freeze_sensitive(
         // Store how far we proceeded into the place so far. Everything to the left of
         // this offset has already been handled, in the sense that the frozen parts
         // have had `action` called on them.
-        let start_addr = place.ptr.addr();
+        let start_addr = place.ptr().addr();
         let mut cur_addr = start_addr;
         // Called when we detected an `UnsafeCell` at the given offset and size.
         // Calls `action` and advances `cur_ptr`.
@@ -413,7 +413,7 @@ fn visit_freeze_sensitive(
             let mut visitor = UnsafeCellVisitor {
                 ecx: this,
                 unsafe_cell_action: |place| {
-                    trace!("unsafe_cell_action on {:?}", place.ptr);
+                    trace!("unsafe_cell_action on {:?}", place.ptr());
                     // We need a size to go on.
                     let unsafe_cell_size = this
                         .size_and_align_of_mplace(place)?
@@ -422,7 +422,7 @@ fn visit_freeze_sensitive(
                         .unwrap_or_else(|| place.layout.size);
                     // Now handle this `UnsafeCell`, unless it is empty.
                     if unsafe_cell_size != Size::ZERO {
-                        unsafe_cell_action(&place.ptr, unsafe_cell_size)
+                        unsafe_cell_action(&place.ptr(), unsafe_cell_size)
                     } else {
                         Ok(())
                     }
@@ -432,7 +432,7 @@ fn visit_freeze_sensitive(
         }
         // The part between the end_ptr and the end of the place is also frozen.
         // So pretend there is a 0-sized `UnsafeCell` at the end.
-        unsafe_cell_action(&place.ptr.offset(size, this)?, Size::ZERO)?;
+        unsafe_cell_action(&place.ptr().offset(size, this)?, Size::ZERO)?;
         // Done!
         return Ok(());
 
@@ -994,10 +994,10 @@ fn check_shim<'a, const N: usize>(
     }
 
     /// Mark a machine allocation that was just created as immutable.
-    fn mark_immutable(&mut self, mplace: &MemPlace<Provenance>) {
+    fn mark_immutable(&mut self, mplace: &MPlaceTy<'tcx, Provenance>) {
         let this = self.eval_context_mut();
         // This got just allocated, so there definitely is a pointer here.
-        let provenance = mplace.ptr.into_pointer_or_addr().unwrap().provenance;
+        let provenance = mplace.ptr().into_pointer_or_addr().unwrap().provenance;
         this.alloc_mark_immutable(provenance.get_alloc_id().unwrap()).unwrap();
     }
 

src/tools/miri/src/machine.rs

@@ -396,14 +396,14 @@ pub struct MiriMachine<'mir, 'tcx> {
     pub(crate) env_vars: EnvVars<'tcx>,
 
     /// Return place of the main function.
-    pub(crate) main_fn_ret_place: Option<MemPlace<Provenance>>,
+    pub(crate) main_fn_ret_place: Option<MPlaceTy<'tcx, Provenance>>,
 
     /// Program arguments (`Option` because we can only initialize them after creating the ecx).
     /// These are *pointers* to argc/argv because macOS.
     /// We also need the full command line as one string because of Windows.
-    pub(crate) argc: Option<MemPlace<Provenance>>,
+    pub(crate) argc: Option<Pointer<Option<Provenance>>>,
-    pub(crate) argv: Option<MemPlace<Provenance>>,
+    pub(crate) argv: Option<Pointer<Option<Provenance>>>,
-    pub(crate) cmd_line: Option<MemPlace<Provenance>>,
+    pub(crate) cmd_line: Option<Pointer<Option<Provenance>>>,
 
     /// TLS state.
     pub(crate) tls: TlsData<'tcx>,
@@ -670,7 +670,7 @@ fn alloc_extern_static(
     ) -> InterpResult<'tcx> {
         let place = this.allocate(val.layout, MiriMemoryKind::ExternStatic.into())?;
         this.write_immediate(*val, &place)?;
-        Self::add_extern_static(this, name, place.ptr);
+        Self::add_extern_static(this, name, place.ptr());
         Ok(())
     }
 
@@ -686,7 +686,7 @@ fn init_extern_statics(this: &mut MiriInterpCx<'mir, 'tcx>) -> InterpResult<'tcx
                 Self::add_extern_static(
                     this,
                     "environ",
-                    this.machine.env_vars.environ.as_ref().unwrap().ptr,
+                    this.machine.env_vars.environ.as_ref().unwrap().ptr(),
                 );
                 // A couple zero-initialized pointer-sized extern statics.
                 // Most of them are for weak symbols, which we all set to null (indicating that the
@@ -703,7 +703,7 @@ fn init_extern_statics(this: &mut MiriInterpCx<'mir, 'tcx>) -> InterpResult<'tcx
                 Self::add_extern_static(
                     this,
                     "environ",
-                    this.machine.env_vars.environ.as_ref().unwrap().ptr,
+                    this.machine.env_vars.environ.as_ref().unwrap().ptr(),
                 );
             }
             "android" => {
@@ -1415,7 +1415,7 @@ fn after_local_allocated(
         local: mir::Local,
         mplace: &MPlaceTy<'tcx, Provenance>,
     ) -> InterpResult<'tcx> {
-        let Some(Provenance::Concrete { alloc_id, .. }) = mplace.ptr.provenance else {
+        let Some(Provenance::Concrete { alloc_id, .. }) = mplace.ptr().provenance else {
             panic!("after_local_allocated should only be called on fresh allocations");
         };
         let local_decl = &ecx.active_thread_stack()[frame].body.local_decls[local];

src/tools/miri/src/shims/backtrace.rs

@@ -89,7 +89,7 @@ fn handle_miri_get_backtrace(
                 }
 
                 this.write_immediate(
-                    Immediate::new_slice(Scalar::from_maybe_pointer(alloc.ptr, this), len, this),
+                    Immediate::new_slice(Scalar::from_maybe_pointer(alloc.ptr(), this), len, this),
                     dest,
                 )?;
             }

src/tools/miri/src/shims/env.rs

@@ -459,7 +459,7 @@ fn update_environ(&mut self) -> InterpResult<'tcx> {
             let place = this.project_field(&vars_place, idx)?;
             this.write_pointer(var, &place)?;
         }
-        this.write_pointer(vars_place.ptr, &this.machine.env_vars.environ.clone().unwrap())?;
+        this.write_pointer(vars_place.ptr(), &this.machine.env_vars.environ.clone().unwrap())?;
 
         Ok(())
     }

src/tools/miri/src/shims/foreign_items.rs

@@ -324,7 +324,7 @@ fn emulate_foreign_item(
         // Second: functions that return immediately.
         match this.emulate_foreign_item_by_name(link_name, abi, args, dest)? {
             EmulateByNameResult::NeedsJumping => {
-                trace!("{:?}", this.dump_place(**dest));
+                trace!("{:?}", this.dump_place(dest));
                 this.go_to_block(ret);
             }
             EmulateByNameResult::AlreadyJumped => (),

src/tools/miri/src/shims/intrinsics/mod.rs

@@ -62,7 +62,7 @@ fn call_intrinsic(
         // The rest jumps to `ret` immediately.
         this.emulate_intrinsic_by_name(intrinsic_name, args, dest)?;
 
-        trace!("{:?}", this.dump_place(**dest));
+        trace!("{:?}", this.dump_place(dest));
         this.go_to_block(ret);
         Ok(())
     }

src/tools/miri/src/shims/os_str.rs

@@ -143,9 +143,9 @@ fn alloc_os_str_as_c_str(
 
         let arg_type = Ty::new_array(this.tcx.tcx, this.tcx.types.u8, size);
         let arg_place = this.allocate(this.layout_of(arg_type).unwrap(), memkind)?;
-        let (written, _) = self.write_os_str_to_c_str(os_str, arg_place.ptr, size).unwrap();
+        let (written, _) = self.write_os_str_to_c_str(os_str, arg_place.ptr(), size).unwrap();
         assert!(written);
-        Ok(arg_place.ptr)
+        Ok(arg_place.ptr())
     }
 
     /// Allocate enough memory to store the given `OsStr` as a null-terminated sequence of `u16`.
@@ -160,9 +160,9 @@ fn alloc_os_str_as_wide_str(
         let arg_type = Ty::new_array(this.tcx.tcx, this.tcx.types.u16, size);
         let arg_place = this.allocate(this.layout_of(arg_type).unwrap(), memkind)?;
         let (written, _) =
-            self.write_os_str_to_wide_str(os_str, arg_place.ptr, size, /*truncate*/ false).unwrap();
+            self.write_os_str_to_wide_str(os_str, arg_place.ptr(), size, /*truncate*/ false).unwrap();
         assert!(written);
-        Ok(arg_place.ptr)
+        Ok(arg_place.ptr())
     }
 
     /// Read a null-terminated sequence of bytes, and perform path separator conversion if needed.

src/tools/miri/src/shims/unix/android/dlsym.rs

@@ -47,7 +47,7 @@ fn call_dlsym(
             }
         }
 
-        log::trace!("{:?}", this.dump_place(**dest));
+        log::trace!("{:?}", this.dump_place(dest));
         this.go_to_block(ret);
         Ok(())
     }

src/tools/miri/src/shims/unix/foreign_items.rs

@@ -597,7 +597,7 @@ fn emulate_foreign_item_by_name(
                 this.write_pointer(buf, &pw_dir)?;
 
                 if written {
-                    this.write_pointer(pwd.ptr, &result)?;
+                    this.write_pointer(pwd.ptr(), &result)?;
                     this.write_null(dest)?;
                 } else {
                     this.write_null(&result)?;

src/tools/miri/src/shims/unix/fs.rs

@@ -1439,7 +1439,7 @@ fn macos_readdir_r(
                 let file_name = dir_entry.file_name(); // not a Path as there are no separators!
                 let (name_fits, file_name_buf_len) = this.write_os_str_to_c_str(
                     &file_name,
-                    name_place.ptr,
+                    name_place.ptr(),
                     name_place.layout.size.bytes(),
                 )?;
                 let file_name_len = file_name_buf_len.checked_sub(1).unwrap();

src/tools/miri/src/shims/unix/linux/sync.rs

@@ -35,7 +35,7 @@ pub fn futex<'tcx>(
     let thread = this.get_active_thread();
     // This is a vararg function so we have to bring our own type for this pointer.
     let addr = MPlaceTy::from_aligned_ptr(addr, this.machine.layouts.i32);
-    let addr_usize = addr.ptr.addr().bytes();
+    let addr_usize = addr.ptr().addr().bytes();
 
     let futex_private = this.eval_libc_i32("FUTEX_PRIVATE_FLAG");
     let futex_wait = this.eval_libc_i32("FUTEX_WAIT");
@@ -90,7 +90,7 @@ pub fn futex<'tcx>(
                 &this.read_immediate(&args[3])?,
                 this.libc_ty_layout("timespec"),
             )?;
-            let timeout_time = if this.ptr_is_null(timeout.ptr)? {
+            let timeout_time = if this.ptr_is_null(timeout.ptr())? {
                 None
             } else {
                 let realtime = op & futex_realtime == futex_realtime;

src/tools/miri/src/shims/unix/macos/dlsym.rs

@@ -45,7 +45,7 @@ fn call_dlsym(
             }
         }
 
-        trace!("{:?}", this.dump_place(**dest));
+        trace!("{:?}", this.dump_place(dest));
         this.go_to_block(ret);
         Ok(())
     }

src/tools/miri/src/shims/unix/macos/foreign_items.rs

@@ -91,7 +91,7 @@ fn emulate_foreign_item_by_name(
                         .environ
                         .as_ref()
                         .expect("machine must be initialized")
-                        .ptr,
+                        .ptr(),
                     dest,
                 )?;
             }
@@ -113,14 +113,14 @@ fn emulate_foreign_item_by_name(
             "_NSGetArgc" => {
                 let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
                 this.write_pointer(
-                    this.machine.argc.expect("machine must be initialized").ptr,
+                    this.machine.argc.expect("machine must be initialized"),
                     dest,
                 )?;
             }
             "_NSGetArgv" => {
                 let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
                 this.write_pointer(
-                    this.machine.argv.expect("machine must be initialized").ptr,
+                    this.machine.argv.expect("machine must be initialized"),
                     dest,
                 )?;
             }

src/tools/miri/src/shims/windows/dlsym.rs

@@ -75,7 +75,7 @@ fn call_dlsym(
             }
         }
 
-        trace!("{:?}", this.dump_place(**dest));
+        trace!("{:?}", this.dump_place(dest));
         this.go_to_block(ret);
         Ok(())
     }

src/tools/miri/src/shims/windows/foreign_items.rs

@@ -190,7 +190,7 @@ fn emulate_foreign_item_by_name(
                     this.deref_pointer_as(system_info, this.windows_ty_layout("SYSTEM_INFO"))?;
                 // Initialize with `0`.
                 this.write_bytes_ptr(
-                    system_info.ptr,
+                    system_info.ptr(),
                     iter::repeat(0u8).take(system_info.layout.size.bytes_usize()),
                 )?;
                 // Set selected fields.
@@ -235,7 +235,7 @@ fn emulate_foreign_item_by_name(
             "GetCommandLineW" => {
                 let [] = this.check_shim(abi, Abi::System { unwind: false }, link_name, args)?;
                 this.write_pointer(
-                    this.machine.cmd_line.expect("machine must be initialized").ptr,
+                    this.machine.cmd_line.expect("machine must be initialized"),
                     dest,
                 )?;
             }

src/tools/miri/src/tag_gc.rs

@@ -1,3 +1,5 @@
+use either::Either;
+
 use rustc_data_structures::fx::FxHashSet;
 
 use crate::*;
@@ -81,46 +83,33 @@ fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) {
     }
 }
 
-impl VisitTags for MemPlace<Provenance> {
+impl VisitTags for ImmTy<'_, Provenance> {
     fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) {
-        let MemPlace { ptr, meta } = self;
+        (**self).visit_tags(visit)
-        ptr.visit_tags(visit);
-        meta.visit_tags(visit);
     }
 }
 
 impl VisitTags for MPlaceTy<'_, Provenance> {
     fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) {
-        (**self).visit_tags(visit)
+        self.ptr().visit_tags(visit);
-    }
+        self.meta().visit_tags(visit);
-}
-
-impl VisitTags for Place<Provenance> {
-    fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) {
-        match self {
-            Place::Ptr(p) => p.visit_tags(visit),
-            Place::Local { .. } => {
-                // Will be visited as part of the stack frame.
-            }
-        }
     }
 }
 
 impl VisitTags for PlaceTy<'_, Provenance> {
     fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) {
-        (**self).visit_tags(visit)
+        match self.as_mplace_or_local() {
+            Either::Left(mplace) => mplace.visit_tags(visit),
+            Either::Right(_) => (),
+        }
     }
 }
 
-impl VisitTags for Operand<Provenance> {
+impl VisitTags for OpTy<'_, Provenance> {
     fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) {
-        match self {
+        match self.as_mplace_or_imm() {
-            Operand::Immediate(imm) => {
+            Either::Left(mplace) => mplace.visit_tags(visit),
-                imm.visit_tags(visit);
+            Either::Right(imm) => imm.visit_tags(visit),
-            }
-            Operand::Indirect(p) => {
-                p.visit_tags(visit);
-            }
         }
     }
 }

tests/mir-opt/building/custom/debuginfo.numbered.built.after.mir

@@ -0,0 +1,11 @@
+// MIR for `numbered` after built
+
+fn numbered(_1: (u32, i32)) -> () {
+    debug first => (_1.0: u32);
+    debug second => (_1.0: u32);
+    let mut _0: ();
+
+    bb0: {
+        return;
+    }
+}

tests/mir-opt/building/custom/debuginfo.pointee.built.after.mir

@@ -0,0 +1,10 @@
+// MIR for `pointee` after built
+
+fn pointee(_1: &mut Option<i32>) -> () {
+    debug foo => (((*_1) as variant#1).0: i32);
+    let mut _0: ();
+
+    bb0: {
+        return;
+    }
+}

tests/mir-opt/building/custom/debuginfo.rs

@@ -0,0 +1,71 @@
+#![feature(custom_mir, core_intrinsics)]
+
+extern crate core;
+use core::intrinsics::mir::*;
+
+// EMIT_MIR debuginfo.pointee.built.after.mir
+#[custom_mir(dialect = "built")]
+fn pointee(opt: &mut Option<i32>) {
+    mir!(
+        debug foo => Field::<i32>(Variant(*opt, 1), 0);
+        {
+            Return()
+        }
+    )
+}
+
+// EMIT_MIR debuginfo.numbered.built.after.mir
+#[custom_mir(dialect = "analysis", phase = "post-cleanup")]
+fn numbered(i: (u32, i32)) {
+    mir!(
+        debug first => i.0;
+        debug second => i.0;
+        {
+            Return()
+        }
+    )
+}
+
+struct S { x: f32 }
+
+// EMIT_MIR debuginfo.structured.built.after.mir
+#[custom_mir(dialect = "analysis", phase = "post-cleanup")]
+fn structured(i: S) {
+    mir!(
+        debug x => i.x;
+        {
+            Return()
+        }
+    )
+}
+
+// EMIT_MIR debuginfo.variant.built.after.mir
+#[custom_mir(dialect = "built")]
+fn variant(opt: Option<i32>) {
+    mir!(
+        debug inner => Field::<i32>(Variant(opt, 1), 0);
+        {
+            Return()
+        }
+    )
+}
+
+// EMIT_MIR debuginfo.variant_deref.built.after.mir
+#[custom_mir(dialect = "built")]
+fn variant_deref(opt: Option<&i32>) {
+    mir!(
+        debug pointer => Field::<&i32>(Variant(opt, 1), 0);
+        debug deref => *Field::<&i32>(Variant(opt, 1), 0);
+        {
+            Return()
+        }
+    )
+}
+
+fn main() {
+    numbered((5, 6));
+    structured(S { x: 5. });
+    variant(Some(5));
+    variant_deref(Some(&5));
+    pointee(&mut Some(5));
+}

tests/mir-opt/building/custom/debuginfo.structured.built.after.mir

@@ -0,0 +1,10 @@
+// MIR for `structured` after built
+
+fn structured(_1: S) -> () {
+    debug x => (_1.0: f32);
+    let mut _0: ();
+
+    bb0: {
+        return;
+    }
+}

tests/mir-opt/building/custom/debuginfo.variant.built.after.mir

@@ -0,0 +1,10 @@
+// MIR for `variant` after built
+
+fn variant(_1: Option<i32>) -> () {
+    debug inner => ((_1 as variant#1).0: i32);
+    let mut _0: ();
+
+    bb0: {
+        return;
+    }
+}

tests/mir-opt/building/custom/debuginfo.variant_deref.built.after.mir

@@ -0,0 +1,11 @@
+// MIR for `variant_deref` after built
+
+fn variant_deref(_1: Option<&i32>) -> () {
+    debug pointer => ((_1 as variant#1).0: &i32);
+    debug deref => (*((_1 as variant#1).0: &i32));
+    let mut _0: ();
+
+    bb0: {
+        return;
+    }
+}