438 lines
21 KiB
Rust
438 lines
21 KiB
Rust
//! This module specifies the type based interner for constants.
|
|
//!
|
|
//! After a const evaluation has computed a value, before we destroy the const evaluator's session
|
|
//! memory, we need to extract all memory allocations to the global memory pool so they stay around.
|
|
//!
|
|
//! In principle, this is not very complicated: we recursively walk the final value, follow all the
|
|
//! pointers, and move all reachable allocations to the global `tcx` memory. The only complication
|
|
//! is picking the right mutability for the allocations in a `static` initializer: we want to make
|
|
//! as many allocations as possible immutable so LLVM can put them into read-only memory. At the
|
|
//! same time, we need to make memory that could be mutated by the program mutable to avoid
|
|
//! incorrect compilations. To achieve this, we do a type-based traversal of the final value,
|
|
//! tracking mutable and shared references and `UnsafeCell` to determine the current mutability.
|
|
//! (In principle, we could skip this type-based part for `const` and promoteds, as they need to be
|
|
//! always immutable. At least for `const` however we use this opportunity to reject any `const`
|
|
//! that contains allocations whose mutability we cannot identify.)
|
|
|
|
use super::validity::RefTracking;
|
|
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
|
|
use rustc_errors::ErrorReported;
|
|
use rustc_hir as hir;
|
|
use rustc_middle::mir::interpret::InterpResult;
|
|
use rustc_middle::ty::{self, layout::TyAndLayout, Ty};
|
|
|
|
use rustc_ast::Mutability;
|
|
|
|
use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy, ValueVisitor};
|
|
use crate::const_eval;
|
|
|
|
pub trait CompileTimeMachine<'mir, 'tcx, T> = Machine<
|
|
'mir,
|
|
'tcx,
|
|
MemoryKind = T,
|
|
PointerTag = AllocId,
|
|
ExtraFnVal = !,
|
|
FrameExtra = (),
|
|
AllocExtra = (),
|
|
MemoryMap = FxHashMap<AllocId, (MemoryKind<T>, Allocation)>,
|
|
>;
|
|
|
|
struct InternVisitor<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval::MemoryKind>> {
|
|
/// The ectx from which we intern.
|
|
ecx: &'rt mut InterpCx<'mir, 'tcx, M>,
|
|
/// Previously encountered safe references.
|
|
ref_tracking: &'rt mut RefTracking<(MPlaceTy<'tcx>, InternMode)>,
|
|
/// A list of all encountered allocations. After type-based interning, we traverse this list to
|
|
/// also intern allocations that are only referenced by a raw pointer or inside a union.
|
|
leftover_allocations: &'rt mut FxHashSet<AllocId>,
|
|
/// The root kind of the value that we're looking at. This field is never mutated for a
|
|
/// particular allocation. It is primarily used to make as many allocations as possible
|
|
/// read-only so LLVM can place them in const memory.
|
|
mode: InternMode,
|
|
/// This field stores whether we are *currently* inside an `UnsafeCell`. This can affect
|
|
/// the intern mode of references we encounter.
|
|
inside_unsafe_cell: bool,
|
|
}
|
|
|
|
#[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)]
|
|
enum InternMode {
|
|
/// A static and its current mutability. Below shared references inside a `static mut`,
|
|
/// this is *immutable*, and below mutable references inside an `UnsafeCell`, this
|
|
/// is *mutable*.
|
|
Static(hir::Mutability),
|
|
/// A `const`.
|
|
Const,
|
|
}
|
|
|
|
/// Signalling data structure to ensure we don't recurse
|
|
/// into the memory of other constants or statics
|
|
struct IsStaticOrFn;
|
|
|
|
/// Intern an allocation without looking at its children.
|
|
/// `mode` is the mode of the environment where we found this pointer.
|
|
/// `mutablity` is the mutability of the place to be interned; even if that says
|
|
/// `immutable` things might become mutable if `ty` is not frozen.
|
|
/// `ty` can be `None` if there is no potential interior mutability
|
|
/// to account for (e.g. for vtables).
|
|
fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval::MemoryKind>>(
|
|
ecx: &'rt mut InterpCx<'mir, 'tcx, M>,
|
|
leftover_allocations: &'rt mut FxHashSet<AllocId>,
|
|
alloc_id: AllocId,
|
|
mode: InternMode,
|
|
ty: Option<Ty<'tcx>>,
|
|
) -> Option<IsStaticOrFn> {
|
|
trace!("intern_shallow {:?} with {:?}", alloc_id, mode);
|
|
// remove allocation
|
|
let tcx = ecx.tcx;
|
|
let (kind, mut alloc) = match ecx.memory.alloc_map.remove(&alloc_id) {
|
|
Some(entry) => entry,
|
|
None => {
|
|
// Pointer not found in local memory map. It is either a pointer to the global
|
|
// map, or dangling.
|
|
// If the pointer is dangling (neither in local nor global memory), we leave it
|
|
// to validation to error -- it has the much better error messages, pointing out where
|
|
// in the value the dangling reference lies.
|
|
// The `delay_span_bug` ensures that we don't forget such a check in validation.
|
|
if tcx.get_global_alloc(alloc_id).is_none() {
|
|
tcx.sess.delay_span_bug(ecx.tcx.span, "tried to intern dangling pointer");
|
|
}
|
|
// treat dangling pointers like other statics
|
|
// just to stop trying to recurse into them
|
|
return Some(IsStaticOrFn);
|
|
}
|
|
};
|
|
// This match is just a canary for future changes to `MemoryKind`, which most likely need
|
|
// changes in this function.
|
|
match kind {
|
|
MemoryKind::Stack
|
|
| MemoryKind::Machine(const_eval::MemoryKind::Heap)
|
|
| MemoryKind::CallerLocation => {}
|
|
}
|
|
// Set allocation mutability as appropriate. This is used by LLVM to put things into
|
|
// read-only memory, and also by Miri when evaluating other globals that
|
|
// access this one.
|
|
if let InternMode::Static(mutability) = mode {
|
|
// For this, we need to take into account `UnsafeCell`. When `ty` is `None`, we assume
|
|
// no interior mutability.
|
|
let frozen = ty.map_or(true, |ty| ty.is_freeze(ecx.tcx, ecx.param_env));
|
|
// For statics, allocation mutability is the combination of place mutability and
|
|
// type mutability.
|
|
// The entire allocation needs to be mutable if it contains an `UnsafeCell` anywhere.
|
|
let immutable = mutability == Mutability::Not && frozen;
|
|
if immutable {
|
|
alloc.mutability = Mutability::Not;
|
|
} else {
|
|
// Just making sure we are not "upgrading" an immutable allocation to mutable.
|
|
assert_eq!(alloc.mutability, Mutability::Mut);
|
|
}
|
|
} else {
|
|
// No matter what, *constants are never mutable*. Mutating them is UB.
|
|
// See const_eval::machine::MemoryExtra::can_access_statics for why
|
|
// immutability is so important.
|
|
|
|
// Validation will ensure that there is no `UnsafeCell` on an immutable allocation.
|
|
alloc.mutability = Mutability::Not;
|
|
};
|
|
// link the alloc id to the actual allocation
|
|
let alloc = tcx.intern_const_alloc(alloc);
|
|
leftover_allocations.extend(alloc.relocations().iter().map(|&(_, alloc_id)| alloc_id));
|
|
tcx.set_alloc_id_memory(alloc_id, alloc);
|
|
None
|
|
}
|
|
|
|
impl<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval::MemoryKind>>
|
|
InternVisitor<'rt, 'mir, 'tcx, M>
|
|
{
|
|
fn intern_shallow(
|
|
&mut self,
|
|
alloc_id: AllocId,
|
|
mode: InternMode,
|
|
ty: Option<Ty<'tcx>>,
|
|
) -> Option<IsStaticOrFn> {
|
|
intern_shallow(self.ecx, self.leftover_allocations, alloc_id, mode, ty)
|
|
}
|
|
}
|
|
|
|
impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::MemoryKind>>
|
|
ValueVisitor<'mir, 'tcx, M> for InternVisitor<'rt, 'mir, 'tcx, M>
|
|
{
|
|
type V = MPlaceTy<'tcx>;
|
|
|
|
#[inline(always)]
|
|
fn ecx(&self) -> &InterpCx<'mir, 'tcx, M> {
|
|
&self.ecx
|
|
}
|
|
|
|
fn visit_aggregate(
|
|
&mut self,
|
|
mplace: &MPlaceTy<'tcx>,
|
|
fields: impl Iterator<Item = InterpResult<'tcx, Self::V>>,
|
|
) -> InterpResult<'tcx> {
|
|
// ZSTs cannot contain pointers, so we can skip them.
|
|
if mplace.layout.is_zst() {
|
|
return Ok(());
|
|
}
|
|
|
|
if let Some(def) = mplace.layout.ty.ty_adt_def() {
|
|
if Some(def.did) == self.ecx.tcx.lang_items().unsafe_cell_type() {
|
|
// We are crossing over an `UnsafeCell`, we can mutate again. This means that
|
|
// References we encounter inside here are interned as pointing to mutable
|
|
// allocations.
|
|
// Remember the `old` value to handle nested `UnsafeCell`.
|
|
let old = std::mem::replace(&mut self.inside_unsafe_cell, true);
|
|
let walked = self.walk_aggregate(mplace, fields);
|
|
self.inside_unsafe_cell = old;
|
|
return walked;
|
|
}
|
|
}
|
|
|
|
self.walk_aggregate(mplace, fields)
|
|
}
|
|
|
|
fn visit_value(&mut self, mplace: &MPlaceTy<'tcx>) -> InterpResult<'tcx> {
|
|
// Handle Reference types, as these are the only relocations supported by const eval.
|
|
// Raw pointers (and boxes) are handled by the `leftover_relocations` logic.
|
|
let tcx = self.ecx.tcx;
|
|
let ty = mplace.layout.ty;
|
|
if let ty::Ref(_, referenced_ty, ref_mutability) = *ty.kind() {
|
|
let value = self.ecx.read_immediate(&(*mplace).into())?;
|
|
let mplace = self.ecx.ref_to_mplace(&value)?;
|
|
assert_eq!(mplace.layout.ty, referenced_ty);
|
|
// Handle trait object vtables.
|
|
if let ty::Dynamic(..) =
|
|
tcx.struct_tail_erasing_lifetimes(referenced_ty, self.ecx.param_env).kind()
|
|
{
|
|
let ptr = self.ecx.scalar_to_ptr(mplace.meta.unwrap_meta());
|
|
if let Some(alloc_id) = ptr.provenance {
|
|
// Explicitly choose const mode here, since vtables are immutable, even
|
|
// if the reference of the fat pointer is mutable.
|
|
self.intern_shallow(alloc_id, InternMode::Const, None);
|
|
} else {
|
|
// Validation will error (with a better message) on an invalid vtable pointer.
|
|
// Let validation show the error message, but make sure it *does* error.
|
|
tcx.sess
|
|
.delay_span_bug(tcx.span, "vtables pointers cannot be integer pointers");
|
|
}
|
|
}
|
|
// Check if we have encountered this pointer+layout combination before.
|
|
// Only recurse for allocation-backed pointers.
|
|
if let Some(alloc_id) = mplace.ptr.provenance {
|
|
// Compute the mode with which we intern this. Our goal here is to make as many
|
|
// statics as we can immutable so they can be placed in read-only memory by LLVM.
|
|
let ref_mode = match self.mode {
|
|
InternMode::Static(mutbl) => {
|
|
// In statics, merge outer mutability with reference mutability and
|
|
// take into account whether we are in an `UnsafeCell`.
|
|
|
|
// The only way a mutable reference actually works as a mutable reference is
|
|
// by being in a `static mut` directly or behind another mutable reference.
|
|
// If there's an immutable reference or we are inside a `static`, then our
|
|
// mutable reference is equivalent to an immutable one. As an example:
|
|
// `&&mut Foo` is semantically equivalent to `&&Foo`
|
|
match ref_mutability {
|
|
_ if self.inside_unsafe_cell => {
|
|
// Inside an `UnsafeCell` is like inside a `static mut`, the "outer"
|
|
// mutability does not matter.
|
|
InternMode::Static(ref_mutability)
|
|
}
|
|
Mutability::Not => {
|
|
// A shared reference, things become immutable.
|
|
// We do *not* consider `freeze` here: `intern_shallow` considers
|
|
// `freeze` for the actual mutability of this allocation; the intern
|
|
// mode for references contained in this allocation is tracked more
|
|
// precisely when traversing the referenced data (by tracking
|
|
// `UnsafeCell`). This makes sure that `&(&i32, &Cell<i32>)` still
|
|
// has the left inner reference interned into a read-only
|
|
// allocation.
|
|
InternMode::Static(Mutability::Not)
|
|
}
|
|
Mutability::Mut => {
|
|
// Mutable reference.
|
|
InternMode::Static(mutbl)
|
|
}
|
|
}
|
|
}
|
|
InternMode::Const => {
|
|
// Ignore `UnsafeCell`, everything is immutable. Validity does some sanity
|
|
// checking for mutable references that we encounter -- they must all be
|
|
// ZST.
|
|
InternMode::Const
|
|
}
|
|
};
|
|
match self.intern_shallow(alloc_id, ref_mode, Some(referenced_ty)) {
|
|
// No need to recurse, these are interned already and statics may have
|
|
// cycles, so we don't want to recurse there
|
|
Some(IsStaticOrFn) => {}
|
|
// intern everything referenced by this value. The mutability is taken from the
|
|
// reference. It is checked above that mutable references only happen in
|
|
// `static mut`
|
|
None => self.ref_tracking.track((mplace, ref_mode), || ()),
|
|
}
|
|
}
|
|
Ok(())
|
|
} else {
|
|
// Not a reference -- proceed recursively.
|
|
self.walk_value(mplace)
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)]
|
|
pub enum InternKind {
|
|
/// The `mutability` of the static, ignoring the type which may have interior mutability.
|
|
Static(hir::Mutability),
|
|
Constant,
|
|
Promoted,
|
|
}
|
|
|
|
/// Intern `ret` and everything it references.
|
|
///
|
|
/// This *cannot raise an interpreter error*. Doing so is left to validation, which
|
|
/// tracks where in the value we are and thus can show much better error messages.
|
|
/// Any errors here would anyway be turned into `const_err` lints, whereas validation failures
|
|
/// are hard errors.
|
|
#[tracing::instrument(level = "debug", skip(ecx))]
|
|
pub fn intern_const_alloc_recursive<M: CompileTimeMachine<'mir, 'tcx, const_eval::MemoryKind>>(
|
|
ecx: &mut InterpCx<'mir, 'tcx, M>,
|
|
intern_kind: InternKind,
|
|
ret: &MPlaceTy<'tcx>,
|
|
) -> Result<(), ErrorReported>
|
|
where
|
|
'tcx: 'mir,
|
|
{
|
|
let tcx = ecx.tcx;
|
|
let base_intern_mode = match intern_kind {
|
|
InternKind::Static(mutbl) => InternMode::Static(mutbl),
|
|
// `Constant` includes array lengths.
|
|
InternKind::Constant | InternKind::Promoted => InternMode::Const,
|
|
};
|
|
|
|
// Type based interning.
|
|
// `ref_tracking` tracks typed references we have already interned and still need to crawl for
|
|
// more typed information inside them.
|
|
// `leftover_allocations` collects *all* allocations we see, because some might not
|
|
// be available in a typed way. They get interned at the end.
|
|
let mut ref_tracking = RefTracking::empty();
|
|
let leftover_allocations = &mut FxHashSet::default();
|
|
|
|
// start with the outermost allocation
|
|
intern_shallow(
|
|
ecx,
|
|
leftover_allocations,
|
|
// The outermost allocation must exist, because we allocated it with
|
|
// `Memory::allocate`.
|
|
ret.ptr.provenance.unwrap(),
|
|
base_intern_mode,
|
|
Some(ret.layout.ty),
|
|
);
|
|
|
|
ref_tracking.track((*ret, base_intern_mode), || ());
|
|
|
|
while let Some(((mplace, mode), _)) = ref_tracking.todo.pop() {
|
|
let res = InternVisitor {
|
|
ref_tracking: &mut ref_tracking,
|
|
ecx,
|
|
mode,
|
|
leftover_allocations,
|
|
inside_unsafe_cell: false,
|
|
}
|
|
.visit_value(&mplace);
|
|
// We deliberately *ignore* interpreter errors here. When there is a problem, the remaining
|
|
// references are "leftover"-interned, and later validation will show a proper error
|
|
// and point at the right part of the value causing the problem.
|
|
match res {
|
|
Ok(()) => {}
|
|
Err(error) => {
|
|
ecx.tcx.sess.delay_span_bug(
|
|
ecx.tcx.span,
|
|
&format!(
|
|
"error during interning should later cause validation failure: {}",
|
|
error
|
|
),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Intern the rest of the allocations as mutable. These might be inside unions, padding, raw
|
|
// pointers, ... So we can't intern them according to their type rules
|
|
|
|
let mut todo: Vec<_> = leftover_allocations.iter().cloned().collect();
|
|
while let Some(alloc_id) = todo.pop() {
|
|
if let Some((_, mut alloc)) = ecx.memory.alloc_map.remove(&alloc_id) {
|
|
// We can't call the `intern_shallow` method here, as its logic is tailored to safe
|
|
// references and a `leftover_allocations` set (where we only have a todo-list here).
|
|
// So we hand-roll the interning logic here again.
|
|
match intern_kind {
|
|
// Statics may contain mutable allocations even behind relocations.
|
|
// Even for immutable statics it would be ok to have mutable allocations behind
|
|
// raw pointers, e.g. for `static FOO: *const AtomicUsize = &AtomicUsize::new(42)`.
|
|
InternKind::Static(_) => {}
|
|
// Raw pointers in promoteds may only point to immutable things so we mark
|
|
// everything as immutable.
|
|
// It is UB to mutate through a raw pointer obtained via an immutable reference:
|
|
// Since all references and pointers inside a promoted must by their very definition
|
|
// be created from an immutable reference (and promotion also excludes interior
|
|
// mutability), mutating through them would be UB.
|
|
// There's no way we can check whether the user is using raw pointers correctly,
|
|
// so all we can do is mark this as immutable here.
|
|
InternKind::Promoted => {
|
|
// See const_eval::machine::MemoryExtra::can_access_statics for why
|
|
// immutability is so important.
|
|
alloc.mutability = Mutability::Not;
|
|
}
|
|
InternKind::Constant => {
|
|
// If it's a constant, we should not have any "leftovers" as everything
|
|
// is tracked by const-checking.
|
|
// FIXME: downgrade this to a warning? It rejects some legitimate consts,
|
|
// such as `const CONST_RAW: *const Vec<i32> = &Vec::new() as *const _;`.
|
|
ecx.tcx
|
|
.sess
|
|
.span_err(ecx.tcx.span, "untyped pointers are not allowed in constant");
|
|
// For better errors later, mark the allocation as immutable.
|
|
alloc.mutability = Mutability::Not;
|
|
}
|
|
}
|
|
let alloc = tcx.intern_const_alloc(alloc);
|
|
tcx.set_alloc_id_memory(alloc_id, alloc);
|
|
for &(_, alloc_id) in alloc.relocations().iter() {
|
|
if leftover_allocations.insert(alloc_id) {
|
|
todo.push(alloc_id);
|
|
}
|
|
}
|
|
} else if ecx.memory.dead_alloc_map.contains_key(&alloc_id) {
|
|
// Codegen does not like dangling pointers, and generally `tcx` assumes that
|
|
// all allocations referenced anywhere actually exist. So, make sure we error here.
|
|
ecx.tcx.sess.span_err(ecx.tcx.span, "encountered dangling pointer in final constant");
|
|
return Err(ErrorReported);
|
|
} else if ecx.tcx.get_global_alloc(alloc_id).is_none() {
|
|
// We have hit an `AllocId` that is neither in local or global memory and isn't
|
|
// marked as dangling by local memory. That should be impossible.
|
|
span_bug!(ecx.tcx.span, "encountered unknown alloc id {:?}", alloc_id);
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>>
|
|
InterpCx<'mir, 'tcx, M>
|
|
{
|
|
/// A helper function that allocates memory for the layout given and gives you access to mutate
|
|
/// it. Once your own mutation code is done, the backing `Allocation` is removed from the
|
|
/// current `Memory` and returned.
|
|
pub fn intern_with_temp_alloc(
|
|
&mut self,
|
|
layout: TyAndLayout<'tcx>,
|
|
f: impl FnOnce(
|
|
&mut InterpCx<'mir, 'tcx, M>,
|
|
&PlaceTy<'tcx, M::PointerTag>,
|
|
) -> InterpResult<'tcx, ()>,
|
|
) -> InterpResult<'tcx, &'tcx Allocation> {
|
|
let dest = self.allocate(layout, MemoryKind::Stack)?;
|
|
f(self, &dest.into())?;
|
|
let mut alloc = self.memory.alloc_map.remove(&dest.ptr.provenance.unwrap()).unwrap().1;
|
|
alloc.mutability = Mutability::Not;
|
|
Ok(self.tcx.intern_const_alloc(alloc))
|
|
}
|
|
}
|