Auto merge of #94597 - nnethercote:ConstAllocation, r=fee1-dead

Introduce `ConstAllocation`.

Currently some `Allocation`s are interned, some are not, and it's very
hard to tell at a use point which is which.

This commit introduces `ConstAllocation` for the known-interned ones,
which makes the division much clearer. `ConstAllocation::inner()` is
used to get the underlying `Allocation`.

In some places it's natural to use an `Allocation`, in some it's natural
to use a `ConstAllocation`, and in some places there's no clear choice.
I've tried to make things look as nice as possible, while generally
favouring `ConstAllocation`, which is the type that embodies more
information. This does require quite a few calls to `inner()`.

The commit also tweaks how `PartialOrd` works for `Interned`. The
previous code was too clever by half, building on `T: Ord` to make the
code shorter. That caused problems with deriving `PartialOrd` and `Ord`
for `ConstAllocation`, so I changed it to build on `T: PartialOrd`,
which is slightly more verbose but much more standard and avoided the
problems.

r? `@fee1-dead`
This commit is contained in:
bors 2022-03-06 22:37:54 +00:00
commit 8876ca3dd4
30 changed files with 166 additions and 119 deletions

View File

@ -4,7 +4,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_errors::ErrorGuaranteed;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::interpret::{
read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Scalar,
read_target_uint, AllocId, ConstAllocation, ConstValue, ErrorHandled, GlobalAlloc, Scalar,
};
use rustc_middle::ty::ConstKind;
use rustc_span::DUMMY_SP;
@ -202,7 +202,7 @@ pub(crate) fn codegen_const_value<'tcx>(
&mut fx.constants_cx,
fx.module,
alloc_id,
alloc.mutability,
alloc.inner().mutability,
);
let local_data_id =
fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
@ -257,11 +257,15 @@ pub(crate) fn codegen_const_value<'tcx>(
pub(crate) fn pointer_for_allocation<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
alloc: &'tcx Allocation,
alloc: ConstAllocation<'tcx>,
) -> crate::pointer::Pointer {
let alloc_id = fx.tcx.create_memory_alloc(alloc);
let data_id =
data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability);
let data_id = data_id_for_alloc_id(
&mut fx.constants_cx,
&mut *fx.module,
alloc_id,
alloc.inner().mutability,
);
let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
if fx.clif_comments.enabled() {
@ -361,7 +365,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
let data_id = *cx.anon_allocs.entry(alloc_id).or_insert_with(|| {
module
.declare_anonymous_data(
alloc.mutability == rustc_hir::Mutability::Mut,
alloc.inner().mutability == rustc_hir::Mutability::Mut,
false,
)
.unwrap()
@ -386,6 +390,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
}
let mut data_ctx = DataContext::new();
let alloc = alloc.inner();
data_ctx.set_align(alloc.align.bytes());
if let Some(section_name) = section_name {
@ -429,7 +434,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
continue;
}
GlobalAlloc::Memory(target_alloc) => {
data_id_for_alloc_id(cx, module, alloc_id, target_alloc.mutability)
data_id_for_alloc_id(cx, module, alloc_id, target_alloc.inner().mutability)
}
GlobalAlloc::Static(def_id) => {
if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL)

View File

@ -159,7 +159,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let idx_bytes = match idx_const {
ConstValue::ByRef { alloc, offset } => {
let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
alloc.get_bytes(fx, alloc_range(offset, size)).unwrap()
alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap()
}
_ => unreachable!("{:?}", idx_const),
};

View File

@ -13,7 +13,7 @@ use rustc_codegen_ssa::traits::{
use rustc_middle::mir::Mutability;
use rustc_middle::ty::ScalarInt;
use rustc_middle::ty::layout::{TyAndLayout, LayoutOf};
use rustc_middle::mir::interpret::{Allocation, GlobalAlloc, Scalar};
use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
use rustc_span::Symbol;
use rustc_target::abi::{self, HasDataLayout, Pointer, Size};
@ -230,6 +230,7 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
match self.tcx.global_alloc(alloc_id) {
GlobalAlloc::Memory(alloc) => {
let init = const_alloc_to_gcc(self, alloc);
let alloc = alloc.inner();
let value =
match alloc.mutability {
Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
@ -262,21 +263,21 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
}
}
fn const_data_from_alloc(&self, alloc: &Allocation) -> Self::Value {
fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value {
const_alloc_to_gcc(self, alloc)
}
fn from_const_alloc(&self, layout: TyAndLayout<'tcx>, alloc: &Allocation, offset: Size) -> PlaceRef<'tcx, RValue<'gcc>> {
assert_eq!(alloc.align, layout.align.abi);
fn from_const_alloc(&self, layout: TyAndLayout<'tcx>, alloc: ConstAllocation<'tcx>, offset: Size) -> PlaceRef<'tcx, RValue<'gcc>> {
assert_eq!(alloc.inner().align, layout.align.abi);
let ty = self.type_ptr_to(layout.gcc_type(self, true));
let value =
if layout.size == Size::ZERO {
let value = self.const_usize(alloc.align.bytes());
let value = self.const_usize(alloc.inner().align.bytes());
self.context.new_cast(None, value, ty)
}
else {
let init = const_alloc_to_gcc(self, alloc);
let base_addr = self.static_addr_of(init, alloc.align, None);
let base_addr = self.static_addr_of(init, alloc.inner().align, None);
let array = self.const_bitcast(base_addr, self.type_i8p());
let value = self.context.new_array_access(None, array, self.const_usize(offset.bytes())).get_address(None);

View File

@ -7,7 +7,7 @@ use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}
use rustc_middle::mir::mono::MonoItem;
use rustc_middle::ty::{self, Instance, Ty};
use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::mir::interpret::{self, Allocation, ErrorHandled, Scalar as InterpScalar, read_target_uint};
use rustc_middle::mir::interpret::{self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint};
use rustc_span::Span;
use rustc_span::def_id::DefId;
use rustc_target::abi::{self, Align, HasDataLayout, Primitive, Size, WrappingRange};
@ -284,7 +284,8 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
}
}
pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: &Allocation) -> RValue<'gcc> {
pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: ConstAllocation<'tcx>) -> RValue<'gcc> {
let alloc = alloc.inner();
let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1);
let dl = cx.data_layout();
let pointer_size = dl.pointer_size.bytes() as usize;
@ -338,7 +339,7 @@ pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: &Alloca
cx.const_struct(&llvals, true)
}
pub fn codegen_static_initializer<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, def_id: DefId) -> Result<(RValue<'gcc>, &'tcx Allocation), ErrorHandled> {
pub fn codegen_static_initializer<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, def_id: DefId) -> Result<(RValue<'gcc>, ConstAllocation<'tcx>), ErrorHandled> {
let alloc = cx.tcx.eval_static_initializer(def_id)?;
Ok((const_alloc_to_gcc(cx, alloc), alloc))
}

View File

@ -11,7 +11,7 @@ use rustc_ast::Mutability;
use rustc_codegen_ssa::mir::place::PlaceRef;
use rustc_codegen_ssa::traits::*;
use rustc_middle::bug;
use rustc_middle::mir::interpret::{Allocation, GlobalAlloc, Scalar};
use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
use rustc_middle::ty::ScalarInt;
use rustc_span::symbol::Symbol;
@ -249,6 +249,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
let (base_addr, base_addr_space) = match self.tcx.global_alloc(alloc_id) {
GlobalAlloc::Memory(alloc) => {
let init = const_alloc_to_llvm(self, alloc);
let alloc = alloc.inner();
let value = match alloc.mutability {
Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
_ => self.static_addr_of(init, alloc.align, None),
@ -285,24 +286,25 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
}
}
fn const_data_from_alloc(&self, alloc: &Allocation) -> Self::Value {
fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value {
const_alloc_to_llvm(self, alloc)
}
fn from_const_alloc(
&self,
layout: TyAndLayout<'tcx>,
alloc: &Allocation,
alloc: ConstAllocation<'tcx>,
offset: Size,
) -> PlaceRef<'tcx, &'ll Value> {
assert_eq!(alloc.align, layout.align.abi);
let alloc_align = alloc.inner().align;
assert_eq!(alloc_align, layout.align.abi);
let llty = self.type_ptr_to(layout.llvm_type(self));
let llval = if layout.size == Size::ZERO {
let llval = self.const_usize(alloc.align.bytes());
let llval = self.const_usize(alloc_align.bytes());
unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
} else {
let init = const_alloc_to_llvm(self, alloc);
let base_addr = self.static_addr_of(init, alloc.align, None);
let base_addr = self.static_addr_of(init, alloc_align, None);
let llval = unsafe {
llvm::LLVMRustConstInBoundsGEP2(

View File

@ -12,7 +12,7 @@ use rustc_codegen_ssa::traits::*;
use rustc_hir::def_id::DefId;
use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
use rustc_middle::mir::interpret::{
read_target_uint, Allocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer,
read_target_uint, Allocation, ConstAllocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer,
Scalar as InterpScalar,
};
use rustc_middle::mir::mono::MonoItem;
@ -25,7 +25,8 @@ use rustc_target::abi::{
use std::ops::Range;
use tracing::debug;
pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll Value {
pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value {
let alloc = alloc.inner();
let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1);
let dl = cx.data_layout();
let pointer_size = dl.pointer_size.bytes() as usize;
@ -127,7 +128,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) ->
pub fn codegen_static_initializer<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
def_id: DefId,
) -> Result<(&'ll Value, &'tcx Allocation), ErrorHandled> {
) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> {
let alloc = cx.tcx.eval_static_initializer(def_id)?;
Ok((const_alloc_to_llvm(cx, alloc), alloc))
}
@ -370,6 +371,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
// Error has already been reported
return;
};
let alloc = alloc.inner();
let g = self.get_static(def_id);

View File

@ -1,6 +1,6 @@
use super::BackendTypes;
use crate::mir::place::PlaceRef;
use rustc_middle::mir::interpret::{Allocation, Scalar};
use rustc_middle::mir::interpret::{ConstAllocation, Scalar};
use rustc_middle::ty::layout::TyAndLayout;
use rustc_span::Symbol;
use rustc_target::abi::{self, Size};
@ -26,13 +26,13 @@ pub trait ConstMethods<'tcx>: BackendTypes {
fn const_to_opt_uint(&self, v: Self::Value) -> Option<u64>;
fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option<u128>;
fn const_data_from_alloc(&self, alloc: &Allocation) -> Self::Value;
fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value;
fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value;
fn from_const_alloc(
&self,
layout: TyAndLayout<'tcx>,
alloc: &Allocation,
alloc: ConstAllocation<'tcx>,
offset: Size,
) -> PlaceRef<'tcx, Self::Value>;

View File

@ -367,7 +367,7 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
"the raw bytes of the constant ({}",
display_allocation(
*ecx.tcx,
ecx.tcx.global_alloc(alloc_id).unwrap_memory()
ecx.tcx.global_alloc(alloc_id).unwrap_memory().inner()
)
));
},

View File

@ -18,8 +18,8 @@ use rustc_target::abi::{Align, Size};
use rustc_target::spec::abi::Abi;
use crate::interpret::{
self, compile_time_machine, AllocId, Allocation, Frame, ImmTy, InterpCx, InterpResult, OpTy,
PlaceTy, Scalar, StackPopUnwind,
self, compile_time_machine, AllocId, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult,
OpTy, PlaceTy, Scalar, StackPopUnwind,
};
use super::error::*;
@ -475,13 +475,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
fn before_access_global(
memory_extra: &MemoryExtra,
alloc_id: AllocId,
allocation: &Allocation,
alloc: ConstAllocation<'tcx>,
static_def_id: Option<DefId>,
is_write: bool,
) -> InterpResult<'tcx> {
let alloc = alloc.inner();
if is_write {
// Write access. These are never allowed, but we give a targeted error message.
if allocation.mutability == Mutability::Not {
if alloc.mutability == Mutability::Not {
Err(err_ub!(WriteToReadOnly(alloc_id)).into())
} else {
Err(ConstEvalErrKind::ModifiedGlobal.into())
@ -504,7 +505,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
// But make sure we never accept a read from something mutable, that would be
// unsound. The reason is that as the content of this allocation may be different
// now and at run-time, so if we permit reading now we might return the wrong value.
assert_eq!(allocation.mutability, Mutability::Not);
assert_eq!(alloc.mutability, Mutability::Not);
Ok(())
}
}

View File

@ -183,7 +183,7 @@ pub(crate) fn deref_const<'tcx>(
let mplace = ecx.deref_operand(&op).unwrap();
if let Some(alloc_id) = mplace.ptr.provenance {
assert_eq!(
tcx.get_global_alloc(alloc_id).unwrap().unwrap_memory().mutability,
tcx.get_global_alloc(alloc_id).unwrap().unwrap_memory().inner().mutability,
Mutability::Not,
"deref_const cannot be used with mutable allocations as \
that could allow pattern matching to observe mutable statics",

View File

@ -23,7 +23,10 @@ use rustc_middle::ty::{self, layout::TyAndLayout, Ty};
use rustc_ast::Mutability;
use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy, ValueVisitor};
use super::{
AllocId, Allocation, ConstAllocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy,
ValueVisitor,
};
use crate::const_eval;
pub trait CompileTimeMachine<'mir, 'tcx, T> = Machine<
@ -131,8 +134,8 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval:
alloc.mutability = Mutability::Not;
};
// link the alloc id to the actual allocation
let alloc = tcx.intern_const_alloc(alloc);
leftover_allocations.extend(alloc.relocations().iter().map(|&(_, alloc_id)| alloc_id));
let alloc = tcx.intern_const_alloc(alloc);
tcx.set_alloc_id_memory(alloc_id, alloc);
None
}
@ -393,7 +396,7 @@ pub fn intern_const_alloc_recursive<
}
let alloc = tcx.intern_const_alloc(alloc);
tcx.set_alloc_id_memory(alloc_id, alloc);
for &(_, alloc_id) in alloc.relocations().iter() {
for &(_, alloc_id) in alloc.inner().relocations().iter() {
if leftover_allocations.insert(alloc_id) {
todo.push(alloc_id);
}
@ -425,7 +428,7 @@ impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>>
&mut InterpCx<'mir, 'tcx, M>,
&PlaceTy<'tcx, M::PointerTag>,
) -> InterpResult<'tcx, ()>,
) -> InterpResult<'tcx, &'tcx Allocation> {
) -> InterpResult<'tcx, ConstAllocation<'tcx>> {
let dest = self.allocate(layout, MemoryKind::Stack)?;
f(self, &dest.into())?;
let mut alloc = self.memory.alloc_map.remove(&dest.ptr.provenance.unwrap()).unwrap().1;

View File

@ -56,7 +56,7 @@ crate fn eval_nullary_intrinsic<'tcx>(
sym::type_name => {
ensure_monomorphic_enough(tcx, tp_ty)?;
let alloc = type_name::alloc_type_name(tcx, tp_ty);
ConstValue::Slice { data: alloc, start: 0, end: alloc.len() }
ConstValue::Slice { data: alloc, start: 0, end: alloc.inner().len() }
}
sym::needs_drop => {
ensure_monomorphic_enough(tcx, tp_ty)?;

View File

@ -1,6 +1,6 @@
use rustc_hir::def_id::CrateNum;
use rustc_hir::definitions::DisambiguatedDefPathData;
use rustc_middle::mir::interpret::Allocation;
use rustc_middle::mir::interpret::{Allocation, ConstAllocation};
use rustc_middle::ty::{
self,
print::{PrettyPrinter, Print, Printer},
@ -188,7 +188,7 @@ impl Write for AbsolutePathPrinter<'_> {
}
/// Directly returns an `Allocation` containing an absolute path representation of the given type.
crate fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> &'tcx Allocation {
crate fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
let path = AbsolutePathPrinter { tcx, path: String::new() }.print_type(ty).unwrap().path;
let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes());
tcx.intern_const_alloc(alloc)

View File

@ -13,8 +13,9 @@ use rustc_target::abi::Size;
use rustc_target::spec::abi::Abi;
use super::{
AllocId, AllocRange, Allocation, Frame, ImmTy, InterpCx, InterpResult, LocalValue, MemPlace,
Memory, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar, StackPopUnwind,
AllocId, AllocRange, Allocation, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult,
LocalValue, MemPlace, Memory, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar,
StackPopUnwind,
};
/// Data returned by Machine::stack_pop,
@ -252,7 +253,7 @@ pub trait Machine<'mir, 'tcx>: Sized {
fn before_access_global(
_memory_extra: &Self::MemoryExtra,
_alloc_id: AllocId,
_allocation: &Allocation,
_allocation: ConstAllocation<'tcx>,
_static_def_id: Option<DefId>,
_is_write: bool,
) -> InterpResult<'tcx> {

View File

@ -525,12 +525,11 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
}
};
M::before_access_global(&self.extra, id, alloc, def_id, is_write)?;
let alloc = Cow::Borrowed(alloc);
// We got tcx memory. Let the machine initialize its "extra" stuff.
let alloc = M::init_allocation_extra(
self,
id, // always use the ID we got as input, not the "hidden" one.
alloc,
Cow::Borrowed(alloc.inner()),
M::GLOBAL_KIND.map(MemoryKind::Machine),
);
Ok(alloc)
@ -711,6 +710,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
Some(GlobalAlloc::Memory(alloc)) => {
// Need to duplicate the logic here, because the global allocations have
// different associated types than the interpreter-local ones.
let alloc = alloc.inner();
Ok((alloc.size(), alloc.align))
}
Some(GlobalAlloc::Function(_)) => bug!("We already checked function pointers above"),
@ -867,7 +867,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> std::fmt::Debug for DumpAllocs<'a,
&mut *fmt,
self.mem.tcx,
&mut allocs_to_print,
alloc,
alloc.inner(),
)?;
}
Some(GlobalAlloc::Function(func)) => {

View File

@ -62,13 +62,17 @@ impl<'a, T> PartialEq for Interned<'a, T> {
impl<'a, T> Eq for Interned<'a, T> {}
// In practice you can't intern any `T` that doesn't implement `Eq`, because
// that's needed for hashing. Therefore, we won't be interning any `T` that
// implements `PartialOrd` without also implementing `Ord`. So we can have the
// bound `T: Ord` here and avoid duplication with the `Ord` impl below.
impl<'a, T: Ord> PartialOrd for Interned<'a, T> {
impl<'a, T: PartialOrd> PartialOrd for Interned<'a, T> {
fn partial_cmp(&self, other: &Interned<'a, T>) -> Option<Ordering> {
Some(self.cmp(other))
// Pointer equality implies equality, due to the uniqueness constraint,
// but the contents must be compared otherwise.
if ptr::eq(self.0, other.0) {
Some(Ordering::Equal)
} else {
let res = self.0.partial_cmp(&other.0);
debug_assert_ne!(res, Some(Ordering::Equal));
res
}
}
}

View File

@ -2,11 +2,13 @@
use std::borrow::Cow;
use std::convert::{TryFrom, TryInto};
use std::fmt;
use std::iter;
use std::ops::{Deref, Range};
use std::ptr;
use rustc_ast::Mutability;
use rustc_data_structures::intern::Interned;
use rustc_data_structures::sorted_map::SortedMap;
use rustc_span::DUMMY_SP;
use rustc_target::abi::{Align, HasDataLayout, Size};
@ -47,6 +49,34 @@ pub struct Allocation<Tag = AllocId, Extra = ()> {
pub extra: Extra,
}
/// Interned types generally have an `Outer` type and an `Inner` type, where
/// `Outer` is a newtype around `Interned<Inner>`, and all the operations are
/// done on `Outer`, because all occurrences are interned. E.g. `Ty` is an
/// outer type and `TyS` is its inner type.
///
/// Here things are different because only const allocations are interned. This
/// means that both the inner type (`Allocation`) and the outer type
/// (`ConstAllocation`) are used quite a bit.
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable)]
#[cfg_attr(not(bootstrap), rustc_pass_by_value)]
pub struct ConstAllocation<'tcx, Tag = AllocId, Extra = ()>(
pub Interned<'tcx, Allocation<Tag, Extra>>,
);
impl<'tcx> fmt::Debug for ConstAllocation<'tcx> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// This matches how `Allocation` is printed. We print it like this to
// avoid having to update expected output in a lot of tests.
write!(f, "{:?}", self.inner())
}
}
impl<'tcx, Tag, Extra> ConstAllocation<'tcx, Tag, Extra> {
pub fn inner(self) -> &'tcx Allocation<Tag, Extra> {
self.0.0
}
}
/// We have our own error type that does not know about the `AllocId`; that information
/// is added when converting to `InterpError`.
#[derive(Debug)]

View File

@ -126,7 +126,8 @@ pub use self::error::{
pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar, ScalarMaybeUninit};
pub use self::allocation::{
alloc_range, AllocRange, Allocation, InitChunk, InitChunkIter, InitMask, Relocations,
alloc_range, AllocRange, Allocation, ConstAllocation, InitChunk, InitChunkIter, InitMask,
Relocations,
};
pub use self::pointer::{Pointer, PointerArithmetic, Provenance};
@ -343,7 +344,7 @@ impl<'s> AllocDecodingSession<'s> {
let alloc_id = decoder.with_position(pos, |decoder| {
match alloc_kind {
AllocDiscriminant::Alloc => {
let alloc = <&'tcx Allocation as Decodable<_>>::decode(decoder);
let alloc = <ConstAllocation<'tcx> as Decodable<_>>::decode(decoder);
// We already have a reserved `AllocId`.
let alloc_id = alloc_id.unwrap();
trace!("decoded alloc {:?}: {:#?}", alloc_id, alloc);
@ -387,14 +388,14 @@ pub enum GlobalAlloc<'tcx> {
/// This is also used to break the cycle in recursive statics.
Static(DefId),
/// The alloc ID points to memory.
Memory(&'tcx Allocation),
Memory(ConstAllocation<'tcx>),
}
impl<'tcx> GlobalAlloc<'tcx> {
/// Panics if the `GlobalAlloc` does not refer to an `GlobalAlloc::Memory`
#[track_caller]
#[inline]
pub fn unwrap_memory(&self) -> &'tcx Allocation {
pub fn unwrap_memory(&self) -> ConstAllocation<'tcx> {
match *self {
GlobalAlloc::Memory(mem) => mem,
_ => bug!("expected memory, got {:?}", self),
@ -512,7 +513,7 @@ impl<'tcx> TyCtxt<'tcx> {
/// Statics with identical content will still point to the same `Allocation`, i.e.,
/// their data will be deduplicated through `Allocation` interning -- but they
/// are different places in memory and as such need different IDs.
pub fn create_memory_alloc(self, mem: &'tcx Allocation) -> AllocId {
pub fn create_memory_alloc(self, mem: ConstAllocation<'tcx>) -> AllocId {
let id = self.reserve_alloc_id();
self.set_alloc_id_memory(id, mem);
id
@ -543,7 +544,7 @@ impl<'tcx> TyCtxt<'tcx> {
/// Freezes an `AllocId` created with `reserve` by pointing it at an `Allocation`. Trying to
/// call this function twice, even with the same `Allocation` will ICE the compiler.
pub fn set_alloc_id_memory(self, id: AllocId, mem: &'tcx Allocation) {
pub fn set_alloc_id_memory(self, id: AllocId, mem: ConstAllocation<'tcx>) {
if let Some(old) = self.alloc_map.lock().alloc_map.insert(id, GlobalAlloc::Memory(mem)) {
bug!("tried to set allocation ID {}, but it was already existing as {:#?}", id, old);
}
@ -551,7 +552,7 @@ impl<'tcx> TyCtxt<'tcx> {
/// Freezes an `AllocId` created with `reserve` by pointing it at an `Allocation`. May be called
/// twice for the same `(AllocId, Allocation)` pair.
fn set_alloc_id_same_memory(self, id: AllocId, mem: &'tcx Allocation) {
fn set_alloc_id_same_memory(self, id: AllocId, mem: ConstAllocation<'tcx>) {
self.alloc_map.lock().alloc_map.insert_same(id, GlobalAlloc::Memory(mem));
}
}

View File

@ -79,7 +79,7 @@ impl<'tcx> TyCtxt<'tcx> {
pub fn eval_static_initializer(
self,
def_id: DefId,
) -> Result<&'tcx mir::Allocation, ErrorHandled> {
) -> Result<mir::ConstAllocation<'tcx>, ErrorHandled> {
trace!("eval_static_initializer: Need to compute {:?}", def_id);
assert!(self.is_static(def_id));
let instance = ty::Instance::mono(self, def_id);
@ -92,7 +92,7 @@ impl<'tcx> TyCtxt<'tcx> {
self,
gid: GlobalId<'tcx>,
param_env: ty::ParamEnv<'tcx>,
) -> Result<&'tcx mir::Allocation, ErrorHandled> {
) -> Result<mir::ConstAllocation<'tcx>, ErrorHandled> {
let param_env = param_env.with_const();
trace!("eval_to_allocation: Need to compute {:?}", gid);
let raw_const = self.eval_to_allocation_raw(param_env.and(gid))?;

View File

@ -11,7 +11,7 @@ use rustc_target::abi::{HasDataLayout, Size};
use crate::ty::{Lift, ParamEnv, ScalarInt, Ty, TyCtxt};
use super::{
AllocId, AllocRange, Allocation, InterpResult, Pointer, PointerArithmetic, Provenance,
AllocId, AllocRange, ConstAllocation, InterpResult, Pointer, PointerArithmetic, Provenance,
};
/// Represents the result of const evaluation via the `eval_to_allocation` query.
@ -34,13 +34,13 @@ pub enum ConstValue<'tcx> {
Scalar(Scalar),
/// Used only for `&[u8]` and `&str`
Slice { data: &'tcx Allocation, start: usize, end: usize },
Slice { data: ConstAllocation<'tcx>, start: usize, end: usize },
/// A value not represented/representable by `Scalar` or `Slice`
ByRef {
/// The backing memory of the value, may contain more memory than needed for just the value
/// in order to share `Allocation`s between values
alloc: &'tcx Allocation,
/// in order to share `ConstAllocation`s between values
alloc: ConstAllocation<'tcx>,
/// Offset into `alloc`
offset: Size,
},
@ -603,11 +603,12 @@ impl<'tcx, Tag: Provenance> ScalarMaybeUninit<Tag> {
pub fn get_slice_bytes<'tcx>(cx: &impl HasDataLayout, val: ConstValue<'tcx>) -> &'tcx [u8] {
if let ConstValue::Slice { data, start, end } = val {
let len = end - start;
data.get_bytes(
cx,
AllocRange { start: Size::from_bytes(start), size: Size::from_bytes(len) },
)
.unwrap_or_else(|err| bug!("const slice is invalid: {:?}", err))
data.inner()
.get_bytes(
cx,
AllocRange { start: Size::from_bytes(start), size: Size::from_bytes(len) },
)
.unwrap_or_else(|err| bug!("const slice is invalid: {:?}", err))
} else {
bug!("expected const slice, but found another const value");
}

View File

@ -3,7 +3,7 @@
//! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/mir/index.html
use crate::mir::coverage::{CodeRegion, CoverageKind};
use crate::mir::interpret::{Allocation, ConstValue, GlobalAlloc, Scalar};
use crate::mir::interpret::{ConstAllocation, ConstValue, GlobalAlloc, Scalar};
use crate::mir::visit::MirVisitable;
use crate::ty::adjustment::PointerCast;
use crate::ty::codec::{TyDecoder, TyEncoder};

View File

@ -12,7 +12,8 @@ use rustc_data_structures::fx::FxHashMap;
use rustc_hir::def_id::DefId;
use rustc_index::vec::Idx;
use rustc_middle::mir::interpret::{
read_target_uint, AllocId, Allocation, ConstValue, GlobalAlloc, Pointer, Provenance,
read_target_uint, AllocId, Allocation, ConstAllocation, ConstValue, GlobalAlloc, Pointer,
Provenance,
};
use rustc_middle::mir::visit::Visitor;
use rustc_middle::mir::MirSource;
@ -652,8 +653,10 @@ pub fn write_allocations<'tcx>(
body: &Body<'_>,
w: &mut dyn Write,
) -> io::Result<()> {
fn alloc_ids_from_alloc(alloc: &Allocation) -> impl DoubleEndedIterator<Item = AllocId> + '_ {
alloc.relocations().values().map(|id| *id)
fn alloc_ids_from_alloc(
alloc: ConstAllocation<'_>,
) -> impl DoubleEndedIterator<Item = AllocId> + '_ {
alloc.inner().relocations().values().map(|id| *id)
}
fn alloc_ids_from_const(val: ConstValue<'_>) -> impl Iterator<Item = AllocId> + '_ {
match val {
@ -686,14 +689,14 @@ pub fn write_allocations<'tcx>(
let mut todo: Vec<_> = seen.iter().copied().collect();
while let Some(id) = todo.pop() {
let mut write_allocation_track_relocs =
|w: &mut dyn Write, alloc: &Allocation| -> io::Result<()> {
|w: &mut dyn Write, alloc: ConstAllocation<'tcx>| -> io::Result<()> {
// `.rev()` because we are popping them from the back of the `todo` vector.
for id in alloc_ids_from_alloc(alloc).rev() {
if seen.insert(id) {
todo.push(id);
}
}
write!(w, "{}", display_allocation(tcx, alloc))
write!(w, "{}", display_allocation(tcx, alloc.inner()))
};
write!(w, "\n{}", id)?;
match tcx.get_global_alloc(id) {

View File

@ -10,7 +10,7 @@ use crate::arena::ArenaAllocatable;
use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
use crate::mir::{
self,
interpret::{AllocId, Allocation},
interpret::{AllocId, ConstAllocation},
};
use crate::thir;
use crate::traits;
@ -150,6 +150,12 @@ impl<'tcx, E: TyEncoder<'tcx>> Encodable<E> for ty::Const<'tcx> {
}
}
impl<'tcx, E: TyEncoder<'tcx>> Encodable<E> for ConstAllocation<'tcx> {
fn encode(&self, e: &mut E) -> Result<(), E::Error> {
self.inner().encode(e)
}
}
impl<'tcx, E: TyEncoder<'tcx>> Encodable<E> for AllocId {
fn encode(&self, e: &mut E) -> Result<(), E::Error> {
e.encode_alloc_id(self)
@ -355,8 +361,8 @@ impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for [ty::ValTree<'tcx>] {
}
}
impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for Allocation {
fn decode(decoder: &mut D) -> &'tcx Self {
impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for ConstAllocation<'tcx> {
fn decode(decoder: &mut D) -> Self {
decoder.tcx().intern_const_alloc(Decodable::decode(decoder))
}
}
@ -399,7 +405,6 @@ impl_decodable_via_ref! {
&'tcx ty::List<Ty<'tcx>>,
&'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
&'tcx traits::ImplSource<'tcx, ()>,
&'tcx Allocation,
&'tcx mir::Body<'tcx>,
&'tcx mir::UnsafetyCheckResult,
&'tcx mir::BorrowCheckResult<'tcx>,

View File

@ -7,7 +7,7 @@ use crate::infer::canonical::{Canonical, CanonicalVarInfo, CanonicalVarInfos};
use crate::lint::{struct_lint_level, LintDiagnosticBuilder, LintLevelSource};
use crate::middle::resolve_lifetime::{self, LifetimeScopeForPath};
use crate::middle::stability;
use crate::mir::interpret::{self, Allocation, ConstValue, Scalar};
use crate::mir::interpret::{self, Allocation, ConstAllocation, ConstValue, Scalar};
use crate::mir::{
Body, BorrowCheckResult, Field, Local, Place, PlaceElem, ProjectionKind, Promoted,
};
@ -1653,22 +1653,6 @@ pub trait Lift<'tcx>: fmt::Debug {
fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted>;
}
// Deprecated: we are in the process of converting all uses to `nop_lift`.
macro_rules! nop_lift_old {
($set:ident; $ty:ty => $lifted:ty) => {
impl<'a, 'tcx> Lift<'tcx> for $ty {
type Lifted = $lifted;
fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
if tcx.interners.$set.contains_pointer_to(&InternedInSet(self)) {
Some(unsafe { mem::transmute(self) })
} else {
None
}
}
}
};
}
macro_rules! nop_lift {
($set:ident; $ty:ty => $lifted:ty) => {
impl<'a, 'tcx> Lift<'tcx> for $ty {
@ -1726,7 +1710,7 @@ macro_rules! nop_list_lift {
nop_lift! {type_; Ty<'a> => Ty<'tcx>}
nop_lift! {region; Region<'a> => Region<'tcx>}
nop_lift! {const_; Const<'a> => Const<'tcx>}
nop_lift_old! {const_allocation; &'a Allocation => &'tcx Allocation}
nop_lift! {const_allocation; ConstAllocation<'a> => ConstAllocation<'tcx>}
nop_lift! {predicate; Predicate<'a> => Predicate<'tcx>}
nop_list_lift! {poly_existential_predicates; ty::Binder<'a, ExistentialPredicate<'a>> => ty::Binder<'tcx, ExistentialPredicate<'tcx>>}
@ -2161,6 +2145,7 @@ macro_rules! direct_interners {
direct_interners! {
region: mk_region(RegionKind): Region -> Region<'tcx>,
const_: mk_const(ConstS<'tcx>): Const -> Const<'tcx>,
const_allocation: intern_const_alloc(Allocation): ConstAllocation -> ConstAllocation<'tcx>,
}
macro_rules! direct_interners_old {
@ -2201,7 +2186,6 @@ macro_rules! direct_interners_old {
// FIXME: eventually these should all be converted to `direct_interners`.
direct_interners_old! {
const_allocation: intern_const_alloc(Allocation),
layout: intern_layout(Layout),
adt_def: intern_adt_def(AdtDef),
}

View File

@ -1267,7 +1267,7 @@ pub trait PrettyPrinter<'tcx>:
Some(GlobalAlloc::Memory(alloc)) => {
let len = int.assert_bits(self.tcx().data_layout.pointer_size);
let range = AllocRange { start: offset, size: Size::from_bytes(len) };
if let Ok(byte_str) = alloc.get_bytes(&self.tcx(), range) {
if let Ok(byte_str) = alloc.inner().get_bytes(&self.tcx(), range) {
p!(pretty_print_byte_str(byte_str))
} else {
p!("<too short allocation>")
@ -1424,7 +1424,8 @@ pub trait PrettyPrinter<'tcx>:
// The `inspect` here is okay since we checked the bounds, and there are
// no relocations (we have an active slice reference here). We don't use
// this result to affect interpreter execution.
let byte_str = data.inspect_with_uninit_and_ptr_outside_interpreter(start..end);
let byte_str =
data.inner().inspect_with_uninit_and_ptr_outside_interpreter(start..end);
self.pretty_print_byte_str(byte_str)
}
(
@ -1434,7 +1435,8 @@ pub trait PrettyPrinter<'tcx>:
// The `inspect` here is okay since we checked the bounds, and there are no
// relocations (we have an active `str` reference here). We don't use this
// result to affect interpreter execution.
let slice = data.inspect_with_uninit_and_ptr_outside_interpreter(start..end);
let slice =
data.inner().inspect_with_uninit_and_ptr_outside_interpreter(start..end);
p!(write("{:?}", String::from_utf8_lossy(slice)));
Ok(self)
}
@ -1443,7 +1445,7 @@ pub trait PrettyPrinter<'tcx>:
// cast is ok because we already checked for pointer size (32 or 64 bit) above
let range = AllocRange { start: offset, size: Size::from_bytes(n) };
let byte_str = alloc.get_bytes(&self.tcx(), range).unwrap();
let byte_str = alloc.inner().get_bytes(&self.tcx(), range).unwrap();
p!("*");
p!(pretty_print_byte_str(byte_str));
Ok(self)

View File

@ -31,8 +31,8 @@ use rustc_trait_selection::traits;
use crate::MirPass;
use rustc_const_eval::const_eval::ConstEvalErr;
use rustc_const_eval::interpret::{
self, compile_time_machine, AllocId, Allocation, ConstValue, CtfeValidationMode, Frame, ImmTy,
Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemPlace, MemoryKind, OpTy,
self, compile_time_machine, AllocId, ConstAllocation, ConstValue, CtfeValidationMode, Frame,
ImmTy, Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemPlace, MemoryKind, OpTy,
Operand as InterpOperand, PlaceTy, Scalar, ScalarMaybeUninit, StackPopCleanup, StackPopUnwind,
};
@ -274,7 +274,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
fn before_access_global(
_memory_extra: &(),
_alloc_id: AllocId,
allocation: &Allocation<Self::PointerTag, Self::AllocExtra>,
alloc: ConstAllocation<'tcx, Self::PointerTag, Self::AllocExtra>,
_static_def_id: Option<DefId>,
is_write: bool,
) -> InterpResult<'tcx> {
@ -283,7 +283,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
}
// If the static allocation is mutable, then we can't const prop it as its content
// might be different at runtime.
if allocation.mutability == Mutability::Mut {
if alloc.inner().mutability == Mutability::Mut {
throw_machine_stop_str!("can't access mutable globals in ConstProp");
}

View File

@ -401,7 +401,7 @@ fn collect_items_rec<'tcx>(
recursion_depth_reset = None;
if let Ok(alloc) = tcx.eval_static_initializer(def_id) {
for &id in alloc.relocations().values() {
for &id in alloc.inner().relocations().values() {
collect_miri(tcx, id, &mut neighbors);
}
}
@ -1370,7 +1370,7 @@ fn collect_miri<'tcx>(
}
GlobalAlloc::Memory(alloc) => {
trace!("collecting {:?} with {:#?}", alloc_id, alloc);
for &inner in alloc.relocations().values() {
for &inner in alloc.inner().relocations().values() {
rustc_data_structures::stack::ensure_sufficient_stack(|| {
collect_miri(tcx, inner, output);
});
@ -1405,7 +1405,7 @@ fn collect_const_value<'tcx>(
match value {
ConstValue::Scalar(Scalar::Ptr(ptr, _size)) => collect_miri(tcx, ptr.provenance, output),
ConstValue::Slice { data: alloc, start: _, end: _ } | ConstValue::ByRef { alloc, .. } => {
for &id in alloc.relocations().values() {
for &id in alloc.inner().relocations().values() {
collect_miri(tcx, id, output);
}
}

View File

@ -633,8 +633,9 @@ impl<'tcx> Printer<'tcx> for &mut SymbolMangler<'tcx> {
// The `inspect` here is okay since we checked the bounds, and there are no
// relocations (we have an active `str` reference here). We don't use this
// result to affect interpreter execution.
let slice =
data.inspect_with_uninit_and_ptr_outside_interpreter(start..end);
let slice = data
.inner()
.inspect_with_uninit_and_ptr_outside_interpreter(start..end);
let s = std::str::from_utf8(slice).expect("non utf8 str from miri");
self.push("e");

View File

@ -553,7 +553,7 @@ fn maybe_check_static_with_link_section(tcx: TyCtxt<'_>, id: LocalDefId, span: S
// the consumer's responsibility to ensure all bytes that have been read
// have defined values.
if let Ok(alloc) = tcx.eval_static_initializer(id.to_def_id()) {
if alloc.relocations().len() != 0 {
if alloc.inner().relocations().len() != 0 {
let msg = "statics with a custom `#[link_section]` must be a \
simple list of bytes on the wasm target with no \
extra levels of indirection such as references";

View File

@ -593,7 +593,7 @@ pub fn miri_to_const(result: ty::Const<'_>) -> Option<Constant> {
ty::ConstKind::Value(ConstValue::Slice { data, start, end }) => match result.ty().kind() {
ty::Ref(_, tam, _) => match tam.kind() {
ty::Str => String::from_utf8(
data.inspect_with_uninit_and_ptr_outside_interpreter(start..end)
data.inner().inspect_with_uninit_and_ptr_outside_interpreter(start..end)
.to_owned(),
)
.ok()
@ -605,7 +605,7 @@ pub fn miri_to_const(result: ty::Const<'_>) -> Option<Constant> {
ty::ConstKind::Value(ConstValue::ByRef { alloc, offset: _ }) => match result.ty().kind() {
ty::Array(sub_type, len) => match sub_type.kind() {
ty::Float(FloatTy::F32) => match miri_to_const(*len) {
Some(Constant::Int(len)) => alloc
Some(Constant::Int(len)) => alloc.inner()
.inspect_with_uninit_and_ptr_outside_interpreter(0..(4 * len as usize))
.to_owned()
.chunks(4)
@ -619,7 +619,7 @@ pub fn miri_to_const(result: ty::Const<'_>) -> Option<Constant> {
_ => None,
},
ty::Float(FloatTy::F64) => match miri_to_const(*len) {
Some(Constant::Int(len)) => alloc
Some(Constant::Int(len)) => alloc.inner()
.inspect_with_uninit_and_ptr_outside_interpreter(0..(8 * len as usize))
.to_owned()
.chunks(8)