mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

make const align_offset useful

Browse Source
This commit is contained in:
Lukas Markeffsky 2022-10-07 22:23:34 +02:00
parent f770fecfe1
commit 211743b2c8
4 changed files with 152 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
compiler/rustc_const_eval/src/const_eval/machine.rs
View File

@ -1,8 +1,11 @@
use rustc_hir::def::DefKind;
use rustc_middle::mir;
use rustc_middle::mir::interpret::PointerArithmetic;
use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::{self, Ty, TyCtxt};
use std::borrow::Borrow;
use std::hash::Hash;
use std::ops::ControlFlow;
use rustc_data_structures::fx::FxIndexMap;
use rustc_data_structures::fx::IndexEntry;
@ -17,8 +20,8 @@ use rustc_target::abi::{Align, Size};
use rustc_target::spec::abi::Abi as CallAbi;
use crate::interpret::{
self, compile_time_machine, AllocId, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult,
OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind,
self, compile_time_machine, AllocId, ConstAllocation, FnVal, Frame, ImmTy, InterpCx,
InterpResult, OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind,
};
use super::error::*;
@ -145,15 +148,19 @@ impl interpret::MayLeak for ! {
}
impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
/// "Intercept" a function call to a panic-related function
/// because we have something special to do for it.
/// If this returns successfully (`Ok`), the function should just be evaluated normally.
/// "Intercept" a function call, because we have something special to do for it.
/// All `#[rustc_do_not_const_check]` functions should be hooked here.
/// If this returns `Some` function, which may be `instance` or a different function with
/// compatible arguments, then evaluation should continue with that function.
/// If this returns `None`, the function call has been handled and the function has returned.
fn hook_special_const_fn(
&mut self,
instance: ty::Instance<'tcx>,
_abi: CallAbi,
args: &[OpTy<'tcx>],
dest: &PlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
// All `#[rustc_do_not_const_check]` functions should be hooked here.
let def_id = instance.def_id();
if Some(def_id) == self.tcx.lang_items().panic_display()
@ -173,20 +180,91 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into());
} else if Some(def_id) == self.tcx.lang_items().panic_fmt() {
// For panic_fmt, call const_panic_fmt instead.
if let Some(const_panic_fmt) = self.tcx.lang_items().const_panic_fmt() {
return Ok(Some(
ty::Instance::resolve(
*self.tcx,
ty::ParamEnv::reveal_all(),
const_panic_fmt,
self.tcx.intern_substs(&[]),
)
.unwrap()
.unwrap(),
));
let Some(const_def_id) = self.tcx.lang_items().const_panic_fmt() else {
bug!("`const_panic_fmt` must be defined to call `panic_fmt` in const eval")
};
let new_instance = ty::Instance::resolve(
*self.tcx,
ty::ParamEnv::reveal_all(),
const_def_id,
instance.substs,
)
.unwrap()
.unwrap();
return Ok(Some(new_instance));
} else if Some(def_id) == self.tcx.lang_items().align_offset_fn() {
// For align_offset, we replace the function call if the pointer has no address.
match self.align_offset(instance, args, dest, ret)? {
ControlFlow::Continue(()) => return Ok(Some(instance)),
ControlFlow::Break(()) => return Ok(None),
}
}
Ok(Some(instance))
}
/// `align_offset(ptr, target_align)` needs special handling in const eval, because the pointer
/// may not have an address.
///
/// If `ptr` does have a known address, then we return `CONTINUE` and the function call should
/// proceed as normal.
///
/// If `ptr` doesn't have an address, but its underlying allocation's alignment is at most
/// `target_align`, then we call the function again with an dummy address relative to the
/// allocation.
///
/// If `ptr` doesn't have an address and `target_align` is stricter than the underlying
/// allocation's alignment, then we return `usize::MAX` immediately.
fn align_offset(
&mut self,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx>],
dest: &PlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
) -> InterpResult<'tcx, ControlFlow<()>> {
assert_eq!(args.len(), 2);
let ptr = self.read_pointer(&args[0])?;
let target_align = self.read_scalar(&args[1])?.to_machine_usize(self)?;
if !target_align.is_power_of_two() {
throw_ub_format!("`align_offset` called with non-power-of-two align: {}", target_align);
}
match self.ptr_try_get_alloc_id(ptr) {
Ok((alloc_id, offset, _extra)) => {
let (_size, alloc_align, _kind) = self.get_alloc_info(alloc_id);
if target_align <= alloc_align.bytes() {
// Extract the address relative to the allocation base that is definitely
// sufficiently aligned and call `align_offset` again.
let addr = ImmTy::from_uint(offset.bytes(), args[0].layout).into();
let align = ImmTy::from_uint(target_align, args[1].layout).into();
let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?;
self.eval_fn_call(
FnVal::Instance(instance),
(CallAbi::Rust, fn_abi),
&[addr, align],
false,
dest,
ret,
StackPopUnwind::NotAllowed,
)?;
Ok(ControlFlow::BREAK)
} else {
// Not alignable in const, return `usize::MAX`.
let usize_max = Scalar::from_machine_usize(self.machine_usize_max(), self);
self.write_scalar(usize_max, dest)?;
self.return_to_block(ret)?;
Ok(ControlFlow::BREAK)
}
}
Err(_addr) => {
// The pointer has an address, continue with function call.
Ok(ControlFlow::CONTINUE)
}
}
Ok(None)
}
/// See documentation on the `ptr_guaranteed_cmp` intrinsic.
@ -269,8 +347,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
instance: ty::Instance<'tcx>,
_abi: CallAbi,
args: &[OpTy<'tcx>],
_dest: &PlaceTy<'tcx>,
_ret: Option<mir::BasicBlock>,
dest: &PlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
_unwind: StackPopUnwind, // unwinding is not supported in consts
) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> {
debug!("find_mir_or_eval_fn: {:?}", instance);
@ -289,7 +367,11 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
}
}
if let Some(new_instance) = ecx.hook_special_const_fn(instance, args)? {
let Some(new_instance) = ecx.hook_special_const_fn(instance, _abi, args, dest, ret)? else {
return Ok(None);
};
if new_instance != instance {
// We call another const fn instead.
// However, we return the *original* instance to make backtraces work out
// (and we hope this does not confuse the FnAbi checks too much).
@ -298,13 +380,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
new_instance,
_abi,
args,
_dest,
_ret,
dest,
ret,
_unwind,
)?
.map(|(body, _instance)| (body, instance)));
}
}
// This is a const fn. Call it.
Ok(Some((ecx.load_mir(instance.def, None)?, instance)))
}

15
library/core/src/ptr/const_ptr.rs
View File

@ -1322,6 +1322,21 @@ impl<T: ?Sized> *const T {
/// ```
#[stable(feature = "align_offset", since = "1.36.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
#[cfg(not(bootstrap))]
pub const fn align_offset(self, align: usize) -> usize
where
T: Sized,
{
assert!(align.is_power_of_two(), "align_offset: align is not a power-of-two");
// SAFETY: `align` has been checked to be a power of 2 above
unsafe { align_offset(self, align) }
}
#[stable(feature = "align_offset", since = "1.36.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
#[allow(missing_docs)]
#[cfg(bootstrap)]
pub const fn align_offset(self, align: usize) -> usize
where
T: Sized,

21
library/core/src/ptr/mod.rs
View File

@ -1574,10 +1574,14 @@ pub unsafe fn write_volatile<T>(dst: *mut T, src: T) {
/// Align pointer `p`.
///
/// Calculate offset (in terms of elements of `stride` stride) that has to be applied
/// Calculate offset (in terms of elements of `size_of::<T>()` stride) that has to be applied
/// to pointer `p` so that pointer `p` would get aligned to `a`.
///
/// Note: This implementation has been carefully tailored to not panic. It is UB for this to panic.
/// # Safety
/// `a` must be a power of two.
///
/// # Notes
/// This implementation has been carefully tailored to not panic. It is UB for this to panic.
/// The only real change that can be made here is change of `INV_TABLE_MOD_16` and associated
/// constants.
///
@ -1586,8 +1590,10 @@ pub unsafe fn write_volatile<T>(dst: *mut T, src: T) {
/// than trying to adapt this to accommodate that change.
///
/// Any questions go to @nagisa.
// #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap
// compiler will always cause an error.
#[lang = "align_offset"]
pub(crate) unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
pub(crate) const unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
// FIXME(#75598): Direct use of these intrinsics improves codegen significantly at opt-level <=
// 1, where the method versions of these operations are not inlined.
use intrinsics::{
@ -1604,7 +1610,7 @@ pub(crate) unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
///
/// Implementation of this function shall not panic. Ever.
#[inline]
unsafe fn mod_inv(x: usize, m: usize) -> usize {
const unsafe fn mod_inv(x: usize, m: usize) -> usize {
/// Multiplicative modular inverse table modulo 2⁴ = 16.
///
/// Note, that this table does not contain values where inverse does not exist (i.e., for
@ -1646,8 +1652,13 @@ pub(crate) unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
inverse & m_minus_one
}
let addr = p.addr();
let stride = mem::size_of::<T>();
// SAFETY: At runtime transmuting a pointer to `usize` is always safe, because they have the
// same layout. During const eval we hook this function to ensure that the pointer always has
// an address (only the standard library can do this).
let addr = unsafe { mem::transmute(p) };
// SAFETY: `a` is a power-of-two, therefore non-zero.
let a_minus_one = unsafe { unchecked_sub(a, 1) };

15
library/core/src/ptr/mut_ptr.rs
View File

@ -1590,6 +1590,21 @@ impl<T: ?Sized> *mut T {
/// ```
#[stable(feature = "align_offset", since = "1.36.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
#[cfg(not(bootstrap))]
pub const fn align_offset(self, align: usize) -> usize
where
T: Sized,
{
assert!(align.is_power_of_two(), "align_offset: align is not a power-of-two");
// SAFETY: `align` has been checked to be a power of 2 above
unsafe { align_offset(self, align) }
}
#[stable(feature = "align_offset", since = "1.36.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
#[allow(missing_docs)]
#[cfg(bootstrap)]
pub const fn align_offset(self, align: usize) -> usize
where
T: Sized,