mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #2382 - RalfJung:atomic, r=RalfJung

Browse Source 
add support for new RMW orders

Implements support for https://github.com/rust-lang/rust/pull/98383.

Also restructure intrinsic shims a bit so we have atomic and SIMD intrinsics in their own file, and refactor atomic intrinsic name parsing.
This commit is contained in:
bors 2022-07-18 12:56:44 +00:00
commit b9c677112a
13 changed files with 1408 additions and 1470 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
rust-version
View File

@ -1 +1 @@
db41351753df840773ca628d8daa040e95d00eef
880416180b0a9ee1141c07d4d17667edb77daebd

9
src/shims/foreign_items.rs
View File

@ -297,8 +297,8 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
Some(p) => p,
};
// Second: functions that return.
match this.emulate_foreign_item_by_name(link_name, abi, args, dest, ret)? {
// Second: functions that return immediately.
match this.emulate_foreign_item_by_name(link_name, abi, args, dest)? {
EmulateByNameResult::NeedsJumping => {
trace!("{:?}", this.dump_place(**dest));
this.go_to_block(ret);
@ -355,7 +355,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();
@ -702,8 +701,8 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
// Platform-specific shims
_ => match this.tcx.sess.target.os.as_ref() {
target if target_os_is_unix(target) => return shims::unix::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest, ret),
"windows" => return shims::windows::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest, ret),
target if target_os_is_unix(target) => return shims::unix::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest),
"windows" => return shims::windows::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest),
target => throw_unsup_format!("the target `{}` is not supported", target),
}
};

1423
src/shims/intrinsics.rs
View File

File diff suppressed because it is too large Load Diff

340
src/shims/intrinsics/atomic.rs Normal file
View File

@ -0,0 +1,340 @@
use rustc_middle::{mir, mir::BinOp, ty};
use rustc_target::abi::Align;
use crate::*;
use helpers::check_arg_count;
pub enum AtomicOp {
/// The `bool` indicates whether the result of the operation should be negated
/// (must be a boolean-typed operation).
MirOp(mir::BinOp, bool),
Max,
Min,
}
impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
/// Calls the atomic intrinsic `intrinsic`; the `atomic_` prefix has already been removed.
fn emulate_atomic_intrinsic(
&mut self,
intrinsic_name: &str,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
let intrinsic_structure: Vec<_> = intrinsic_name.split('_').collect();
fn read_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicReadOrd> {
Ok(match ord {
"seqcst" => AtomicReadOrd::SeqCst,
"acquire" => AtomicReadOrd::Acquire,
"relaxed" => AtomicReadOrd::Relaxed,
_ => throw_unsup_format!("unsupported read ordering `{ord}`"),
})
}
fn write_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicWriteOrd> {
Ok(match ord {
"seqcst" => AtomicWriteOrd::SeqCst,
"release" => AtomicWriteOrd::Release,
"relaxed" => AtomicWriteOrd::Relaxed,
_ => throw_unsup_format!("unsupported write ordering `{ord}`"),
})
}
fn rw_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicRwOrd> {
Ok(match ord {
"seqcst" => AtomicRwOrd::SeqCst,
"acqrel" => AtomicRwOrd::AcqRel,
"acquire" => AtomicRwOrd::Acquire,
"release" => AtomicRwOrd::Release,
"relaxed" => AtomicRwOrd::Relaxed,
_ => throw_unsup_format!("unsupported read-write ordering `{ord}`"),
})
}
fn fence_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicFenceOrd> {
Ok(match ord {
"seqcst" => AtomicFenceOrd::SeqCst,
"acqrel" => AtomicFenceOrd::AcqRel,
"acquire" => AtomicFenceOrd::Acquire,
"release" => AtomicFenceOrd::Release,
_ => throw_unsup_format!("unsupported fence ordering `{ord}`"),
})
}
match &*intrinsic_structure {
["load", ord] => this.atomic_load(args, dest, read_ord(ord)?)?,
["store", ord] => this.atomic_store(args, write_ord(ord)?)?,
["fence", ord] => this.atomic_fence(args, fence_ord(ord)?)?,
["singlethreadfence", ord] => this.compiler_fence(args, fence_ord(ord)?)?,
["xchg", ord] => this.atomic_exchange(args, dest, rw_ord(ord)?)?,
["cxchg", ord1, ord2] =>
this.atomic_compare_exchange(args, dest, rw_ord(ord1)?, read_ord(ord2)?)?,
["cxchgweak", ord1, ord2] =>
this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1)?, read_ord(ord2)?)?,
["or", ord] =>
this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord)?)?,
["xor", ord] =>
this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord)?)?,
["and", ord] =>
this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord)?)?,
["nand", ord] =>
this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord)?)?,
["xadd", ord] =>
this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord)?)?,
["xsub", ord] =>
this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord)?)?,
["min", ord] => {
// Later we will use the type to indicate signed vs unsigned,
// so make sure it matches the intrinsic name.
assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
this.atomic_op(args, dest, AtomicOp::Min, rw_ord(ord)?)?;
}
["umin", ord] => {
// Later we will use the type to indicate signed vs unsigned,
// so make sure it matches the intrinsic name.
assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));
this.atomic_op(args, dest, AtomicOp::Min, rw_ord(ord)?)?;
}
["max", ord] => {
// Later we will use the type to indicate signed vs unsigned,
// so make sure it matches the intrinsic name.
assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
this.atomic_op(args, dest, AtomicOp::Max, rw_ord(ord)?)?;
}
["umax", ord] => {
// Later we will use the type to indicate signed vs unsigned,
// so make sure it matches the intrinsic name.
assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));
this.atomic_op(args, dest, AtomicOp::Max, rw_ord(ord)?)?;
}
_ => throw_unsup_format!("unimplemented intrinsic: `atomic_{intrinsic_name}`"),
}
Ok(())
}
fn atomic_load(
&mut self,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
atomic: AtomicReadOrd,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
let [place] = check_arg_count(args)?;
let place = this.deref_operand(place)?;
// make sure it fits into a scalar; otherwise it cannot be atomic
let val = this.read_scalar_atomic(&place, atomic)?;
// Check alignment requirements. Atomics must always be aligned to their size,
// even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
// be 8-aligned).
let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
this.check_ptr_access_align(
place.ptr,
place.layout.size,
align,
CheckInAllocMsg::MemoryAccessTest,
)?;
// Perform regular access.
this.write_scalar(val, dest)?;
Ok(())
}
fn atomic_store(
&mut self,
args: &[OpTy<'tcx, Tag>],
atomic: AtomicWriteOrd,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
let [place, val] = check_arg_count(args)?;
let place = this.deref_operand(place)?;
let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
// Check alignment requirements. Atomics must always be aligned to their size,
// even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
// be 8-aligned).
let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
this.check_ptr_access_align(
place.ptr,
place.layout.size,
align,
CheckInAllocMsg::MemoryAccessTest,
)?;
// Perform atomic store
this.write_scalar_atomic(val, &place, atomic)?;
Ok(())
}
fn compiler_fence(
&mut self,
args: &[OpTy<'tcx, Tag>],
atomic: AtomicFenceOrd,
) -> InterpResult<'tcx> {
let [] = check_arg_count(args)?;
let _ = atomic;
//FIXME: compiler fences are currently ignored
Ok(())
}
fn atomic_fence(
&mut self,
args: &[OpTy<'tcx, Tag>],
atomic: AtomicFenceOrd,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
let [] = check_arg_count(args)?;
this.validate_atomic_fence(atomic)?;
Ok(())
}
fn atomic_op(
&mut self,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
atomic_op: AtomicOp,
atomic: AtomicRwOrd,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
let [place, rhs] = check_arg_count(args)?;
let place = this.deref_operand(place)?;
let rhs = this.read_immediate(rhs)?;
if !place.layout.ty.is_integral() && !place.layout.ty.is_unsafe_ptr() {
span_bug!(
this.cur_span(),
"atomic arithmetic operations only work on integer and raw pointer types",
);
}
if rhs.layout.ty != place.layout.ty {
span_bug!(this.cur_span(), "atomic arithmetic operation type mismatch");
}
// Check alignment requirements. Atomics must always be aligned to their size,
// even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
// be 8-aligned).
let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
this.check_ptr_access_align(
place.ptr,
place.layout.size,
align,
CheckInAllocMsg::MemoryAccessTest,
)?;
match atomic_op {
AtomicOp::Min => {
let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
this.write_immediate(*old, dest)?; // old value is returned
Ok(())
}
AtomicOp::Max => {
let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
this.write_immediate(*old, dest)?; // old value is returned
Ok(())
}
AtomicOp::MirOp(op, neg) => {
let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
this.write_immediate(*old, dest)?; // old value is returned
Ok(())
}
}
}
fn atomic_exchange(
&mut self,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
atomic: AtomicRwOrd,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
let [place, new] = check_arg_count(args)?;
let place = this.deref_operand(place)?;
let new = this.read_scalar(new)?;
// Check alignment requirements. Atomics must always be aligned to their size,
// even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
// be 8-aligned).
let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
this.check_ptr_access_align(
place.ptr,
place.layout.size,
align,
CheckInAllocMsg::MemoryAccessTest,
)?;
let old = this.atomic_exchange_scalar(&place, new, atomic)?;
this.write_scalar(old, dest)?; // old value is returned
Ok(())
}
fn atomic_compare_exchange_impl(
&mut self,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
success: AtomicRwOrd,
fail: AtomicReadOrd,
can_fail_spuriously: bool,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
let [place, expect_old, new] = check_arg_count(args)?;
let place = this.deref_operand(place)?;
let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
let new = this.read_scalar(new)?;
// Check alignment requirements. Atomics must always be aligned to their size,
// even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
// be 8-aligned).
let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
this.check_ptr_access_align(
place.ptr,
place.layout.size,
align,
CheckInAllocMsg::MemoryAccessTest,
)?;
let old = this.atomic_compare_exchange_scalar(
&place,
&expect_old,
new,
success,
fail,
can_fail_spuriously,
)?;
// Return old value.
this.write_immediate(old, dest)?;
Ok(())
}
fn atomic_compare_exchange(
&mut self,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
success: AtomicRwOrd,
fail: AtomicReadOrd,
) -> InterpResult<'tcx> {
self.atomic_compare_exchange_impl(args, dest, success, fail, false)
}
fn atomic_compare_exchange_weak(
&mut self,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
success: AtomicRwOrd,
fail: AtomicReadOrd,
) -> InterpResult<'tcx> {
self.atomic_compare_exchange_impl(args, dest, success, fail, true)
}
}

426
src/shims/intrinsics/mod.rs Normal file
View File

@ -0,0 +1,426 @@
mod atomic;
mod simd;
use std::iter;
use log::trace;
use rustc_apfloat::{Float, Round};
use rustc_middle::ty::layout::{IntegerExt, LayoutOf};
use rustc_middle::{mir, ty, ty::FloatTy};
use rustc_target::abi::Integer;
use crate::*;
use atomic::EvalContextExt as _;
use helpers::check_arg_count;
use simd::EvalContextExt as _;
impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
fn call_intrinsic(
&mut self,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
ret: Option<mir::BasicBlock>,
_unwind: StackPopUnwind,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
// See if the core engine can handle this intrinsic.
if this.emulate_intrinsic(instance, args, dest, ret)? {
return Ok(());
}
// All remaining supported intrinsics have a return place.
let intrinsic_name = this.tcx.item_name(instance.def_id());
let intrinsic_name = intrinsic_name.as_str();
let ret = match ret {
None => throw_unsup_format!("unimplemented (diverging) intrinsic: `{intrinsic_name}`"),
Some(p) => p,
};
// Some intrinsics are special and need the "ret".
match intrinsic_name {
"try" => return this.handle_try(args, dest, ret),
_ => {}
}
// The rest jumps to `ret` immediately.
this.emulate_intrinsic_by_name(intrinsic_name, args, dest)?;
trace!("{:?}", this.dump_place(**dest));
this.go_to_block(ret);
Ok(())
}
/// Emulates a Miri-supported intrinsic (not supported by the core engine).
fn emulate_intrinsic_by_name(
&mut self,
intrinsic_name: &str,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
return this.emulate_atomic_intrinsic(name, args, dest);
}
if let Some(name) = intrinsic_name.strip_prefix("simd_") {
return this.emulate_simd_intrinsic(name, args, dest);
}
match intrinsic_name {
// Miri overwriting CTFE intrinsics.
"ptr_guaranteed_eq" => {
let [left, right] = check_arg_count(args)?;
let left = this.read_immediate(left)?;
let right = this.read_immediate(right)?;
this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
}
"ptr_guaranteed_ne" => {
let [left, right] = check_arg_count(args)?;
let left = this.read_immediate(left)?;
let right = this.read_immediate(right)?;
this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
}
"const_allocate" => {
// For now, for compatibility with the run-time implementation of this, we just return null.
// See <https://github.com/rust-lang/rust/issues/93935>.
this.write_null(dest)?;
}
"const_deallocate" => {
// complete NOP
}
// Raw memory accesses
"volatile_load" => {
let [place] = check_arg_count(args)?;
let place = this.deref_operand(place)?;
this.copy_op(&place.into(), dest, /*allow_transmute*/ false)?;
}
"volatile_store" => {
let [place, dest] = check_arg_count(args)?;
let place = this.deref_operand(place)?;
this.copy_op(dest, &place.into(), /*allow_transmute*/ false)?;
}
"write_bytes" | "volatile_set_memory" => {
let [ptr, val_byte, count] = check_arg_count(args)?;
let ty = ptr.layout.ty.builtin_deref(true).unwrap().ty;
let ty_layout = this.layout_of(ty)?;
let val_byte = this.read_scalar(val_byte)?.to_u8()?;
let ptr = this.read_pointer(ptr)?;
let count = this.read_scalar(count)?.to_machine_usize(this)?;
// `checked_mul` enforces a too small bound (the correct one would probably be machine_isize_max),
// but no actual allocation can be big enough for the difference to be noticeable.
let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
err_ub_format!("overflow computing total size of `{intrinsic_name}`")
})?;
this.write_bytes_ptr(
ptr,
iter::repeat(val_byte).take(byte_count.bytes() as usize),
)?;
}
// Floating-point operations
"fabsf32" => {
let [f] = check_arg_count(args)?;
let f = this.read_scalar(f)?.to_f32()?;
// Can be implemented in soft-floats.
this.write_scalar(Scalar::from_f32(f.abs()), dest)?;
}
"fabsf64" => {
let [f] = check_arg_count(args)?;
let f = this.read_scalar(f)?.to_f64()?;
// Can be implemented in soft-floats.
this.write_scalar(Scalar::from_f64(f.abs()), dest)?;
}
#[rustfmt::skip]
| "sinf32"
| "cosf32"
| "sqrtf32"
| "expf32"
| "exp2f32"
| "logf32"
| "log10f32"
| "log2f32"
| "floorf32"
| "ceilf32"
| "truncf32"
| "roundf32"
=> {
let [f] = check_arg_count(args)?;
// FIXME: Using host floats.
let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
let f = match intrinsic_name {
"sinf32" => f.sin(),
"cosf32" => f.cos(),
"sqrtf32" => f.sqrt(),
"expf32" => f.exp(),
"exp2f32" => f.exp2(),
"logf32" => f.ln(),
"log10f32" => f.log10(),
"log2f32" => f.log2(),
"floorf32" => f.floor(),
"ceilf32" => f.ceil(),
"truncf32" => f.trunc(),
"roundf32" => f.round(),
_ => bug!(),
};
this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
}
#[rustfmt::skip]
| "sinf64"
| "cosf64"
| "sqrtf64"
| "expf64"
| "exp2f64"
| "logf64"
| "log10f64"
| "log2f64"
| "floorf64"
| "ceilf64"
| "truncf64"
| "roundf64"
=> {
let [f] = check_arg_count(args)?;
// FIXME: Using host floats.
let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
let f = match intrinsic_name {
"sinf64" => f.sin(),
"cosf64" => f.cos(),
"sqrtf64" => f.sqrt(),
"expf64" => f.exp(),
"exp2f64" => f.exp2(),
"logf64" => f.ln(),
"log10f64" => f.log10(),
"log2f64" => f.log2(),
"floorf64" => f.floor(),
"ceilf64" => f.ceil(),
"truncf64" => f.trunc(),
"roundf64" => f.round(),
_ => bug!(),
};
this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
}
#[rustfmt::skip]
| "fadd_fast"
| "fsub_fast"
| "fmul_fast"
| "fdiv_fast"
| "frem_fast"
=> {
let [a, b] = check_arg_count(args)?;
let a = this.read_immediate(a)?;
let b = this.read_immediate(b)?;
let op = match intrinsic_name {
"fadd_fast" => mir::BinOp::Add,
"fsub_fast" => mir::BinOp::Sub,
"fmul_fast" => mir::BinOp::Mul,
"fdiv_fast" => mir::BinOp::Div,
"frem_fast" => mir::BinOp::Rem,
_ => bug!(),
};
let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
Ok(match x.layout.ty.kind() {
ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
_ => bug!(
"`{intrinsic_name}` called with non-float input type {ty:?}",
ty = x.layout.ty,
),
})
};
match (float_finite(a)?, float_finite(b)?) {
(false, false) => throw_ub_format!(
"`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
),
(false, _) => throw_ub_format!(
"`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
),
(_, false) => throw_ub_format!(
"`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
),
_ => {}
}
this.binop_ignore_overflow(op, &a, &b, dest)?;
}
#[rustfmt::skip]
| "minnumf32"
| "maxnumf32"
| "copysignf32"
=> {
let [a, b] = check_arg_count(args)?;
let a = this.read_scalar(a)?.to_f32()?;
let b = this.read_scalar(b)?.to_f32()?;
let res = match intrinsic_name {
"minnumf32" => a.min(b),
"maxnumf32" => a.max(b),
"copysignf32" => a.copy_sign(b),
_ => bug!(),
};
this.write_scalar(Scalar::from_f32(res), dest)?;
}
#[rustfmt::skip]
| "minnumf64"
| "maxnumf64"
| "copysignf64"
=> {
let [a, b] = check_arg_count(args)?;
let a = this.read_scalar(a)?.to_f64()?;
let b = this.read_scalar(b)?.to_f64()?;
let res = match intrinsic_name {
"minnumf64" => a.min(b),
"maxnumf64" => a.max(b),
"copysignf64" => a.copy_sign(b),
_ => bug!(),
};
this.write_scalar(Scalar::from_f64(res), dest)?;
}
"powf32" => {
let [f, f2] = check_arg_count(args)?;
// FIXME: Using host floats.
let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
}
"powf64" => {
let [f, f2] = check_arg_count(args)?;
// FIXME: Using host floats.
let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
}
"fmaf32" => {
let [a, b, c] = check_arg_count(args)?;
let a = this.read_scalar(a)?.to_f32()?;
let b = this.read_scalar(b)?.to_f32()?;
let c = this.read_scalar(c)?.to_f32()?;
let res = a.mul_add(b, c).value;
this.write_scalar(Scalar::from_f32(res), dest)?;
}
"fmaf64" => {
let [a, b, c] = check_arg_count(args)?;
let a = this.read_scalar(a)?.to_f64()?;
let b = this.read_scalar(b)?.to_f64()?;
let c = this.read_scalar(c)?.to_f64()?;
let res = a.mul_add(b, c).value;
this.write_scalar(Scalar::from_f64(res), dest)?;
}
"powif32" => {
let [f, i] = check_arg_count(args)?;
// FIXME: Using host floats.
let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
let i = this.read_scalar(i)?.to_i32()?;
this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
}
"powif64" => {
let [f, i] = check_arg_count(args)?;
// FIXME: Using host floats.
let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
let i = this.read_scalar(i)?.to_i32()?;
this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
}
"float_to_int_unchecked" => {
let [val] = check_arg_count(args)?;
let val = this.read_immediate(val)?;
let res = match val.layout.ty.kind() {
ty::Float(FloatTy::F32) =>
this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
ty::Float(FloatTy::F64) =>
this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
_ =>
span_bug!(
this.cur_span(),
"`float_to_int_unchecked` called with non-float input type {:?}",
val.layout.ty
),
};
this.write_scalar(res, dest)?;
}
// Other
"exact_div" => {
let [num, denom] = check_arg_count(args)?;
this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
}
"breakpoint" => {
let [] = check_arg_count(args)?;
// normally this would raise a SIGTRAP, which aborts if no debugger is connected
throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
}
name => throw_unsup_format!("unimplemented intrinsic: `{name}`"),
}
Ok(())
}
fn float_to_int_unchecked<F>(
&self,
f: F,
dest_ty: ty::Ty<'tcx>,
) -> InterpResult<'tcx, Scalar<Tag>>
where
F: Float + Into<Scalar<Tag>>,
{
let this = self.eval_context_ref();
// Step 1: cut off the fractional part of `f`. The result of this is
// guaranteed to be precisely representable in IEEE floats.
let f = f.round_to_integral(Round::TowardZero).value;
// Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
Ok(match dest_ty.kind() {
// Unsigned
ty::Uint(t) => {
let size = Integer::from_uint_ty(this, *t).size();
let res = f.to_u128(size.bits_usize());
if res.status.is_empty() {
// No status flags means there was no further rounding or other loss of precision.
Scalar::from_uint(res.value, size)
} else {
// `f` was not representable in this integer type.
throw_ub_format!(
"`float_to_int_unchecked` intrinsic called on {f} which cannot be represented in target type `{dest_ty:?}`",
);
}
}
// Signed
ty::Int(t) => {
let size = Integer::from_int_ty(this, *t).size();
let res = f.to_i128(size.bits_usize());
if res.status.is_empty() {
// No status flags means there was no further rounding or other loss of precision.
Scalar::from_int(res.value, size)
} else {
// `f` was not representable in this integer type.
throw_ub_format!(
"`float_to_int_unchecked` intrinsic called on {f} which cannot be represented in target type `{dest_ty:?}`",
);
}
}
// Nothing else
_ =>
span_bug!(
this.cur_span(),
"`float_to_int_unchecked` called with non-int output type {dest_ty:?}"
),
})
}
}

613
src/shims/intrinsics/simd.rs Normal file
View File

@ -0,0 +1,613 @@
use rustc_apfloat::Float;
use rustc_middle::ty::layout::{HasParamEnv, LayoutOf};
use rustc_middle::{mir, ty, ty::FloatTy};
use rustc_target::abi::{Endian, HasDataLayout, Size};
use crate::*;
use helpers::check_arg_count;
impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
/// Calls the simd intrinsic `intrinsic`; the `simd_` prefix has already been removed.
fn emulate_simd_intrinsic(
&mut self,
intrinsic_name: &str,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
) -> InterpResult<'tcx> {
let this = self.eval_context_mut();
match intrinsic_name {
#[rustfmt::skip]
| "neg"
| "fabs"
| "ceil"
| "floor"
| "round"
| "trunc"
| "fsqrt" => {
let [op] = check_arg_count(args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.place_to_simd(dest)?;
assert_eq!(dest_len, op_len);
#[derive(Copy, Clone)]
enum HostFloatOp {
Ceil,
Floor,
Round,
Trunc,
Sqrt,
}
#[derive(Copy, Clone)]
enum Op {
MirOp(mir::UnOp),
Abs,
HostOp(HostFloatOp),
}
let which = match intrinsic_name {
"neg" => Op::MirOp(mir::UnOp::Neg),
"fabs" => Op::Abs,
"ceil" => Op::HostOp(HostFloatOp::Ceil),
"floor" => Op::HostOp(HostFloatOp::Floor),
"round" => Op::HostOp(HostFloatOp::Round),
"trunc" => Op::HostOp(HostFloatOp::Trunc),
"fsqrt" => Op::HostOp(HostFloatOp::Sqrt),
_ => unreachable!(),
};
for i in 0..dest_len {
let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
let dest = this.mplace_index(&dest, i)?;
let val = match which {
Op::MirOp(mir_op) => this.unary_op(mir_op, &op)?.to_scalar()?,
Op::Abs => {
// Works for f32 and f64.
let ty::Float(float_ty) = op.layout.ty.kind() else {
span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
};
let op = op.to_scalar()?;
match float_ty {
FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()),
FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()),
}
}
Op::HostOp(host_op) => {
let ty::Float(float_ty) = op.layout.ty.kind() else {
span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
};
// FIXME using host floats
match float_ty {
FloatTy::F32 => {
let f = f32::from_bits(op.to_scalar()?.to_u32()?);
let res = match host_op {
HostFloatOp::Ceil => f.ceil(),
HostFloatOp::Floor => f.floor(),
HostFloatOp::Round => f.round(),
HostFloatOp::Trunc => f.trunc(),
HostFloatOp::Sqrt => f.sqrt(),
};
Scalar::from_u32(res.to_bits())
}
FloatTy::F64 => {
let f = f64::from_bits(op.to_scalar()?.to_u64()?);
let res = match host_op {
HostFloatOp::Ceil => f.ceil(),
HostFloatOp::Floor => f.floor(),
HostFloatOp::Round => f.round(),
HostFloatOp::Trunc => f.trunc(),
HostFloatOp::Sqrt => f.sqrt(),
};
Scalar::from_u64(res.to_bits())
}
}
}
};
this.write_scalar(val, &dest.into())?;
}
}
#[rustfmt::skip]
| "add"
| "sub"
| "mul"
| "div"
| "rem"
| "shl"
| "shr"
| "and"
| "or"
| "xor"
| "eq"
| "ne"
| "lt"
| "le"
| "gt"
| "ge"
| "fmax"
| "fmin"
| "saturating_add"
| "saturating_sub"
| "arith_offset" => {
use mir::BinOp;
let [left, right] = check_arg_count(args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.place_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
enum Op {
MirOp(BinOp),
SaturatingOp(BinOp),
FMax,
FMin,
WrappingOffset,
}
let which = match intrinsic_name {
"add" => Op::MirOp(BinOp::Add),
"sub" => Op::MirOp(BinOp::Sub),
"mul" => Op::MirOp(BinOp::Mul),
"div" => Op::MirOp(BinOp::Div),
"rem" => Op::MirOp(BinOp::Rem),
"shl" => Op::MirOp(BinOp::Shl),
"shr" => Op::MirOp(BinOp::Shr),
"and" => Op::MirOp(BinOp::BitAnd),
"or" => Op::MirOp(BinOp::BitOr),
"xor" => Op::MirOp(BinOp::BitXor),
"eq" => Op::MirOp(BinOp::Eq),
"ne" => Op::MirOp(BinOp::Ne),
"lt" => Op::MirOp(BinOp::Lt),
"le" => Op::MirOp(BinOp::Le),
"gt" => Op::MirOp(BinOp::Gt),
"ge" => Op::MirOp(BinOp::Ge),
"fmax" => Op::FMax,
"fmin" => Op::FMin,
"saturating_add" => Op::SaturatingOp(BinOp::Add),
"saturating_sub" => Op::SaturatingOp(BinOp::Sub),
"arith_offset" => Op::WrappingOffset,
_ => unreachable!(),
};
for i in 0..dest_len {
let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
let dest = this.mplace_index(&dest, i)?;
let val = match which {
Op::MirOp(mir_op) => {
let (val, overflowed, ty) = this.overflowing_binary_op(mir_op, &left, &right)?;
if matches!(mir_op, BinOp::Shl | BinOp::Shr) {
// Shifts have extra UB as SIMD operations that the MIR binop does not have.
// See <https://github.com/rust-lang/rust/issues/91237>.
if overflowed {
let r_val = right.to_scalar()?.to_bits(right.layout.size)?;
throw_ub_format!("overflowing shift by {r_val} in `simd_{intrinsic_name}` in SIMD lane {i}");
}
}
if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) {
// Special handling for boolean-returning operations
assert_eq!(ty, this.tcx.types.bool);
let val = val.to_bool().unwrap();
bool_to_simd_element(val, dest.layout.size)
} else {
assert_ne!(ty, this.tcx.types.bool);
assert_eq!(ty, dest.layout.ty);
val
}
}
Op::SaturatingOp(mir_op) => {
this.saturating_arith(mir_op, &left, &right)?
}
Op::WrappingOffset => {
let ptr = this.scalar_to_ptr(left.to_scalar()?)?;
let offset_count = right.to_scalar()?.to_machine_isize(this)?;
let pointee_ty = left.layout.ty.builtin_deref(true).unwrap().ty;
let pointee_size = i64::try_from(this.layout_of(pointee_ty)?.size.bytes()).unwrap();
let offset_bytes = offset_count.wrapping_mul(pointee_size);
let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, this);
Scalar::from_maybe_pointer(offset_ptr, this)
}
Op::FMax => {
fmax_op(&left, &right)?
}
Op::FMin => {
fmin_op(&left, &right)?
}
};
this.write_scalar(val, &dest.into())?;
}
}
"fma" => {
let [a, b, c] = check_arg_count(args)?;
let (a, a_len) = this.operand_to_simd(a)?;
let (b, b_len) = this.operand_to_simd(b)?;
let (c, c_len) = this.operand_to_simd(c)?;
let (dest, dest_len) = this.place_to_simd(dest)?;
assert_eq!(dest_len, a_len);
assert_eq!(dest_len, b_len);
assert_eq!(dest_len, c_len);
for i in 0..dest_len {
let a = this.read_immediate(&this.mplace_index(&a, i)?.into())?.to_scalar()?;
let b = this.read_immediate(&this.mplace_index(&b, i)?.into())?.to_scalar()?;
let c = this.read_immediate(&this.mplace_index(&c, i)?.into())?.to_scalar()?;
let dest = this.mplace_index(&dest, i)?;
// Works for f32 and f64.
let ty::Float(float_ty) = dest.layout.ty.kind() else {
span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
};
let val = match float_ty {
FloatTy::F32 =>
Scalar::from_f32(a.to_f32()?.mul_add(b.to_f32()?, c.to_f32()?).value),
FloatTy::F64 =>
Scalar::from_f64(a.to_f64()?.mul_add(b.to_f64()?, c.to_f64()?).value),
};
this.write_scalar(val, &dest.into())?;
}
}
#[rustfmt::skip]
| "reduce_and"
| "reduce_or"
| "reduce_xor"
| "reduce_any"
| "reduce_all"
| "reduce_max"
| "reduce_min" => {
use mir::BinOp;
let [op] = check_arg_count(args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let imm_from_bool =
|b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool);
enum Op {
MirOp(BinOp),
MirOpBool(BinOp),
Max,
Min,
}
let which = match intrinsic_name {
"reduce_and" => Op::MirOp(BinOp::BitAnd),
"reduce_or" => Op::MirOp(BinOp::BitOr),
"reduce_xor" => Op::MirOp(BinOp::BitXor),
"reduce_any" => Op::MirOpBool(BinOp::BitOr),
"reduce_all" => Op::MirOpBool(BinOp::BitAnd),
"reduce_max" => Op::Max,
"reduce_min" => Op::Min,
_ => unreachable!(),
};
// Initialize with first lane, then proceed with the rest.
let mut res = this.read_immediate(&this.mplace_index(&op, 0)?.into())?;
if matches!(which, Op::MirOpBool(_)) {
// Convert to `bool` scalar.
res = imm_from_bool(simd_element_to_bool(res)?);
}
for i in 1..op_len {
let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
res = match which {
Op::MirOp(mir_op) => {
this.binary_op(mir_op, &res, &op)?
}
Op::MirOpBool(mir_op) => {
let op = imm_from_bool(simd_element_to_bool(op)?);
this.binary_op(mir_op, &res, &op)?
}
Op::Max => {
if matches!(res.layout.ty.kind(), ty::Float(_)) {
ImmTy::from_scalar(fmax_op(&res, &op)?, res.layout)
} else {
// Just boring integers, so NaNs to worry about
if this.binary_op(BinOp::Ge, &res, &op)?.to_scalar()?.to_bool()? {
res
} else {
op
}
}
}
Op::Min => {
if matches!(res.layout.ty.kind(), ty::Float(_)) {
ImmTy::from_scalar(fmin_op(&res, &op)?, res.layout)
} else {
// Just boring integers, so NaNs to worry about
if this.binary_op(BinOp::Le, &res, &op)?.to_scalar()?.to_bool()? {
res
} else {
op
}
}
}
};
}
this.write_immediate(*res, dest)?;
}
#[rustfmt::skip]
| "reduce_add_ordered"
| "reduce_mul_ordered" => {
use mir::BinOp;
let [op, init] = check_arg_count(args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let init = this.read_immediate(init)?;
let mir_op = match intrinsic_name {
"reduce_add_ordered" => BinOp::Add,
"reduce_mul_ordered" => BinOp::Mul,
_ => unreachable!(),
};
let mut res = init;
for i in 0..op_len {
let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
res = this.binary_op(mir_op, &res, &op)?;
}
this.write_immediate(*res, dest)?;
}
"select" => {
let [mask, yes, no] = check_arg_count(args)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (yes, yes_len) = this.operand_to_simd(yes)?;
let (no, no_len) = this.operand_to_simd(no)?;
let (dest, dest_len) = this.place_to_simd(dest)?;
assert_eq!(dest_len, mask_len);
assert_eq!(dest_len, yes_len);
assert_eq!(dest_len, no_len);
for i in 0..dest_len {
let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
let dest = this.mplace_index(&dest, i)?;
let val = if simd_element_to_bool(mask)? { yes } else { no };
this.write_immediate(*val, &dest.into())?;
}
}
"select_bitmask" => {
let [mask, yes, no] = check_arg_count(args)?;
let (yes, yes_len) = this.operand_to_simd(yes)?;
let (no, no_len) = this.operand_to_simd(no)?;
let (dest, dest_len) = this.place_to_simd(dest)?;
let bitmask_len = dest_len.max(8);
assert!(mask.layout.ty.is_integral());
assert!(bitmask_len <= 64);
assert_eq!(bitmask_len, mask.layout.size.bits());
assert_eq!(dest_len, yes_len);
assert_eq!(dest_len, no_len);
let mask: u64 = this
.read_scalar(mask)?
.check_init()?
.to_bits(mask.layout.size)?
.try_into()
.unwrap();
for i in 0..dest_len {
let mask =
mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian));
let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
let dest = this.mplace_index(&dest, i)?;
let val = if mask != 0 { yes } else { no };
this.write_immediate(*val, &dest.into())?;
}
for i in dest_len..bitmask_len {
// If the mask is "padded", ensure that padding is all-zero.
let mask = mask & (1 << i);
if mask != 0 {
throw_ub_format!(
"a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits"
);
}
}
}
#[rustfmt::skip]
"cast" | "as" => {
let [op] = check_arg_count(args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.place_to_simd(dest)?;
assert_eq!(dest_len, op_len);
let safe_cast = intrinsic_name == "as";
for i in 0..dest_len {
let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
let dest = this.mplace_index(&dest, i)?;
let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) {
// Int-to-(int|float): always safe
(ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) =>
this.misc_cast(&op, dest.layout.ty)?,
// Float-to-float: always safe
(ty::Float(_), ty::Float(_)) =>
this.misc_cast(&op, dest.layout.ty)?,
// Float-to-int in safe mode
(ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast =>
this.misc_cast(&op, dest.layout.ty)?,
// Float-to-int in unchecked mode
(ty::Float(FloatTy::F32), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
this.float_to_int_unchecked(op.to_scalar()?.to_f32()?, dest.layout.ty)?.into(),
(ty::Float(FloatTy::F64), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
this.float_to_int_unchecked(op.to_scalar()?.to_f64()?, dest.layout.ty)?.into(),
_ =>
throw_unsup_format!(
"Unsupported SIMD cast from element type {from_ty} to {to_ty}",
from_ty = op.layout.ty,
to_ty = dest.layout.ty,
),
};
this.write_immediate(val, &dest.into())?;
}
}
"shuffle" => {
let [left, right, index] = check_arg_count(args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.place_to_simd(dest)?;
// `index` is an array, not a SIMD type
let ty::Array(_, index_len) = index.layout.ty.kind() else {
span_bug!(this.cur_span(), "simd_shuffle index argument has non-array type {}", index.layout.ty)
};
let index_len = index_len.eval_usize(*this.tcx, this.param_env());
assert_eq!(left_len, right_len);
assert_eq!(index_len, dest_len);
for i in 0..dest_len {
let src_index: u64 = this
.read_immediate(&this.operand_index(index, i)?)?
.to_scalar()?
.to_u32()?
.into();
let dest = this.mplace_index(&dest, i)?;
let val = if src_index < left_len {
this.read_immediate(&this.mplace_index(&left, src_index)?.into())?
} else if src_index < left_len.checked_add(right_len).unwrap() {
this.read_immediate(
&this.mplace_index(&right, src_index - left_len)?.into(),
)?
} else {
span_bug!(
this.cur_span(),
"simd_shuffle index {src_index} is out of bounds for 2 vectors of size {left_len}",
);
};
this.write_immediate(*val, &dest.into())?;
}
}
"gather" => {
let [passthru, ptrs, mask] = check_arg_count(args)?;
let (passthru, passthru_len) = this.operand_to_simd(passthru)?;
let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (dest, dest_len) = this.place_to_simd(dest)?;
assert_eq!(dest_len, passthru_len);
assert_eq!(dest_len, ptrs_len);
assert_eq!(dest_len, mask_len);
for i in 0..dest_len {
let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?;
let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
let dest = this.mplace_index(&dest, i)?;
let val = if simd_element_to_bool(mask)? {
let place = this.deref_operand(&ptr.into())?;
this.read_immediate(&place.into())?
} else {
passthru
};
this.write_immediate(*val, &dest.into())?;
}
}
"scatter" => {
let [value, ptrs, mask] = check_arg_count(args)?;
let (value, value_len) = this.operand_to_simd(value)?;
let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
assert_eq!(ptrs_len, value_len);
assert_eq!(ptrs_len, mask_len);
for i in 0..ptrs_len {
let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?;
let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
if simd_element_to_bool(mask)? {
let place = this.deref_operand(&ptr.into())?;
this.write_immediate(*value, &place.into())?;
}
}
}
"bitmask" => {
let [op] = check_arg_count(args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let bitmask_len = op_len.max(8);
assert!(dest.layout.ty.is_integral());
assert!(bitmask_len <= 64);
assert_eq!(bitmask_len, dest.layout.size.bits());
let mut res = 0u64;
for i in 0..op_len {
let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
if simd_element_to_bool(op)? {
res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian);
}
}
this.write_int(res, dest)?;
}
name => throw_unsup_format!("unimplemented intrinsic: `simd_{name}`"),
}
Ok(())
}
}
fn bool_to_simd_element(b: bool, size: Size) -> Scalar<Tag> {
// SIMD uses all-1 as pattern for "true"
let val = if b { -1 } else { 0 };
Scalar::from_int(val, size)
}
fn simd_element_to_bool(elem: ImmTy<'_, Tag>) -> InterpResult<'_, bool> {
let val = elem.to_scalar()?.to_int(elem.layout.size)?;
Ok(match val {
0 => false,
-1 => true,
_ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"),
})
}
fn simd_bitmask_index(idx: u64, vec_len: u64, endianess: Endian) -> u64 {
assert!(idx < vec_len);
match endianess {
Endian::Little => idx,
Endian::Big => vec_len - 1 - idx, // reverse order of bits
}
}
fn fmax_op<'tcx>(
left: &ImmTy<'tcx, Tag>,
right: &ImmTy<'tcx, Tag>,
) -> InterpResult<'tcx, Scalar<Tag>> {
assert_eq!(left.layout.ty, right.layout.ty);
let ty::Float(float_ty) = left.layout.ty.kind() else {
bug!("fmax operand is not a float")
};
let left = left.to_scalar()?;
let right = right.to_scalar()?;
Ok(match float_ty {
FloatTy::F32 => Scalar::from_f32(left.to_f32()?.max(right.to_f32()?)),
FloatTy::F64 => Scalar::from_f64(left.to_f64()?.max(right.to_f64()?)),
})
}
fn fmin_op<'tcx>(
left: &ImmTy<'tcx, Tag>,
right: &ImmTy<'tcx, Tag>,
) -> InterpResult<'tcx, Scalar<Tag>> {
assert_eq!(left.layout.ty, right.layout.ty);
let ty::Float(float_ty) = left.layout.ty.kind() else {
bug!("fmin operand is not a float")
};
let left = left.to_scalar()?;
let right = right.to_scalar()?;
Ok(match float_ty {
FloatTy::F32 => Scalar::from_f32(left.to_f32()?.min(right.to_f32()?)),
FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)),
})
}

8
src/shims/unix/foreign_items.rs
View File

@ -2,7 +2,6 @@ use std::ffi::OsStr;
use log::trace;
use rustc_middle::mir;
use rustc_middle::ty::layout::LayoutOf;
use rustc_span::Symbol;
use rustc_target::abi::{Align, Size};
@ -22,7 +21,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();
@ -533,9 +531,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
// Platform-specific shims
_ => {
match this.tcx.sess.target.os.as_ref() {
"linux" => return shims::unix::linux::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest, ret),
"macos" => return shims::unix::macos::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest, ret),
"freebsd" => return shims::unix::freebsd::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest, ret),
"linux" => return shims::unix::linux::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest),
"macos" => return shims::unix::macos::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest),
"freebsd" => return shims::unix::freebsd::foreign_items::EvalContextExt::emulate_foreign_item_by_name(this, link_name, abi, args, dest),
_ => unreachable!(),
}
}

2
src/shims/unix/freebsd/foreign_items.rs
View File

@ -1,4 +1,3 @@
use rustc_middle::mir;
use rustc_span::Symbol;
use rustc_target::spec::abi::Abi;
@ -14,7 +13,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
_ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();
match link_name.as_str() {

2
src/shims/unix/linux/foreign_items.rs
View File

@ -1,4 +1,3 @@
use rustc_middle::mir;
use rustc_span::Symbol;
use rustc_target::spec::abi::Abi;
@ -17,7 +16,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
_ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();

4
src/shims/unix/linux/sync.rs
View File

@ -169,7 +169,7 @@ pub fn futex<'tcx>(
//
// Thankfully, preemptions cannot happen inside a Miri shim, so we do not need to
// do anything special to guarantee fence-load-comparison atomicity.
this.atomic_fence(&[], AtomicFenceOrd::SeqCst)?;
this.validate_atomic_fence(AtomicFenceOrd::SeqCst)?;
// Read an `i32` through the pointer, regardless of any wrapper types.
// It's not uncommon for `addr` to be passed as another type than `*mut i32`, such as `*const AtomicI32`.
let futex_val = this
@ -240,7 +240,7 @@ pub fn futex<'tcx>(
// Together with the SeqCst fence in futex_wait, this makes sure that futex_wait
// will see the latest value on addr which could be changed by our caller
// before doing the syscall.
this.atomic_fence(&[], AtomicFenceOrd::SeqCst)?;
this.validate_atomic_fence(AtomicFenceOrd::SeqCst)?;
let mut n = 0;
for _ in 0..val {
if let Some(thread) = this.futex_wake(addr_usize, bitset) {

2
src/shims/unix/macos/foreign_items.rs
View File

@ -1,4 +1,3 @@
use rustc_middle::mir;
use rustc_span::Symbol;
use rustc_target::spec::abi::Abi;
@ -15,7 +14,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
_ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();

2
src/shims/windows/foreign_items.rs
View File

@ -1,6 +1,5 @@
use std::iter;
use rustc_middle::mir;
use rustc_span::Symbol;
use rustc_target::abi::Size;
use rustc_target::spec::abi::Abi;
@ -18,7 +17,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
abi: Abi,
args: &[OpTy<'tcx, Tag>],
dest: &PlaceTy<'tcx, Tag>,
_ret: mir::BasicBlock,
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
let this = self.eval_context_mut();

45
tests/pass/atomic.rs
View File

@ -51,18 +51,22 @@ fn atomic_all_ops() {
static ATOMIC: AtomicIsize = AtomicIsize::new(0);
static ATOMIC_UNSIGNED: AtomicU64 = AtomicU64::new(0);
let load_orders = [Relaxed, Acquire, SeqCst];
let stored_orders = [Relaxed, Release, SeqCst];
let rmw_orders = [Relaxed, Release, Acquire, AcqRel, SeqCst];
// loads
for o in [Relaxed, Acquire, SeqCst] {
for o in load_orders {
ATOMIC.load(o);
}
// stores
for o in [Relaxed, Release, SeqCst] {
for o in stored_orders {
ATOMIC.store(1, o);
}
// most RMWs
for o in [Relaxed, Release, Acquire, AcqRel, SeqCst] {
for o in rmw_orders {
ATOMIC.swap(0, o);
ATOMIC.fetch_or(0, o);
ATOMIC.fetch_xor(0, o);
@ -76,29 +80,13 @@ fn atomic_all_ops() {
ATOMIC_UNSIGNED.fetch_max(0, o);
}
// RMWs with deparate failure ordering
ATOMIC.store(0, SeqCst);
assert_eq!(ATOMIC.compare_exchange(0, 1, Relaxed, Relaxed), Ok(0));
assert_eq!(ATOMIC.compare_exchange(0, 2, Acquire, Relaxed), Err(1));
assert_eq!(ATOMIC.compare_exchange(0, 1, Release, Relaxed), Err(1));
assert_eq!(ATOMIC.compare_exchange(1, 0, AcqRel, Relaxed), Ok(1));
ATOMIC.compare_exchange(0, 1, SeqCst, Relaxed).ok();
ATOMIC.compare_exchange(0, 1, Acquire, Acquire).ok();
ATOMIC.compare_exchange(0, 1, AcqRel, Acquire).ok();
ATOMIC.compare_exchange(0, 1, SeqCst, Acquire).ok();
ATOMIC.compare_exchange(0, 1, SeqCst, SeqCst).ok();
ATOMIC.store(0, SeqCst);
compare_exchange_weak_loop!(ATOMIC, 0, 1, Relaxed, Relaxed);
assert_eq!(ATOMIC.compare_exchange_weak(0, 2, Acquire, Relaxed), Err(1));
assert_eq!(ATOMIC.compare_exchange_weak(0, 1, Release, Relaxed), Err(1));
compare_exchange_weak_loop!(ATOMIC, 1, 0, AcqRel, Relaxed);
assert_eq!(ATOMIC.load(Relaxed), 0);
ATOMIC.compare_exchange_weak(0, 1, SeqCst, Relaxed).ok();
ATOMIC.compare_exchange_weak(0, 1, Acquire, Acquire).ok();
ATOMIC.compare_exchange_weak(0, 1, AcqRel, Acquire).ok();
ATOMIC.compare_exchange_weak(0, 1, SeqCst, Acquire).ok();
ATOMIC.compare_exchange_weak(0, 1, SeqCst, SeqCst).ok();
// RMWs with separate failure ordering
for o1 in rmw_orders {
for o2 in load_orders {
let _res = ATOMIC.compare_exchange(0, 0, o1, o2);
let _res = ATOMIC.compare_exchange_weak(0, 0, o1, o2);
}
}
}
fn atomic_u64() {
@ -106,7 +94,12 @@ fn atomic_u64() {
ATOMIC.store(1, SeqCst);
assert_eq!(ATOMIC.compare_exchange(0, 0x100, AcqRel, Acquire), Err(1));
assert_eq!(ATOMIC.compare_exchange(0, 1, Release, Relaxed), Err(1));
assert_eq!(ATOMIC.compare_exchange(1, 0, AcqRel, Relaxed), Ok(1));
assert_eq!(ATOMIC.compare_exchange(0, 1, Relaxed, Relaxed), Ok(0));
compare_exchange_weak_loop!(ATOMIC, 1, 0x100, AcqRel, Acquire);
assert_eq!(ATOMIC.compare_exchange_weak(0, 2, Acquire, Relaxed), Err(0x100));
assert_eq!(ATOMIC.compare_exchange_weak(0, 1, Release, Relaxed), Err(0x100));
assert_eq!(ATOMIC.load(Relaxed), 0x100);
assert_eq!(ATOMIC.fetch_max(0x10, SeqCst), 0x100);