mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #118310 - scottmcm:three-way-compare, r=davidtwco

Browse Source 
Add `Ord::cmp` for primitives as a `BinOp` in MIR

Update: most of this OP was written months ago.  See https://github.com/rust-lang/rust/pull/118310#issuecomment-2016940014 below for where we got to recently that made it ready for review.

---

There are dozens of reasonable ways to implement `Ord::cmp` for integers using comparison, bit-ops, and branches.  Those differences are irrelevant at the rust level, however, so we can make things better by adding `BinOp::Cmp` at the MIR level:

1. Exactly how to implement it is left up to the backends, so LLVM can use whatever pattern its optimizer best recognizes and cranelift can use whichever pattern codegens the fastest.
2. By not inlining those details for every use of `cmp`, we drastically reduce the amount of MIR generated for `derive`d `PartialOrd`, while also making it more amenable to MIR-level optimizations.

Having extremely careful `if` ordering to μoptimize resource usage on broadwell (#63767) is great, but it really feels to me like libcore is the wrong place to put that logic.  Similarly, using subtraction [tricks](https://graphics.stanford.edu/~seander/bithacks.html#CopyIntegerSign) (#105840) is arguably even nicer, but depends on the optimizer understanding it (https://github.com/llvm/llvm-project/issues/73417) to be practical.  Or maybe [bitor is better than add](https://discourse.llvm.org/t/representing-in-ir/67369/2?u=scottmcm)?  But maybe only on a future version that [has `or disjoint` support](https://discourse.llvm.org/t/rfc-add-or-disjoint-flag/75036?u=scottmcm)?  And just because one of those forms happens to be good for LLVM, there's no guarantee that it'd be the same form that GCC or Cranelift would rather see -- especially given their very different optimizers.  Not to mention that if LLVM gets a spaceship intrinsic -- [which it should](https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/Suboptimal.20inlining.20in.20std.20function.20.60binary_search.60/near/404250586) -- we'll need at least a rustc intrinsic to be able to call it.

As for simplifying it in Rust, we now regularly inline `{integer}::partial_cmp`, but it's quite a large amount of IR.  The best way to see that is with 8811efa88b (diff-d134c32d028fbe2bf835fef2df9aca9d13332dd82284ff21ee7ebf717bfa4765R113) -- I added a new pre-codegen MIR test for a simple 3-tuple struct, and this PR change it from 36 locals and 26 basic blocks down to 24 locals and 8 basic blocks.  Even better, as soon as the construct-`Some`-then-match-it-in-same-BB noise is cleaned up, this'll expose the `Cmp == 0` branches clearly in MIR, so that an InstCombine (#105808) can simplify that to just a `BinOp::Eq` and thus fix some of our generated code perf issues.  (Tracking that through today's `if a < b { Less } else if a == b { Equal } else { Greater }` would be *much* harder.)

---

r? `@ghost`
But first I should check that perf is ok with this
~~...and my true nemesis, tidy.~~
This commit is contained in:
bors 2024-04-02 19:21:44 +00:00
commit a77322c16f
36 changed files with 601 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
compiler/rustc_codegen_cranelift/src/codegen_i128.rs
View File

@ -68,7 +68,7 @@ pub(crate) fn maybe_codegen<'tcx>(
Some(CValue::by_val(ret_val, lhs.layout()))
}
}
BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => None,
BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne | BinOp::Cmp => None,
BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => None,
}
}
@ -134,6 +134,7 @@ pub(crate) fn maybe_codegen_checked<'tcx>(
BinOp::AddUnchecked | BinOp::SubUnchecked | BinOp::MulUnchecked => unreachable!(),
BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
BinOp::Div | BinOp::Rem => unreachable!(),
BinOp::Cmp => unreachable!(),
BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => unreachable!(),
BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => unreachable!(),
}

24
compiler/rustc_codegen_cranelift/src/num.rs
View File

@ -40,6 +40,22 @@ pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
})
}
fn codegen_three_way_compare<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
signed: bool,
lhs: Value,
rhs: Value,
) -> CValue<'tcx> {
// This emits `(lhs > rhs) - (lhs < rhs)`, which is cranelift's preferred form per
// <https://github.com/bytecodealliance/wasmtime/blob/8052bb9e3b792503b225f2a5b2ba3bc023bff462/cranelift/codegen/src/prelude_opt.isle#L41-L47>
let gt_cc = crate::num::bin_op_to_intcc(BinOp::Gt, signed).unwrap();
let lt_cc = crate::num::bin_op_to_intcc(BinOp::Lt, signed).unwrap();
let gt = fx.bcx.ins().icmp(gt_cc, lhs, rhs);
let lt = fx.bcx.ins().icmp(lt_cc, lhs, rhs);
let val = fx.bcx.ins().isub(gt, lt);
CValue::by_val(val, fx.layout_of(fx.tcx.ty_ordering_enum(Some(fx.mir.span))))
}
fn codegen_compare_bin_op<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
@ -47,6 +63,10 @@ fn codegen_compare_bin_op<'tcx>(
lhs: Value,
rhs: Value,
) -> CValue<'tcx> {
if bin_op == BinOp::Cmp {
return codegen_three_way_compare(fx, signed, lhs, rhs);
}
let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap();
let val = fx.bcx.ins().icmp(intcc, lhs, rhs);
CValue::by_val(val, fx.layout_of(fx.tcx.types.bool))
@ -59,7 +79,7 @@ pub(crate) fn codegen_binop<'tcx>(
in_rhs: CValue<'tcx>,
) -> CValue<'tcx> {
match bin_op {
BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt | BinOp::Cmp => {
match in_lhs.layout().ty.kind() {
ty::Bool | ty::Uint(_) | ty::Int(_) | ty::Char => {
let signed = type_sign(in_lhs.layout().ty);
@ -160,7 +180,7 @@ pub(crate) fn codegen_int_binop<'tcx>(
}
BinOp::Offset => unreachable!("Offset is not an integer operation"),
// Compare binops handles by `codegen_binop`.
BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge => {
BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge | BinOp::Cmp => {
unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty);
}
};

4
compiler/rustc_codegen_gcc/src/common.rs
View File

@ -94,6 +94,10 @@ fn const_i32(&self, i: i32) -> RValue<'gcc> {
self.const_int(self.type_i32(), i as i64)
}
fn const_i8(&self, i: i8) -> RValue<'gcc> {
self.const_int(self.type_i8(), i as i64)
}
fn const_u32(&self, i: u32) -> RValue<'gcc> {
self.const_uint(self.type_u32(), i as u64)
}

4
compiler/rustc_codegen_llvm/src/common.rs
View File

@ -160,6 +160,10 @@ fn const_i32(&self, i: i32) -> &'ll Value {
self.const_int(self.type_i32(), i as i64)
}
fn const_i8(&self, i: i8) -> &'ll Value {
self.const_int(self.type_i8(), i as i64)
}
fn const_u32(&self, i: u32) -> &'ll Value {
self.const_uint(self.type_i32(), i as u64)
}

30
compiler/rustc_codegen_ssa/src/mir/rvalue.rs
View File

@ -7,6 +7,7 @@
use crate::traits::*;
use crate::MemFlags;
use rustc_hir as hir;
use rustc_middle::mir;
use rustc_middle::mir::Operand;
use rustc_middle::ty::cast::{CastTy, IntTy};
@ -882,6 +883,35 @@ pub fn codegen_scalar_binop(
bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs)
}
}
mir::BinOp::Cmp => {
use std::cmp::Ordering;
debug_assert!(!is_float);
let pred = |op| base::bin_op_to_icmp_predicate(op, is_signed);
if bx.cx().tcx().sess.opts.optimize == OptLevel::No {
// FIXME: This actually generates tighter assembly, and is a classic trick
// <https://graphics.stanford.edu/~seander/bithacks.html#CopyIntegerSign>
// However, as of 2023-11 it optimizes worse in things like derived
// `PartialOrd`, so only use it in debug for now. Once LLVM can handle it
// better (see <https://github.com/llvm/llvm-project/issues/73417>), it'll
// be worth trying it in optimized builds as well.
let is_gt = bx.icmp(pred(hir::BinOpKind::Gt), lhs, rhs);
let gtext = bx.zext(is_gt, bx.type_i8());
let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs);
let ltext = bx.zext(is_lt, bx.type_i8());
bx.unchecked_ssub(gtext, ltext)
} else {
// These operations are those expected by `tests/codegen/integer-cmp.rs`,
// from <https://github.com/rust-lang/rust/pull/63767>.
let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs);
let is_ne = bx.icmp(pred(hir::BinOpKind::Ne), lhs, rhs);
let ge = bx.select(
is_ne,
bx.cx().const_i8(Ordering::Greater as i8),
bx.cx().const_i8(Ordering::Equal as i8),
);
bx.select(is_lt, bx.cx().const_i8(Ordering::Less as i8), ge)
}
}
}
}

1
compiler/rustc_codegen_ssa/src/traits/consts.rs
View File

@ -19,6 +19,7 @@ pub trait ConstMethods<'tcx>: BackendTypes {
fn const_bool(&self, val: bool) -> Self::Value;
fn const_i16(&self, i: i16) -> Self::Value;
fn const_i32(&self, i: i32) -> Self::Value;
fn const_i8(&self, i: i8) -> Self::Value;
fn const_u32(&self, i: u32) -> Self::Value;
fn const_u64(&self, i: u64) -> Self::Value;
fn const_u128(&self, i: u128) -> Self::Value;

7
compiler/rustc_const_eval/src/interpret/operand.rs
View File

@ -235,6 +235,13 @@ pub fn from_bool(b: bool, tcx: TyCtxt<'tcx>) -> Self {
Self::from_scalar(Scalar::from_bool(b), layout)
}
#[inline]
pub fn from_ordering(c: std::cmp::Ordering, tcx: TyCtxt<'tcx>) -> Self {
let ty = tcx.ty_ordering_enum(None);
let layout = tcx.layout_of(ty::ParamEnv::reveal_all().and(ty)).unwrap();
Self::from_scalar(Scalar::from_i8(c as i8), layout)
}
#[inline]
pub fn to_const_int(self) -> ConstInt {
assert!(self.layout.ty.is_integral());

18
compiler/rustc_const_eval/src/interpret/operator.rs
View File

@ -61,6 +61,11 @@ pub fn binop_ignore_overflow(
}
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
fn three_way_compare<T: Ord>(&self, lhs: T, rhs: T) -> (ImmTy<'tcx, M::Provenance>, bool) {
let res = Ord::cmp(&lhs, &rhs);
return (ImmTy::from_ordering(res, *self.tcx), false);
}
fn binary_char_op(
&self,
bin_op: mir::BinOp,
@ -69,6 +74,10 @@ fn binary_char_op(
) -> (ImmTy<'tcx, M::Provenance>, bool) {
use rustc_middle::mir::BinOp::*;
if bin_op == Cmp {
return self.three_way_compare(l, r);
}
let res = match bin_op {
Eq => l == r,
Ne => l != r,
@ -231,6 +240,11 @@ fn binary_int_op(
let r = self.sign_extend(r, right_layout) as i128;
return Ok((ImmTy::from_bool(op(&l, &r), *self.tcx), false));
}
if bin_op == Cmp {
let l = self.sign_extend(l, left_layout) as i128;
let r = self.sign_extend(r, right_layout) as i128;
return Ok(self.three_way_compare(l, r));
}
let op: Option<fn(i128, i128) -> (i128, bool)> = match bin_op {
Div if r == 0 => throw_ub!(DivisionByZero),
Rem if r == 0 => throw_ub!(RemainderByZero),
@ -270,6 +284,10 @@ fn binary_int_op(
}
}
if bin_op == Cmp {
return Ok(self.three_way_compare(l, r));
}
let val = match bin_op {
Eq => ImmTy::from_bool(l == r, *self.tcx),
Ne => ImmTy::from_bool(l != r, *self.tcx),

9
compiler/rustc_const_eval/src/transform/validate.rs
View File

@ -986,6 +986,15 @@ macro_rules! check_kinds {
)
}
}
Cmp => {
for x in [a, b] {
check_kinds!(
x,
"Cannot three-way compare non-integer type {:?}",
ty::Char | ty::Uint(..) | ty::Int(..)
)
}
}
AddUnchecked | SubUnchecked | MulUnchecked | Shl | ShlUnchecked | Shr
| ShrUnchecked => {
for x in [a, b] {

4
compiler/rustc_const_eval/src/util/mod.rs
View File

@ -19,7 +19,7 @@ pub fn binop_left_homogeneous(op: mir::BinOp) -> bool {
match op {
Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
| BitAnd | BitOr | Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => true,
Eq | Ne | Lt | Le | Gt | Ge => false,
Eq | Ne | Lt | Le | Gt | Ge | Cmp => false,
}
}
@ -30,7 +30,7 @@ pub fn binop_right_homogeneous(op: mir::BinOp) -> bool {
use rustc_middle::mir::BinOp::*;
match op {
Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
| BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge => true,
| BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge | Cmp => true,
Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => false,
}
}

1
compiler/rustc_hir/src/lang_items.rs
View File

@ -226,6 +226,7 @@ pub fn extract(attrs: &[ast::Attribute]) -> Option<(Symbol, Span)> {
Unpin, sym::unpin, unpin_trait, Target::Trait, GenericRequirement::None;
Pin, sym::pin, pin_type, Target::Struct, GenericRequirement::None;
OrderingEnum, sym::Ordering, ordering_enum, Target::Enum, GenericRequirement::Exact(0);
PartialEq, sym::eq, eq_trait, Target::Trait, GenericRequirement::Exact(1);
PartialOrd, sym::partial_ord, partial_ord_trait, Target::Trait, GenericRequirement::Exact(1);
CVoid, sym::c_void, c_void, Target::Enum, GenericRequirement::None;

5
compiler/rustc_hir_analysis/src/check/intrinsic.rs
View File

@ -107,6 +107,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
| sym::cttz
| sym::bswap
| sym::bitreverse
| sym::three_way_compare
| sym::discriminant_value
| sym::type_id
| sym::likely
@ -418,6 +419,10 @@ pub fn check_intrinsic_type(
| sym::bswap
| sym::bitreverse => (1, 0, vec![param(0)], param(0)),
sym::three_way_compare => {
(1, 0, vec![param(0), param(0)], tcx.ty_ordering_enum(Some(span)))
}
sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
(1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
}

10
compiler/rustc_middle/src/mir/syntax.rs
View File

@ -1438,6 +1438,16 @@ pub enum BinOp {
Ge,
/// The `>` operator (greater than)
Gt,
/// The `<=>` operator (three-way comparison, like `Ord::cmp`)
///
/// This is supported only on the integer types and `char`, always returning
/// [`rustc_hir::LangItem::OrderingEnum`] (aka [`std::cmp::Ordering`]).
///
/// [`Rvalue::BinaryOp`]`(BinOp::Cmp, A, B)` returns
/// - `Ordering::Less` (`-1_i8`, as a Scalar) if `A < B`
/// - `Ordering::Equal` (`0_i8`, as a Scalar) if `A == B`
/// - `Ordering::Greater` (`+1_i8`, as a Scalar) if `A > B`
Cmp,
/// The `ptr.offset` operator
Offset,
}

8
compiler/rustc_middle/src/mir/tcx.rs
View File

@ -276,6 +276,11 @@ pub fn ty(&self, tcx: TyCtxt<'tcx>, lhs_ty: Ty<'tcx>, rhs_ty: Ty<'tcx>) -> Ty<'t
&BinOp::Eq | &BinOp::Lt | &BinOp::Le | &BinOp::Ne | &BinOp::Ge | &BinOp::Gt => {
tcx.types.bool
}
&BinOp::Cmp => {
// these should be integer-like types of the same size.
assert_eq!(lhs_ty, rhs_ty);
tcx.ty_ordering_enum(None)
}
}
}
}
@ -312,7 +317,8 @@ pub fn to_hir_binop(self) -> hir::BinOpKind {
BinOp::Gt => hir::BinOpKind::Gt,
BinOp::Le => hir::BinOpKind::Le,
BinOp::Ge => hir::BinOpKind::Ge,
BinOp::AddUnchecked
BinOp::Cmp
| BinOp::AddUnchecked
| BinOp::SubUnchecked
| BinOp::MulUnchecked
| BinOp::ShlUnchecked

7
compiler/rustc_middle/src/ty/context.rs
View File

@ -956,6 +956,13 @@ pub fn lang_items(self) -> &'tcx rustc_hir::lang_items::LanguageItems {
self.get_lang_items(())
}
/// Gets a `Ty` representing the [`LangItem::OrderingEnum`]
#[track_caller]
pub fn ty_ordering_enum(self, span: Option<Span>) -> Ty<'tcx> {
let ordering_enum = self.require_lang_item(hir::LangItem::OrderingEnum, span);
self.type_of(ordering_enum).no_bound_vars().unwrap()
}
/// Obtain the given diagnostic item's `DefId`. Use `is_diagnostic_item` if you just want to
/// compare against another `DefId`, since `is_diagnostic_item` is cheaper.
pub fn get_diagnostic_item(self, name: Symbol) -> Option<DefId> {

2
compiler/rustc_mir_transform/src/lower_intrinsics.rs
View File

@ -90,6 +90,7 @@ fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
sym::wrapping_add
| sym::wrapping_sub
| sym::wrapping_mul
| sym::three_way_compare
| sym::unchecked_add
| sym::unchecked_sub
| sym::unchecked_mul
@ -109,6 +110,7 @@ fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
sym::wrapping_add => BinOp::Add,
sym::wrapping_sub => BinOp::Sub,
sym::wrapping_mul => BinOp::Mul,
sym::three_way_compare => BinOp::Cmp,
sym::unchecked_add => BinOp::AddUnchecked,
sym::unchecked_sub => BinOp::SubUnchecked,
sym::unchecked_mul => BinOp::MulUnchecked,

1
compiler/rustc_mir_transform/src/promote_consts.rs
View File

@ -525,6 +525,7 @@ fn validate_rvalue(&mut self, rvalue: &Rvalue<'tcx>) -> Result<(), Unpromotable>
| BinOp::Lt
| BinOp::Ge
| BinOp::Gt
| BinOp::Cmp
| BinOp::Offset
| BinOp::Add
| BinOp::AddUnchecked

1
compiler/rustc_smir/src/rustc_smir/convert/mir.rs
View File

@ -493,6 +493,7 @@ fn stable(&self, _: &mut Tables<'_>) -> Self::T {
BinOp::Ne => stable_mir::mir::BinOp::Ne,
BinOp::Ge => stable_mir::mir::BinOp::Ge,
BinOp::Gt => stable_mir::mir::BinOp::Gt,
BinOp::Cmp => stable_mir::mir::BinOp::Cmp,
BinOp::Offset => stable_mir::mir::BinOp::Offset,
}
}

1
compiler/rustc_span/src/symbol.rs
View File

@ -1813,6 +1813,7 @@
thread,
thread_local,
thread_local_macro,
three_way_compare,
thumb2,
thumb_mode: "thumb-mode",
tmm_reg,

2
compiler/rustc_ty_utils/src/consts.rs
View File

@ -82,7 +82,7 @@ fn check_binop(op: mir::BinOp) -> bool {
match op {
Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
| BitAnd | BitOr | Shl | ShlUnchecked | Shr | ShrUnchecked | Eq | Lt | Le | Ne | Ge
| Gt => true,
| Gt | Cmp => true,
Offset => false,
}
}

4
compiler/stable_mir/src/mir/body.rs
View File

@ -329,6 +329,7 @@ pub enum BinOp {
Ne,
Ge,
Gt,
Cmp,
Offset,
}
@ -368,6 +369,9 @@ pub fn ty(&self, lhs_ty: Ty, rhs_ty: Ty) -> Ty {
assert!(lhs_kind.is_primitive() || lhs_kind.is_raw_ptr() || lhs_kind.is_fn_ptr());
Ty::bool_ty()
}
BinOp::Cmp => {
unimplemented!("Should cmp::Ordering be a RigidTy?");
}
}
}
}

30
library/core/src/cmp.rs
View File

@ -376,6 +376,10 @@ pub struct AssertParamIsEq<T: Eq + ?Sized> {
/// ```
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
#[stable(feature = "rust1", since = "1.0.0")]
// This is a lang item only so that `BinOp::Cmp` in MIR can return it.
// It has no special behaviour, but does require that the three variants
// `Less`/`Equal`/`Greater` remain `-1_i8`/`0_i8`/`+1_i8` respectively.
#[cfg_attr(not(bootstrap), lang = "Ordering")]
#[repr(i8)]
pub enum Ordering {
/// An ordering where a compared value is less than another.
@ -1554,7 +1558,14 @@ macro_rules! ord_impl {
impl PartialOrd for $t {
#[inline]
fn partial_cmp(&self, other: &$t) -> Option<Ordering> {
Some(self.cmp(other))
#[cfg(bootstrap)]
{
Some(self.cmp(other))
}
#[cfg(not(bootstrap))]
{
Some(crate::intrinsics::three_way_compare(*self, *other))
}
}
#[inline(always)]
fn lt(&self, other: &$t) -> bool { (*self) < (*other) }
@ -1570,11 +1581,18 @@ fn gt(&self, other: &$t) -> bool { (*self) > (*other) }
impl Ord for $t {
#[inline]
fn cmp(&self, other: &$t) -> Ordering {
// The order here is important to generate more optimal assembly.
// See <https://github.com/rust-lang/rust/issues/63758> for more info.
if *self < *other { Less }
else if *self == *other { Equal }
else { Greater }
#[cfg(bootstrap)]
{
// The order here is important to generate more optimal assembly.
// See <https://github.com/rust-lang/rust/issues/63758> for more info.
if *self < *other { Less }
else if *self == *other { Equal }
else { Greater }
}
#[cfg(not(bootstrap))]
{
crate::intrinsics::three_way_compare(*self, *other)
}
}
}
)*)

12
library/core/src/intrinsics.rs
View File

@ -2146,6 +2146,18 @@ pub const fn unlikely(b: bool) -> bool {
#[rustc_nounwind]
pub fn bitreverse<T: Copy>(x: T) -> T;
/// Does a three-way comparison between the two integer arguments.
///
/// This is included as an intrinsic as it's useful to let it be one thing
/// in MIR, rather than the multiple checks and switches that make its IR
/// large and difficult to optimize.
///
/// The stabilized version of this intrinsic is [`Ord::cmp`].
#[cfg(not(bootstrap))]
#[rustc_const_unstable(feature = "const_three_way_compare", issue = "none")]
#[rustc_safe_intrinsic]
pub fn three_way_compare<T: Copy>(lhs: T, rhs: T) -> crate::cmp::Ordering;
/// Performs checked integer addition.
///
/// Note that, unlike most intrinsics, this is safe to call;

27
library/core/tests/intrinsics.rs
View File

@ -99,3 +99,30 @@ fn test_const_deallocate_at_runtime() {
const_deallocate(core::ptr::null_mut(), 1, 1); // nop
}
}
#[cfg(not(bootstrap))]
#[test]
fn test_three_way_compare_in_const_contexts() {
use core::cmp::Ordering::{self, *};
use core::intrinsics::three_way_compare;
const UNSIGNED_LESS: Ordering = three_way_compare(123_u16, 456);
const UNSIGNED_EQUAL: Ordering = three_way_compare(456_u16, 456);
const UNSIGNED_GREATER: Ordering = three_way_compare(789_u16, 456);
const CHAR_LESS: Ordering = three_way_compare('A', 'B');
const CHAR_EQUAL: Ordering = three_way_compare('B', 'B');
const CHAR_GREATER: Ordering = three_way_compare('C', 'B');
const SIGNED_LESS: Ordering = three_way_compare(123_i64, 456);
const SIGNED_EQUAL: Ordering = three_way_compare(456_i64, 456);
const SIGNED_GREATER: Ordering = three_way_compare(789_i64, 456);
assert_eq!(UNSIGNED_LESS, Less);
assert_eq!(UNSIGNED_EQUAL, Equal);
assert_eq!(UNSIGNED_GREATER, Greater);
assert_eq!(CHAR_LESS, Less);
assert_eq!(CHAR_EQUAL, Equal);
assert_eq!(CHAR_GREATER, Greater);
assert_eq!(SIGNED_LESS, Less);
assert_eq!(SIGNED_EQUAL, Equal);
assert_eq!(SIGNED_GREATER, Greater);
}

1
library/core/tests/lib.rs
View File

@ -22,6 +22,7 @@
#![feature(const_pointer_is_aligned)]
#![feature(const_ptr_as_ref)]
#![feature(const_ptr_write)]
#![cfg_attr(not(bootstrap), feature(const_three_way_compare))]
#![feature(const_trait_impl)]
#![feature(const_likely)]
#![feature(const_location_fields)]

51
tests/assembly/x86_64-cmp.rs Normal file
View File

@ -0,0 +1,51 @@
//@ revisions: DEBUG OPTIM
//@ [DEBUG] compile-flags: -C opt-level=0
//@ [OPTIM] compile-flags: -C opt-level=3
//@ assembly-output: emit-asm
//@ compile-flags: --crate-type=lib -C llvm-args=-x86-asm-syntax=intel
//@ only-x86_64
//@ ignore-sgx
#![feature(core_intrinsics)]
use std::intrinsics::three_way_compare;
#[no_mangle]
// CHECK-LABEL: signed_cmp:
pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
// DEBUG: cmp
// DEBUG: setg
// DEBUG: and
// DEBUG: cmp
// DEBUG: setl
// DEBUG: and
// DEBUG: sub
// OPTIM: xor
// OPTIM: cmp
// OPTIM: setne
// OPTIM: mov
// OPTIM: cmovge
// OPTIM: ret
three_way_compare(a, b)
}
#[no_mangle]
// CHECK-LABEL: unsigned_cmp:
pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
// DEBUG: cmp
// DEBUG: seta
// DEBUG: and
// DEBUG: cmp
// DEBUG: setb
// DEBUG: and
// DEBUG: sub
// OPTIM: xor
// OPTIM: cmp
// OPTIM: setne
// OPTIM: mov
// OPTIM: cmovae
// OPTIM: ret
three_way_compare(a, b)
}

47
tests/codegen/intrinsics/three_way_compare.rs Normal file
View File

@ -0,0 +1,47 @@
//@ revisions: DEBUG OPTIM
//@ [DEBUG] compile-flags: -C opt-level=0
//@ [OPTIM] compile-flags: -C opt-level=3
//@ compile-flags: -C no-prepopulate-passes
#![crate_type = "lib"]
#![feature(core_intrinsics)]
use std::intrinsics::three_way_compare;
#[no_mangle]
// CHECK-LABEL: @signed_cmp
// DEBUG-SAME: (i16 %a, i16 %b)
// OPTIM-SAME: (i16 noundef %a, i16 noundef %b)
pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
// DEBUG: %[[GT:.+]] = icmp sgt i16 %a, %b
// DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
// DEBUG: %[[LT:.+]] = icmp slt i16 %a, %b
// DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
// DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
// OPTIM: %[[LT:.+]] = icmp slt i16 %a, %b
// OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
// OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
// OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
// OPTIM: ret i8 %[[CGEL]]
three_way_compare(a, b)
}
#[no_mangle]
// CHECK-LABEL: @unsigned_cmp
// DEBUG-SAME: (i16 %a, i16 %b)
// OPTIM-SAME: (i16 noundef %a, i16 noundef %b)
pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
// DEBUG: %[[GT:.+]] = icmp ugt i16 %a, %b
// DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
// DEBUG: %[[LT:.+]] = icmp ult i16 %a, %b
// DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
// DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
// OPTIM: %[[LT:.+]] = icmp ult i16 %a, %b
// OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
// OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
// OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
// OPTIM: ret i8 %[[CGEL]]
three_way_compare(a, b)
}

15
tests/mir-opt/lower_intrinsics.rs
View File

@ -229,3 +229,18 @@ pub unsafe fn ptr_offset(p: *const i32, d: isize) -> *const i32 {
core::intrinsics::offset(p, d)
}
// EMIT_MIR lower_intrinsics.three_way_compare_char.LowerIntrinsics.diff
pub fn three_way_compare_char(a: char, b: char) {
let _x = core::intrinsics::three_way_compare(a, b);
}
// EMIT_MIR lower_intrinsics.three_way_compare_signed.LowerIntrinsics.diff
pub fn three_way_compare_signed(a: i16, b: i16) {
core::intrinsics::three_way_compare(a, b);
}
// EMIT_MIR lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.diff
pub fn three_way_compare_unsigned(a: u32, b: u32) {
let _x = core::intrinsics::three_way_compare(a, b);
}

34
tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-abort.diff Normal file
View File

@ -0,0 +1,34 @@
- // MIR for `three_way_compare_char` before LowerIntrinsics
+ // MIR for `three_way_compare_char` after LowerIntrinsics
fn three_way_compare_char(_1: char, _2: char) -> () {
debug a => _1;
debug b => _2;
let mut _0: ();
let _3: std::cmp::Ordering;
let mut _4: char;
let mut _5: char;
scope 1 {
debug _x => _3;
}
bb0: {
StorageLive(_3);
StorageLive(_4);
_4 = _1;
StorageLive(_5);
_5 = _2;
- _3 = three_way_compare::<char>(move _4, move _5) -> [return: bb1, unwind unreachable];
+ _3 = Cmp(move _4, move _5);
+ goto -> bb1;
}
bb1: {
StorageDead(_5);
StorageDead(_4);
_0 = const ();
StorageDead(_3);
return;
}
}

34
tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-unwind.diff Normal file
View File

@ -0,0 +1,34 @@
- // MIR for `three_way_compare_char` before LowerIntrinsics
+ // MIR for `three_way_compare_char` after LowerIntrinsics
fn three_way_compare_char(_1: char, _2: char) -> () {
debug a => _1;
debug b => _2;
let mut _0: ();
let _3: std::cmp::Ordering;
let mut _4: char;
let mut _5: char;
scope 1 {
debug _x => _3;
}
bb0: {
StorageLive(_3);
StorageLive(_4);
_4 = _1;
StorageLive(_5);
_5 = _2;
- _3 = three_way_compare::<char>(move _4, move _5) -> [return: bb1, unwind continue];
+ _3 = Cmp(move _4, move _5);
+ goto -> bb1;
}
bb1: {
StorageDead(_5);
StorageDead(_4);
_0 = const ();
StorageDead(_3);
return;
}
}

31
tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-abort.diff Normal file
View File

@ -0,0 +1,31 @@
- // MIR for `three_way_compare_signed` before LowerIntrinsics
+ // MIR for `three_way_compare_signed` after LowerIntrinsics
fn three_way_compare_signed(_1: i16, _2: i16) -> () {
debug a => _1;
debug b => _2;
let mut _0: ();
let _3: std::cmp::Ordering;
let mut _4: i16;
let mut _5: i16;
bb0: {
StorageLive(_3);
StorageLive(_4);
_4 = _1;
StorageLive(_5);
_5 = _2;
- _3 = three_way_compare::<i16>(move _4, move _5) -> [return: bb1, unwind unreachable];
+ _3 = Cmp(move _4, move _5);
+ goto -> bb1;
}
bb1: {
StorageDead(_5);
StorageDead(_4);
StorageDead(_3);
_0 = const ();
return;
}
}

31
tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-unwind.diff Normal file
View File

@ -0,0 +1,31 @@
- // MIR for `three_way_compare_signed` before LowerIntrinsics
+ // MIR for `three_way_compare_signed` after LowerIntrinsics
fn three_way_compare_signed(_1: i16, _2: i16) -> () {
debug a => _1;
debug b => _2;
let mut _0: ();
let _3: std::cmp::Ordering;
let mut _4: i16;
let mut _5: i16;
bb0: {
StorageLive(_3);
StorageLive(_4);
_4 = _1;
StorageLive(_5);
_5 = _2;
- _3 = three_way_compare::<i16>(move _4, move _5) -> [return: bb1, unwind continue];
+ _3 = Cmp(move _4, move _5);
+ goto -> bb1;
}
bb1: {
StorageDead(_5);
StorageDead(_4);
StorageDead(_3);
_0 = const ();
return;
}
}

34
tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-abort.diff Normal file
View File

@ -0,0 +1,34 @@
- // MIR for `three_way_compare_unsigned` before LowerIntrinsics
+ // MIR for `three_way_compare_unsigned` after LowerIntrinsics
fn three_way_compare_unsigned(_1: u32, _2: u32) -> () {
debug a => _1;
debug b => _2;
let mut _0: ();
let _3: std::cmp::Ordering;
let mut _4: u32;
let mut _5: u32;
scope 1 {
debug _x => _3;
}
bb0: {
StorageLive(_3);
StorageLive(_4);
_4 = _1;
StorageLive(_5);
_5 = _2;
- _3 = three_way_compare::<u32>(move _4, move _5) -> [return: bb1, unwind unreachable];
+ _3 = Cmp(move _4, move _5);
+ goto -> bb1;
}
bb1: {
StorageDead(_5);
StorageDead(_4);
_0 = const ();
StorageDead(_3);
return;
}
}

34
tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-unwind.diff Normal file
View File

@ -0,0 +1,34 @@
- // MIR for `three_way_compare_unsigned` before LowerIntrinsics
+ // MIR for `three_way_compare_unsigned` after LowerIntrinsics
fn three_way_compare_unsigned(_1: u32, _2: u32) -> () {
debug a => _1;
debug b => _2;
let mut _0: ();
let _3: std::cmp::Ordering;
let mut _4: u32;
let mut _5: u32;
scope 1 {
debug _x => _3;
}
bb0: {
StorageLive(_3);
StorageLive(_4);
_4 = _1;
StorageLive(_5);
_5 = _2;
- _3 = three_way_compare::<u32>(move _4, move _5) -> [return: bb1, unwind continue];
+ _3 = Cmp(move _4, move _5);
+ goto -> bb1;
}
bb1: {
StorageDead(_5);
StorageDead(_4);
_0 = const ();
StorageDead(_3);
return;
}
}

9
tests/mir-opt/pre-codegen/derived_ord.rs Normal file
View File

@ -0,0 +1,9 @@
// skip-filecheck
//@ compile-flags: -O -Zmir-opt-level=2 -Cdebuginfo=0
#![crate_type = "lib"]
#[derive(PartialOrd, PartialEq)]
pub struct MultiField(char, i16);
// EMIT_MIR derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir

78
tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir Normal file
View File

@ -0,0 +1,78 @@
// MIR for `<impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp` after PreCodegen
fn <impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp(_1: &MultiField, _2: &MultiField) -> Option<std::cmp::Ordering> {
debug self => _1;
debug other => _2;
let mut _0: std::option::Option<std::cmp::Ordering>;
let mut _3: &char;
let mut _4: &char;
let mut _8: std::option::Option<std::cmp::Ordering>;
let mut _9: i8;
let mut _10: &i16;
let mut _11: &i16;
scope 1 {
debug cmp => _8;
}
scope 2 (inlined std::cmp::impls::<impl PartialOrd for char>::partial_cmp) {
debug self => _3;
debug other => _4;
let mut _5: char;
let mut _6: char;
let mut _7: std::cmp::Ordering;
}
scope 3 (inlined std::cmp::impls::<impl PartialOrd for i16>::partial_cmp) {
debug self => _10;
debug other => _11;
let mut _12: i16;
let mut _13: i16;
let mut _14: std::cmp::Ordering;
}
bb0: {
StorageLive(_3);
_3 = &((*_1).0: char);
StorageLive(_4);
_4 = &((*_2).0: char);
StorageLive(_5);
_5 = ((*_1).0: char);
StorageLive(_6);
_6 = ((*_2).0: char);
_7 = Cmp(move _5, move _6);
StorageDead(_6);
StorageDead(_5);
_8 = Option::<std::cmp::Ordering>::Some(_7);
StorageDead(_4);
StorageDead(_3);
_9 = discriminant(_7);
switchInt(move _9) -> [0: bb1, otherwise: bb2];
}
bb1: {
StorageLive(_10);
_10 = &((*_1).1: i16);
StorageLive(_11);
_11 = &((*_2).1: i16);
StorageLive(_14);
StorageLive(_12);
_12 = ((*_1).1: i16);
StorageLive(_13);
_13 = ((*_2).1: i16);
_14 = Cmp(move _12, move _13);
StorageDead(_13);
StorageDead(_12);
_0 = Option::<std::cmp::Ordering>::Some(move _14);
StorageDead(_14);
StorageDead(_11);
StorageDead(_10);
goto -> bb3;
}
bb2: {
_0 = _8;
goto -> bb3;
}
bb3: {
return;
}
}