Auto merge of #118310 - scottmcm:three-way-compare, r=davidtwco

Add `Ord::cmp` for primitives as a `BinOp` in MIR

Update: most of this OP was written months ago.  See https://github.com/rust-lang/rust/pull/118310#issuecomment-2016940014 below for where we got to recently that made it ready for review.

---

There are dozens of reasonable ways to implement `Ord::cmp` for integers using comparison, bit-ops, and branches.  Those differences are irrelevant at the rust level, however, so we can make things better by adding `BinOp::Cmp` at the MIR level:

1. Exactly how to implement it is left up to the backends, so LLVM can use whatever pattern its optimizer best recognizes and cranelift can use whichever pattern codegens the fastest.
2. By not inlining those details for every use of `cmp`, we drastically reduce the amount of MIR generated for `derive`d `PartialOrd`, while also making it more amenable to MIR-level optimizations.

Having extremely careful `if` ordering to μoptimize resource usage on broadwell (#63767) is great, but it really feels to me like libcore is the wrong place to put that logic.  Similarly, using subtraction [tricks](https://graphics.stanford.edu/~seander/bithacks.html#CopyIntegerSign) (#105840) is arguably even nicer, but depends on the optimizer understanding it (https://github.com/llvm/llvm-project/issues/73417) to be practical.  Or maybe [bitor is better than add](https://discourse.llvm.org/t/representing-in-ir/67369/2?u=scottmcm)?  But maybe only on a future version that [has `or disjoint` support](https://discourse.llvm.org/t/rfc-add-or-disjoint-flag/75036?u=scottmcm)?  And just because one of those forms happens to be good for LLVM, there's no guarantee that it'd be the same form that GCC or Cranelift would rather see -- especially given their very different optimizers.  Not to mention that if LLVM gets a spaceship intrinsic -- [which it should](https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/Suboptimal.20inlining.20in.20std.20function.20.60binary_search.60/near/404250586) -- we'll need at least a rustc intrinsic to be able to call it.

As for simplifying it in Rust, we now regularly inline `{integer}::partial_cmp`, but it's quite a large amount of IR.  The best way to see that is with 8811efa88b (diff-d134c32d028fbe2bf835fef2df9aca9d13332dd82284ff21ee7ebf717bfa4765R113) -- I added a new pre-codegen MIR test for a simple 3-tuple struct, and this PR change it from 36 locals and 26 basic blocks down to 24 locals and 8 basic blocks.  Even better, as soon as the construct-`Some`-then-match-it-in-same-BB noise is cleaned up, this'll expose the `Cmp == 0` branches clearly in MIR, so that an InstCombine (#105808) can simplify that to just a `BinOp::Eq` and thus fix some of our generated code perf issues.  (Tracking that through today's `if a < b { Less } else if a == b { Equal } else { Greater }` would be *much* harder.)

---

r? `@ghost`
But first I should check that perf is ok with this
~~...and my true nemesis, tidy.~~

compiler/rustc_codegen_cranelift/src/codegen_i128.rs

@@ -68,7 +68,7 @@ pub(crate) fn maybe_codegen<'tcx>(
                 Some(CValue::by_val(ret_val, lhs.layout()))
             }
         }
-        BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => None,
+        BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne | BinOp::Cmp => None,
         BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => None,
     }
 }
@@ -134,6 +134,7 @@ pub(crate) fn maybe_codegen_checked<'tcx>(
         BinOp::AddUnchecked | BinOp::SubUnchecked | BinOp::MulUnchecked => unreachable!(),
         BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
         BinOp::Div | BinOp::Rem => unreachable!(),
+        BinOp::Cmp => unreachable!(),
         BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => unreachable!(),
         BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => unreachable!(),
     }

compiler/rustc_codegen_cranelift/src/num.rs

@@ -40,6 +40,22 @@ pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
     })
 }
 
+fn codegen_three_way_compare<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    signed: bool,
+    lhs: Value,
+    rhs: Value,
+) -> CValue<'tcx> {
+    // This emits `(lhs > rhs) - (lhs < rhs)`, which is cranelift's preferred form per
+    // <https://github.com/bytecodealliance/wasmtime/blob/8052bb9e3b792503b225f2a5b2ba3bc023bff462/cranelift/codegen/src/prelude_opt.isle#L41-L47>
+    let gt_cc = crate::num::bin_op_to_intcc(BinOp::Gt, signed).unwrap();
+    let lt_cc = crate::num::bin_op_to_intcc(BinOp::Lt, signed).unwrap();
+    let gt = fx.bcx.ins().icmp(gt_cc, lhs, rhs);
+    let lt = fx.bcx.ins().icmp(lt_cc, lhs, rhs);
+    let val = fx.bcx.ins().isub(gt, lt);
+    CValue::by_val(val, fx.layout_of(fx.tcx.ty_ordering_enum(Some(fx.mir.span))))
+}
+
 fn codegen_compare_bin_op<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
@@ -47,6 +63,10 @@ fn codegen_compare_bin_op<'tcx>(
     lhs: Value,
     rhs: Value,
 ) -> CValue<'tcx> {
+    if bin_op == BinOp::Cmp {
+        return codegen_three_way_compare(fx, signed, lhs, rhs);
+    }
+
     let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap();
     let val = fx.bcx.ins().icmp(intcc, lhs, rhs);
     CValue::by_val(val, fx.layout_of(fx.tcx.types.bool))
@@ -59,7 +79,7 @@ pub(crate) fn codegen_binop<'tcx>(
     in_rhs: CValue<'tcx>,
 ) -> CValue<'tcx> {
     match bin_op {
-        BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
+        BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt | BinOp::Cmp => {
             match in_lhs.layout().ty.kind() {
                 ty::Bool | ty::Uint(_) | ty::Int(_) | ty::Char => {
                     let signed = type_sign(in_lhs.layout().ty);
@@ -160,7 +180,7 @@ pub(crate) fn codegen_int_binop<'tcx>(
         }
         BinOp::Offset => unreachable!("Offset is not an integer operation"),
         // Compare binops handles by `codegen_binop`.
-        BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge => {
+        BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge | BinOp::Cmp => {
             unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty);
         }
     };

compiler/rustc_codegen_gcc/src/common.rs

@@ -94,6 +94,10 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         self.const_int(self.type_i32(), i as i64)
     }
 
+    fn const_i8(&self, i: i8) -> RValue<'gcc> {
+        self.const_int(self.type_i8(), i as i64)
+    }
+
     fn const_u32(&self, i: u32) -> RValue<'gcc> {
         self.const_uint(self.type_u32(), i as u64)
     }

compiler/rustc_codegen_llvm/src/common.rs

@@ -160,6 +160,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         self.const_int(self.type_i32(), i as i64)
     }
 
+    fn const_i8(&self, i: i8) -> &'ll Value {
+        self.const_int(self.type_i8(), i as i64)
+    }
+
     fn const_u32(&self, i: u32) -> &'ll Value {
         self.const_uint(self.type_i32(), i as u64)
     }

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -7,6 +7,7 @@ use crate::common::{self, IntPredicate};
 use crate::traits::*;
 use crate::MemFlags;
 
+use rustc_hir as hir;
 use rustc_middle::mir;
 use rustc_middle::mir::Operand;
 use rustc_middle::ty::cast::{CastTy, IntTy};
@@ -882,6 +883,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs)
                 }
             }
+            mir::BinOp::Cmp => {
+                use std::cmp::Ordering;
+                debug_assert!(!is_float);
+                let pred = |op| base::bin_op_to_icmp_predicate(op, is_signed);
+                if bx.cx().tcx().sess.opts.optimize == OptLevel::No {
+                    // FIXME: This actually generates tighter assembly, and is a classic trick
+                    // <https://graphics.stanford.edu/~seander/bithacks.html#CopyIntegerSign>
+                    // However, as of 2023-11 it optimizes worse in things like derived
+                    // `PartialOrd`, so only use it in debug for now. Once LLVM can handle it
+                    // better (see <https://github.com/llvm/llvm-project/issues/73417>), it'll
+                    // be worth trying it in optimized builds as well.
+                    let is_gt = bx.icmp(pred(hir::BinOpKind::Gt), lhs, rhs);
+                    let gtext = bx.zext(is_gt, bx.type_i8());
+                    let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs);
+                    let ltext = bx.zext(is_lt, bx.type_i8());
+                    bx.unchecked_ssub(gtext, ltext)
+                } else {
+                    // These operations are those expected by `tests/codegen/integer-cmp.rs`,
+                    // from <https://github.com/rust-lang/rust/pull/63767>.
+                    let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs);
+                    let is_ne = bx.icmp(pred(hir::BinOpKind::Ne), lhs, rhs);
+                    let ge = bx.select(
+                        is_ne,
+                        bx.cx().const_i8(Ordering::Greater as i8),
+                        bx.cx().const_i8(Ordering::Equal as i8),
+                    );
+                    bx.select(is_lt, bx.cx().const_i8(Ordering::Less as i8), ge)
+                }
+            }
         }
     }
 

compiler/rustc_codegen_ssa/src/traits/consts.rs

@@ -19,6 +19,7 @@ pub trait ConstMethods<'tcx>: BackendTypes {
     fn const_bool(&self, val: bool) -> Self::Value;
     fn const_i16(&self, i: i16) -> Self::Value;
     fn const_i32(&self, i: i32) -> Self::Value;
+    fn const_i8(&self, i: i8) -> Self::Value;
     fn const_u32(&self, i: u32) -> Self::Value;
     fn const_u64(&self, i: u64) -> Self::Value;
     fn const_u128(&self, i: u128) -> Self::Value;

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -235,6 +235,13 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
         Self::from_scalar(Scalar::from_bool(b), layout)
     }
 
+    #[inline]
+    pub fn from_ordering(c: std::cmp::Ordering, tcx: TyCtxt<'tcx>) -> Self {
+        let ty = tcx.ty_ordering_enum(None);
+        let layout = tcx.layout_of(ty::ParamEnv::reveal_all().and(ty)).unwrap();
+        Self::from_scalar(Scalar::from_i8(c as i8), layout)
+    }
+
     #[inline]
     pub fn to_const_int(self) -> ConstInt {
         assert!(self.layout.ty.is_integral());

compiler/rustc_const_eval/src/interpret/operator.rs

@@ -61,6 +61,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 }
 
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+    fn three_way_compare<T: Ord>(&self, lhs: T, rhs: T) -> (ImmTy<'tcx, M::Provenance>, bool) {
+        let res = Ord::cmp(&lhs, &rhs);
+        return (ImmTy::from_ordering(res, *self.tcx), false);
+    }
+
     fn binary_char_op(
         &self,
         bin_op: mir::BinOp,
@@ -69,6 +74,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     ) -> (ImmTy<'tcx, M::Provenance>, bool) {
         use rustc_middle::mir::BinOp::*;
 
+        if bin_op == Cmp {
+            return self.three_way_compare(l, r);
+        }
+
         let res = match bin_op {
             Eq => l == r,
             Ne => l != r,
@@ -231,6 +240,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let r = self.sign_extend(r, right_layout) as i128;
                 return Ok((ImmTy::from_bool(op(&l, &r), *self.tcx), false));
             }
+            if bin_op == Cmp {
+                let l = self.sign_extend(l, left_layout) as i128;
+                let r = self.sign_extend(r, right_layout) as i128;
+                return Ok(self.three_way_compare(l, r));
+            }
             let op: Option<fn(i128, i128) -> (i128, bool)> = match bin_op {
                 Div if r == 0 => throw_ub!(DivisionByZero),
                 Rem if r == 0 => throw_ub!(RemainderByZero),
@@ -270,6 +284,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
         }
 
+        if bin_op == Cmp {
+            return Ok(self.three_way_compare(l, r));
+        }
+
         let val = match bin_op {
             Eq => ImmTy::from_bool(l == r, *self.tcx),
             Ne => ImmTy::from_bool(l != r, *self.tcx),

compiler/rustc_const_eval/src/transform/validate.rs

@@ -986,6 +986,15 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                             )
                         }
                     }
+                    Cmp => {
+                        for x in [a, b] {
+                            check_kinds!(
+                                x,
+                                "Cannot three-way compare non-integer type {:?}",
+                                ty::Char | ty::Uint(..) | ty::Int(..)
+                            )
+                        }
+                    }
                     AddUnchecked | SubUnchecked | MulUnchecked | Shl | ShlUnchecked | Shr
                     | ShrUnchecked => {
                         for x in [a, b] {

compiler/rustc_const_eval/src/util/mod.rs

@@ -19,7 +19,7 @@ pub fn binop_left_homogeneous(op: mir::BinOp) -> bool {
     match op {
         Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
         | BitAnd | BitOr | Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => true,
-        Eq | Ne | Lt | Le | Gt | Ge => false,
+        Eq | Ne | Lt | Le | Gt | Ge | Cmp => false,
     }
 }
 
@@ -30,7 +30,7 @@ pub fn binop_right_homogeneous(op: mir::BinOp) -> bool {
     use rustc_middle::mir::BinOp::*;
     match op {
         Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
-        | BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge => true,
+        | BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge | Cmp => true,
         Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => false,
     }
 }

compiler/rustc_hir/src/lang_items.rs

@@ -226,6 +226,7 @@ language_item_table! {
     Unpin,                   sym::unpin,               unpin_trait,                Target::Trait,          GenericRequirement::None;
     Pin,                     sym::pin,                 pin_type,                   Target::Struct,         GenericRequirement::None;
 
+    OrderingEnum,            sym::Ordering,            ordering_enum,              Target::Enum,           GenericRequirement::Exact(0);
     PartialEq,               sym::eq,                  eq_trait,                   Target::Trait,          GenericRequirement::Exact(1);
     PartialOrd,              sym::partial_ord,         partial_ord_trait,          Target::Trait,          GenericRequirement::Exact(1);
     CVoid,                   sym::c_void,              c_void,                     Target::Enum,           GenericRequirement::None;

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -107,6 +107,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
         | sym::cttz
         | sym::bswap
         | sym::bitreverse
+        | sym::three_way_compare
         | sym::discriminant_value
         | sym::type_id
         | sym::likely
@@ -418,6 +419,10 @@ pub fn check_intrinsic_type(
             | sym::bswap
             | sym::bitreverse => (1, 0, vec![param(0)], param(0)),
 
+            sym::three_way_compare => {
+                (1, 0, vec![param(0), param(0)], tcx.ty_ordering_enum(Some(span)))
+            }
+
             sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
                 (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
             }

compiler/rustc_middle/src/mir/syntax.rs

@@ -1438,6 +1438,16 @@ pub enum BinOp {
     Ge,
     /// The `>` operator (greater than)
     Gt,
+    /// The `<=>` operator (three-way comparison, like `Ord::cmp`)
+    ///
+    /// This is supported only on the integer types and `char`, always returning
+    /// [`rustc_hir::LangItem::OrderingEnum`] (aka [`std::cmp::Ordering`]).
+    ///
+    /// [`Rvalue::BinaryOp`]`(BinOp::Cmp, A, B)` returns
+    /// - `Ordering::Less` (`-1_i8`, as a Scalar) if `A < B`
+    /// - `Ordering::Equal` (`0_i8`, as a Scalar) if `A == B`
+    /// - `Ordering::Greater` (`+1_i8`, as a Scalar) if `A > B`
+    Cmp,
     /// The `ptr.offset` operator
     Offset,
 }

compiler/rustc_middle/src/mir/tcx.rs

@@ -276,6 +276,11 @@ impl<'tcx> BinOp {
             &BinOp::Eq | &BinOp::Lt | &BinOp::Le | &BinOp::Ne | &BinOp::Ge | &BinOp::Gt => {
                 tcx.types.bool
             }
+            &BinOp::Cmp => {
+                // these should be integer-like types of the same size.
+                assert_eq!(lhs_ty, rhs_ty);
+                tcx.ty_ordering_enum(None)
+            }
         }
     }
 }
@@ -312,7 +317,8 @@ impl BinOp {
             BinOp::Gt => hir::BinOpKind::Gt,
             BinOp::Le => hir::BinOpKind::Le,
             BinOp::Ge => hir::BinOpKind::Ge,
-            BinOp::AddUnchecked
+            BinOp::Cmp
+            | BinOp::AddUnchecked
             | BinOp::SubUnchecked
             | BinOp::MulUnchecked
             | BinOp::ShlUnchecked

compiler/rustc_middle/src/ty/context.rs

@@ -956,6 +956,13 @@ impl<'tcx> TyCtxt<'tcx> {
         self.get_lang_items(())
     }
 
+    /// Gets a `Ty` representing the [`LangItem::OrderingEnum`]
+    #[track_caller]
+    pub fn ty_ordering_enum(self, span: Option<Span>) -> Ty<'tcx> {
+        let ordering_enum = self.require_lang_item(hir::LangItem::OrderingEnum, span);
+        self.type_of(ordering_enum).no_bound_vars().unwrap()
+    }
+
     /// Obtain the given diagnostic item's `DefId`. Use `is_diagnostic_item` if you just want to
     /// compare against another `DefId`, since `is_diagnostic_item` is cheaper.
     pub fn get_diagnostic_item(self, name: Symbol) -> Option<DefId> {

compiler/rustc_mir_transform/src/lower_intrinsics.rs

@@ -90,6 +90,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                     sym::wrapping_add
                     | sym::wrapping_sub
                     | sym::wrapping_mul
+                    | sym::three_way_compare
                     | sym::unchecked_add
                     | sym::unchecked_sub
                     | sym::unchecked_mul
@@ -109,6 +110,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                             sym::wrapping_add => BinOp::Add,
                             sym::wrapping_sub => BinOp::Sub,
                             sym::wrapping_mul => BinOp::Mul,
+                            sym::three_way_compare => BinOp::Cmp,
                             sym::unchecked_add => BinOp::AddUnchecked,
                             sym::unchecked_sub => BinOp::SubUnchecked,
                             sym::unchecked_mul => BinOp::MulUnchecked,

compiler/rustc_mir_transform/src/promote_consts.rs

@@ -525,6 +525,7 @@ impl<'tcx> Validator<'_, 'tcx> {
                     | BinOp::Lt
                     | BinOp::Ge
                     | BinOp::Gt
+                    | BinOp::Cmp
                     | BinOp::Offset
                     | BinOp::Add
                     | BinOp::AddUnchecked

compiler/rustc_smir/src/rustc_smir/convert/mir.rs

@@ -493,6 +493,7 @@ impl<'tcx> Stable<'tcx> for mir::BinOp {
             BinOp::Ne => stable_mir::mir::BinOp::Ne,
             BinOp::Ge => stable_mir::mir::BinOp::Ge,
             BinOp::Gt => stable_mir::mir::BinOp::Gt,
+            BinOp::Cmp => stable_mir::mir::BinOp::Cmp,
             BinOp::Offset => stable_mir::mir::BinOp::Offset,
         }
     }

compiler/rustc_span/src/symbol.rs

@@ -1813,6 +1813,7 @@ symbols! {
         thread,
         thread_local,
         thread_local_macro,
+        three_way_compare,
         thumb2,
         thumb_mode: "thumb-mode",
         tmm_reg,

compiler/rustc_ty_utils/src/consts.rs

@@ -82,7 +82,7 @@ fn check_binop(op: mir::BinOp) -> bool {
     match op {
         Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
         | BitAnd | BitOr | Shl | ShlUnchecked | Shr | ShrUnchecked | Eq | Lt | Le | Ne | Ge
-        | Gt => true,
+        | Gt | Cmp => true,
         Offset => false,
     }
 }

compiler/stable_mir/src/mir/body.rs

@@ -329,6 +329,7 @@ pub enum BinOp {
     Ne,
     Ge,
     Gt,
+    Cmp,
     Offset,
 }
 
@@ -368,6 +369,9 @@ impl BinOp {
                 assert!(lhs_kind.is_primitive() || lhs_kind.is_raw_ptr() || lhs_kind.is_fn_ptr());
                 Ty::bool_ty()
             }
+            BinOp::Cmp => {
+                unimplemented!("Should cmp::Ordering be a RigidTy?");
+            }
         }
     }
 }

library/core/src/cmp.rs

@@ -376,6 +376,10 @@ pub struct AssertParamIsEq<T: Eq + ?Sized> {
 /// ```
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
 #[stable(feature = "rust1", since = "1.0.0")]
+// This is a lang item only so that `BinOp::Cmp` in MIR can return it.
+// It has no special behaviour, but does require that the three variants
+// `Less`/`Equal`/`Greater` remain `-1_i8`/`0_i8`/`+1_i8` respectively.
+#[cfg_attr(not(bootstrap), lang = "Ordering")]
 #[repr(i8)]
 pub enum Ordering {
     /// An ordering where a compared value is less than another.
@@ -1554,7 +1558,14 @@ mod impls {
             impl PartialOrd for $t {
                 #[inline]
                 fn partial_cmp(&self, other: &$t) -> Option<Ordering> {
-                    Some(self.cmp(other))
+                    #[cfg(bootstrap)]
+                    {
+                        Some(self.cmp(other))
+                    }
+                    #[cfg(not(bootstrap))]
+                    {
+                        Some(crate::intrinsics::three_way_compare(*self, *other))
+                    }
                 }
                 #[inline(always)]
                 fn lt(&self, other: &$t) -> bool { (*self) < (*other) }
@@ -1570,11 +1581,18 @@ mod impls {
             impl Ord for $t {
                 #[inline]
                 fn cmp(&self, other: &$t) -> Ordering {
-                    // The order here is important to generate more optimal assembly.
+                    #[cfg(bootstrap)]
-                    // See <https://github.com/rust-lang/rust/issues/63758> for more info.
+                    {
-                    if *self < *other { Less }
+                        // The order here is important to generate more optimal assembly.
-                    else if *self == *other { Equal }
+                        // See <https://github.com/rust-lang/rust/issues/63758> for more info.
-                    else { Greater }
+                        if *self < *other { Less }
+                        else if *self == *other { Equal }
+                        else { Greater }
+                    }
+                    #[cfg(not(bootstrap))]
+                    {
+                        crate::intrinsics::three_way_compare(*self, *other)
+                    }
                 }
             }
         )*)

library/core/src/intrinsics.rs

@@ -2146,6 +2146,18 @@ extern "rust-intrinsic" {
     #[rustc_nounwind]
     pub fn bitreverse<T: Copy>(x: T) -> T;
 
+    /// Does a three-way comparison between the two integer arguments.
+    ///
+    /// This is included as an intrinsic as it's useful to let it be one thing
+    /// in MIR, rather than the multiple checks and switches that make its IR
+    /// large and difficult to optimize.
+    ///
+    /// The stabilized version of this intrinsic is [`Ord::cmp`].
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_three_way_compare", issue = "none")]
+    #[rustc_safe_intrinsic]
+    pub fn three_way_compare<T: Copy>(lhs: T, rhs: T) -> crate::cmp::Ordering;
+
     /// Performs checked integer addition.
     ///
     /// Note that, unlike most intrinsics, this is safe to call;

library/core/tests/intrinsics.rs

@@ -99,3 +99,30 @@ fn test_const_deallocate_at_runtime() {
         const_deallocate(core::ptr::null_mut(), 1, 1); // nop
     }
 }
+
+#[cfg(not(bootstrap))]
+#[test]
+fn test_three_way_compare_in_const_contexts() {
+    use core::cmp::Ordering::{self, *};
+    use core::intrinsics::three_way_compare;
+
+    const UNSIGNED_LESS: Ordering = three_way_compare(123_u16, 456);
+    const UNSIGNED_EQUAL: Ordering = three_way_compare(456_u16, 456);
+    const UNSIGNED_GREATER: Ordering = three_way_compare(789_u16, 456);
+    const CHAR_LESS: Ordering = three_way_compare('A', 'B');
+    const CHAR_EQUAL: Ordering = three_way_compare('B', 'B');
+    const CHAR_GREATER: Ordering = three_way_compare('C', 'B');
+    const SIGNED_LESS: Ordering = three_way_compare(123_i64, 456);
+    const SIGNED_EQUAL: Ordering = three_way_compare(456_i64, 456);
+    const SIGNED_GREATER: Ordering = three_way_compare(789_i64, 456);
+
+    assert_eq!(UNSIGNED_LESS, Less);
+    assert_eq!(UNSIGNED_EQUAL, Equal);
+    assert_eq!(UNSIGNED_GREATER, Greater);
+    assert_eq!(CHAR_LESS, Less);
+    assert_eq!(CHAR_EQUAL, Equal);
+    assert_eq!(CHAR_GREATER, Greater);
+    assert_eq!(SIGNED_LESS, Less);
+    assert_eq!(SIGNED_EQUAL, Equal);
+    assert_eq!(SIGNED_GREATER, Greater);
+}

library/core/tests/lib.rs

@@ -22,6 +22,7 @@
 #![feature(const_pointer_is_aligned)]
 #![feature(const_ptr_as_ref)]
 #![feature(const_ptr_write)]
+#![cfg_attr(not(bootstrap), feature(const_three_way_compare))]
 #![feature(const_trait_impl)]
 #![feature(const_likely)]
 #![feature(const_location_fields)]

tests/assembly/x86_64-cmp.rs

@@ -0,0 +1,51 @@
+//@ revisions: DEBUG OPTIM
+//@ [DEBUG] compile-flags: -C opt-level=0
+//@ [OPTIM] compile-flags: -C opt-level=3
+//@ assembly-output: emit-asm
+//@ compile-flags: --crate-type=lib -C llvm-args=-x86-asm-syntax=intel
+//@ only-x86_64
+//@ ignore-sgx
+
+#![feature(core_intrinsics)]
+
+use std::intrinsics::three_way_compare;
+
+#[no_mangle]
+// CHECK-LABEL: signed_cmp:
+pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
+    // DEBUG: cmp
+    // DEBUG: setg
+    // DEBUG: and
+    // DEBUG: cmp
+    // DEBUG: setl
+    // DEBUG: and
+    // DEBUG: sub
+
+    // OPTIM: xor
+    // OPTIM: cmp
+    // OPTIM: setne
+    // OPTIM: mov
+    // OPTIM: cmovge
+    // OPTIM: ret
+    three_way_compare(a, b)
+}
+
+#[no_mangle]
+// CHECK-LABEL: unsigned_cmp:
+pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
+    // DEBUG: cmp
+    // DEBUG: seta
+    // DEBUG: and
+    // DEBUG: cmp
+    // DEBUG: setb
+    // DEBUG: and
+    // DEBUG: sub
+
+    // OPTIM: xor
+    // OPTIM: cmp
+    // OPTIM: setne
+    // OPTIM: mov
+    // OPTIM: cmovae
+    // OPTIM: ret
+    three_way_compare(a, b)
+}

tests/codegen/intrinsics/three_way_compare.rs

@@ -0,0 +1,47 @@
+//@ revisions: DEBUG OPTIM
+//@ [DEBUG] compile-flags: -C opt-level=0
+//@ [OPTIM] compile-flags: -C opt-level=3
+//@ compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::three_way_compare;
+
+#[no_mangle]
+// CHECK-LABEL: @signed_cmp
+// DEBUG-SAME: (i16 %a, i16 %b)
+// OPTIM-SAME: (i16 noundef %a, i16 noundef %b)
+pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
+    // DEBUG: %[[GT:.+]] = icmp sgt i16 %a, %b
+    // DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
+    // DEBUG: %[[LT:.+]] = icmp slt i16 %a, %b
+    // DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
+    // DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
+
+    // OPTIM: %[[LT:.+]] = icmp slt i16 %a, %b
+    // OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
+    // OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
+    // OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
+    // OPTIM: ret i8 %[[CGEL]]
+    three_way_compare(a, b)
+}
+
+#[no_mangle]
+// CHECK-LABEL: @unsigned_cmp
+// DEBUG-SAME: (i16 %a, i16 %b)
+// OPTIM-SAME: (i16 noundef %a, i16 noundef %b)
+pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
+    // DEBUG: %[[GT:.+]] = icmp ugt i16 %a, %b
+    // DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
+    // DEBUG: %[[LT:.+]] = icmp ult i16 %a, %b
+    // DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
+    // DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
+
+    // OPTIM: %[[LT:.+]] = icmp ult i16 %a, %b
+    // OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
+    // OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
+    // OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
+    // OPTIM: ret i8 %[[CGEL]]
+    three_way_compare(a, b)
+}

tests/mir-opt/lower_intrinsics.rs

@@ -229,3 +229,18 @@ pub unsafe fn ptr_offset(p: *const i32, d: isize) -> *const i32 {
 
     core::intrinsics::offset(p, d)
 }
+
+// EMIT_MIR lower_intrinsics.three_way_compare_char.LowerIntrinsics.diff
+pub fn three_way_compare_char(a: char, b: char) {
+    let _x = core::intrinsics::three_way_compare(a, b);
+}
+
+// EMIT_MIR lower_intrinsics.three_way_compare_signed.LowerIntrinsics.diff
+pub fn three_way_compare_signed(a: i16, b: i16) {
+    core::intrinsics::three_way_compare(a, b);
+}
+
+// EMIT_MIR lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.diff
+pub fn three_way_compare_unsigned(a: u32, b: u32) {
+    let _x = core::intrinsics::three_way_compare(a, b);
+}

tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-abort.diff

@@ -0,0 +1,34 @@
+- // MIR for `three_way_compare_char` before LowerIntrinsics
++ // MIR for `three_way_compare_char` after LowerIntrinsics
+  
+  fn three_way_compare_char(_1: char, _2: char) -> () {
+      debug a => _1;
+      debug b => _2;
+      let mut _0: ();
+      let _3: std::cmp::Ordering;
+      let mut _4: char;
+      let mut _5: char;
+      scope 1 {
+          debug _x => _3;
+      }
+  
+      bb0: {
+          StorageLive(_3);
+          StorageLive(_4);
+          _4 = _1;
+          StorageLive(_5);
+          _5 = _2;
+-         _3 = three_way_compare::<char>(move _4, move _5) -> [return: bb1, unwind unreachable];
++         _3 = Cmp(move _4, move _5);
++         goto -> bb1;
+      }
+  
+      bb1: {
+          StorageDead(_5);
+          StorageDead(_4);
+          _0 = const ();
+          StorageDead(_3);
+          return;
+      }
+  }
+  

tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-unwind.diff

@@ -0,0 +1,34 @@
+- // MIR for `three_way_compare_char` before LowerIntrinsics
++ // MIR for `three_way_compare_char` after LowerIntrinsics
+  
+  fn three_way_compare_char(_1: char, _2: char) -> () {
+      debug a => _1;
+      debug b => _2;
+      let mut _0: ();
+      let _3: std::cmp::Ordering;
+      let mut _4: char;
+      let mut _5: char;
+      scope 1 {
+          debug _x => _3;
+      }
+  
+      bb0: {
+          StorageLive(_3);
+          StorageLive(_4);
+          _4 = _1;
+          StorageLive(_5);
+          _5 = _2;
+-         _3 = three_way_compare::<char>(move _4, move _5) -> [return: bb1, unwind continue];
++         _3 = Cmp(move _4, move _5);
++         goto -> bb1;
+      }
+  
+      bb1: {
+          StorageDead(_5);
+          StorageDead(_4);
+          _0 = const ();
+          StorageDead(_3);
+          return;
+      }
+  }
+  

tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-abort.diff

@@ -0,0 +1,31 @@
+- // MIR for `three_way_compare_signed` before LowerIntrinsics
++ // MIR for `three_way_compare_signed` after LowerIntrinsics
+  
+  fn three_way_compare_signed(_1: i16, _2: i16) -> () {
+      debug a => _1;
+      debug b => _2;
+      let mut _0: ();
+      let _3: std::cmp::Ordering;
+      let mut _4: i16;
+      let mut _5: i16;
+  
+      bb0: {
+          StorageLive(_3);
+          StorageLive(_4);
+          _4 = _1;
+          StorageLive(_5);
+          _5 = _2;
+-         _3 = three_way_compare::<i16>(move _4, move _5) -> [return: bb1, unwind unreachable];
++         _3 = Cmp(move _4, move _5);
++         goto -> bb1;
+      }
+  
+      bb1: {
+          StorageDead(_5);
+          StorageDead(_4);
+          StorageDead(_3);
+          _0 = const ();
+          return;
+      }
+  }
+  

tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-unwind.diff

@@ -0,0 +1,31 @@
+- // MIR for `three_way_compare_signed` before LowerIntrinsics
++ // MIR for `three_way_compare_signed` after LowerIntrinsics
+  
+  fn three_way_compare_signed(_1: i16, _2: i16) -> () {
+      debug a => _1;
+      debug b => _2;
+      let mut _0: ();
+      let _3: std::cmp::Ordering;
+      let mut _4: i16;
+      let mut _5: i16;
+  
+      bb0: {
+          StorageLive(_3);
+          StorageLive(_4);
+          _4 = _1;
+          StorageLive(_5);
+          _5 = _2;
+-         _3 = three_way_compare::<i16>(move _4, move _5) -> [return: bb1, unwind continue];
++         _3 = Cmp(move _4, move _5);
++         goto -> bb1;
+      }
+  
+      bb1: {
+          StorageDead(_5);
+          StorageDead(_4);
+          StorageDead(_3);
+          _0 = const ();
+          return;
+      }
+  }
+  

tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-abort.diff

@@ -0,0 +1,34 @@
+- // MIR for `three_way_compare_unsigned` before LowerIntrinsics
++ // MIR for `three_way_compare_unsigned` after LowerIntrinsics
+  
+  fn three_way_compare_unsigned(_1: u32, _2: u32) -> () {
+      debug a => _1;
+      debug b => _2;
+      let mut _0: ();
+      let _3: std::cmp::Ordering;
+      let mut _4: u32;
+      let mut _5: u32;
+      scope 1 {
+          debug _x => _3;
+      }
+  
+      bb0: {
+          StorageLive(_3);
+          StorageLive(_4);
+          _4 = _1;
+          StorageLive(_5);
+          _5 = _2;
+-         _3 = three_way_compare::<u32>(move _4, move _5) -> [return: bb1, unwind unreachable];
++         _3 = Cmp(move _4, move _5);
++         goto -> bb1;
+      }
+  
+      bb1: {
+          StorageDead(_5);
+          StorageDead(_4);
+          _0 = const ();
+          StorageDead(_3);
+          return;
+      }
+  }
+  

tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-unwind.diff

@@ -0,0 +1,34 @@
+- // MIR for `three_way_compare_unsigned` before LowerIntrinsics
++ // MIR for `three_way_compare_unsigned` after LowerIntrinsics
+  
+  fn three_way_compare_unsigned(_1: u32, _2: u32) -> () {
+      debug a => _1;
+      debug b => _2;
+      let mut _0: ();
+      let _3: std::cmp::Ordering;
+      let mut _4: u32;
+      let mut _5: u32;
+      scope 1 {
+          debug _x => _3;
+      }
+  
+      bb0: {
+          StorageLive(_3);
+          StorageLive(_4);
+          _4 = _1;
+          StorageLive(_5);
+          _5 = _2;
+-         _3 = three_way_compare::<u32>(move _4, move _5) -> [return: bb1, unwind continue];
++         _3 = Cmp(move _4, move _5);
++         goto -> bb1;
+      }
+  
+      bb1: {
+          StorageDead(_5);
+          StorageDead(_4);
+          _0 = const ();
+          StorageDead(_3);
+          return;
+      }
+  }
+  

tests/mir-opt/pre-codegen/derived_ord.rs

@@ -0,0 +1,9 @@
+// skip-filecheck
+//@ compile-flags: -O -Zmir-opt-level=2 -Cdebuginfo=0
+
+#![crate_type = "lib"]
+
+#[derive(PartialOrd, PartialEq)]
+pub struct MultiField(char, i16);
+
+// EMIT_MIR derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir

tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir

@@ -0,0 +1,78 @@
+// MIR for `<impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp` after PreCodegen
+
+fn <impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp(_1: &MultiField, _2: &MultiField) -> Option<std::cmp::Ordering> {
+    debug self => _1;
+    debug other => _2;
+    let mut _0: std::option::Option<std::cmp::Ordering>;
+    let mut _3: &char;
+    let mut _4: &char;
+    let mut _8: std::option::Option<std::cmp::Ordering>;
+    let mut _9: i8;
+    let mut _10: &i16;
+    let mut _11: &i16;
+    scope 1 {
+        debug cmp => _8;
+    }
+    scope 2 (inlined std::cmp::impls::<impl PartialOrd for char>::partial_cmp) {
+        debug self => _3;
+        debug other => _4;
+        let mut _5: char;
+        let mut _6: char;
+        let mut _7: std::cmp::Ordering;
+    }
+    scope 3 (inlined std::cmp::impls::<impl PartialOrd for i16>::partial_cmp) {
+        debug self => _10;
+        debug other => _11;
+        let mut _12: i16;
+        let mut _13: i16;
+        let mut _14: std::cmp::Ordering;
+    }
+
+    bb0: {
+        StorageLive(_3);
+        _3 = &((*_1).0: char);
+        StorageLive(_4);
+        _4 = &((*_2).0: char);
+        StorageLive(_5);
+        _5 = ((*_1).0: char);
+        StorageLive(_6);
+        _6 = ((*_2).0: char);
+        _7 = Cmp(move _5, move _6);
+        StorageDead(_6);
+        StorageDead(_5);
+        _8 = Option::<std::cmp::Ordering>::Some(_7);
+        StorageDead(_4);
+        StorageDead(_3);
+        _9 = discriminant(_7);
+        switchInt(move _9) -> [0: bb1, otherwise: bb2];
+    }
+
+    bb1: {
+        StorageLive(_10);
+        _10 = &((*_1).1: i16);
+        StorageLive(_11);
+        _11 = &((*_2).1: i16);
+        StorageLive(_14);
+        StorageLive(_12);
+        _12 = ((*_1).1: i16);
+        StorageLive(_13);
+        _13 = ((*_2).1: i16);
+        _14 = Cmp(move _12, move _13);
+        StorageDead(_13);
+        StorageDead(_12);
+        _0 = Option::<std::cmp::Ordering>::Some(move _14);
+        StorageDead(_14);
+        StorageDead(_11);
+        StorageDead(_10);
+        goto -> bb3;
+    }
+
+    bb2: {
+        _0 = _8;
+        goto -> bb3;
+    }
+
+    bb3: {
+        return;
+    }
+}