Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 3da115a

Browse files
scottmcmscottmcm
committedMar 24, 2024
Add+Use mir::BinOp::Cmp 
1 parent 744c664 commit 3da115a

35 files changed

+514
-112
lines changed
 

‎compiler/rustc_codegen_cranelift/src/codegen_i128.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ pub(crate) fn maybe_codegen<'tcx>(
6868
Some(CValue::by_val(ret_val, lhs.layout()))
6969
}
7070
}
71-
BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => None,
71+
BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne | BinOp::Cmp => None,
7272
BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => None,
7373
}
7474
}
@@ -134,6 +134,7 @@ pub(crate) fn maybe_codegen_checked<'tcx>(
134134
BinOp::AddUnchecked | BinOp::SubUnchecked | BinOp::MulUnchecked => unreachable!(),
135135
BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
136136
BinOp::Div | BinOp::Rem => unreachable!(),
137+
BinOp::Cmp => unreachable!(),
137138
BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => unreachable!(),
138139
BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => unreachable!(),
139140
}

‎compiler/rustc_codegen_cranelift/src/num.rs

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,33 @@ pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
4040
})
4141
}
4242

43+
fn codegen_three_way_compare<'tcx>(
44+
fx: &mut FunctionCx<'_, '_, 'tcx>,
45+
signed: bool,
46+
lhs: Value,
47+
rhs: Value,
48+
) -> CValue<'tcx> {
49+
// This emits `(lhs > rhs) - (lhs < rhs)`, which is cranelift's preferred form per
50+
// <https://github.com/bytecodealliance/wasmtime/blob/8052bb9e3b792503b225f2a5b2ba3bc023bff462/cranelift/codegen/src/prelude_opt.isle#L41-L47>
51+
let gt_cc = crate::num::bin_op_to_intcc(BinOp::Gt, signed).unwrap();
52+
let lt_cc = crate::num::bin_op_to_intcc(BinOp::Lt, signed).unwrap();
53+
let gt = fx.bcx.ins().icmp(gt_cc, lhs, rhs);
54+
let lt = fx.bcx.ins().icmp(lt_cc, lhs, rhs);
55+
let val = fx.bcx.ins().isub(gt, lt);
56+
CValue::by_val(val, fx.layout_of(fx.tcx.ty_ordering_enum(Some(fx.mir.span))))
57+
}
58+
4359
fn codegen_compare_bin_op<'tcx>(
4460
fx: &mut FunctionCx<'_, '_, 'tcx>,
4561
bin_op: BinOp,
4662
signed: bool,
4763
lhs: Value,
4864
rhs: Value,
4965
) -> CValue<'tcx> {
66+
if bin_op == BinOp::Cmp {
67+
return codegen_three_way_compare(fx, signed, lhs, rhs);
68+
}
69+
5070
let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap();
5171
let val = fx.bcx.ins().icmp(intcc, lhs, rhs);
5272
CValue::by_val(val, fx.layout_of(fx.tcx.types.bool))
@@ -59,7 +79,7 @@ pub(crate) fn codegen_binop<'tcx>(
5979
in_rhs: CValue<'tcx>,
6080
) -> CValue<'tcx> {
6181
match bin_op {
62-
BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
82+
BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt | BinOp::Cmp => {
6383
match in_lhs.layout().ty.kind() {
6484
ty::Bool | ty::Uint(_) | ty::Int(_) | ty::Char => {
6585
let signed = type_sign(in_lhs.layout().ty);
@@ -160,7 +180,7 @@ pub(crate) fn codegen_int_binop<'tcx>(
160180
}
161181
BinOp::Offset => unreachable!("Offset is not an integer operation"),
162182
// Compare binops handles by `codegen_binop`.
163-
BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge => {
183+
BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge | BinOp::Cmp => {
164184
unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty);
165185
}
166186
};

‎compiler/rustc_codegen_gcc/src/common.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,10 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
9494
self.const_int(self.type_i32(), i as i64)
9595
}
9696

97+
fn const_i8(&self, i: i8) -> RValue<'gcc> {
98+
self.const_int(self.type_i8(), i as i64)
99+
}
100+
97101
fn const_u32(&self, i: u32) -> RValue<'gcc> {
98102
self.const_uint(self.type_u32(), i as u64)
99103
}

‎compiler/rustc_codegen_llvm/src/common.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
160160
self.const_int(self.type_i32(), i as i64)
161161
}
162162

163+
fn const_i8(&self, i: i8) -> &'ll Value {
164+
self.const_int(self.type_i8(), i as i64)
165+
}
166+
163167
fn const_u32(&self, i: u32) -> &'ll Value {
164168
self.const_uint(self.type_i32(), i as u64)
165169
}

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use crate::common::{self, IntPredicate};
77
use crate::traits::*;
88
use crate::MemFlags;
99

10+
use rustc_hir as hir;
1011
use rustc_middle::mir;
1112
use rustc_middle::mir::Operand;
1213
use rustc_middle::ty::cast::{CastTy, IntTy};
@@ -882,6 +883,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
882883
bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs)
883884
}
884885
}
886+
mir::BinOp::Cmp => {
887+
use std::cmp::Ordering;
888+
debug_assert!(!is_float);
889+
let pred = |op| base::bin_op_to_icmp_predicate(op, is_signed);
890+
if bx.cx().tcx().sess.opts.optimize == OptLevel::No {
891+
// FIXME: This actually generates tighter assembly, and is a classic trick
892+
// <https://graphics.stanford.edu/~seander/bithacks.html#CopyIntegerSign>
893+
// However, as of 2023-11 it optimizes worse in things like derived
894+
// `PartialOrd`, so only use it in debug for now. Once LLVM can handle it
895+
// better (see <https://github.com/llvm/llvm-project/issues/73417>), it'll
896+
// be worth trying it in optimized builds as well.
897+
let is_gt = bx.icmp(pred(hir::BinOpKind::Gt), lhs, rhs);
898+
let gtext = bx.zext(is_gt, bx.type_i8());
899+
let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs);
900+
let ltext = bx.zext(is_lt, bx.type_i8());
901+
bx.unchecked_ssub(gtext, ltext)
902+
} else {
903+
// These operations are those expected by `tests/codegen/integer-cmp.rs`,
904+
// from <https://github.com/rust-lang/rust/pull/63767>.
905+
let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs);
906+
let is_ne = bx.icmp(pred(hir::BinOpKind::Ne), lhs, rhs);
907+
let ge = bx.select(
908+
is_ne,
909+
bx.cx().const_i8(Ordering::Greater as i8),
910+
bx.cx().const_i8(Ordering::Equal as i8),
911+
);
912+
bx.select(is_lt, bx.cx().const_i8(Ordering::Less as i8), ge)
913+
}
914+
}
885915
}
886916
}
887917

‎compiler/rustc_codegen_ssa/src/traits/consts.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ pub trait ConstMethods<'tcx>: BackendTypes {
1919
fn const_bool(&self, val: bool) -> Self::Value;
2020
fn const_i16(&self, i: i16) -> Self::Value;
2121
fn const_i32(&self, i: i32) -> Self::Value;
22+
fn const_i8(&self, i: i8) -> Self::Value;
2223
fn const_u32(&self, i: u32) -> Self::Value;
2324
fn const_u64(&self, i: u64) -> Self::Value;
2425
fn const_u128(&self, i: u128) -> Self::Value;

‎compiler/rustc_const_eval/src/interpret/operand.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,13 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
235235
Self::from_scalar(Scalar::from_bool(b), layout)
236236
}
237237

238+
#[inline]
239+
pub fn from_ordering(c: std::cmp::Ordering, tcx: TyCtxt<'tcx>) -> Self {
240+
let ty = tcx.ty_ordering_enum(None);
241+
let layout = tcx.layout_of(ty::ParamEnv::reveal_all().and(ty)).unwrap();
242+
Self::from_scalar(Scalar::from_i8(c as i8), layout)
243+
}
244+
238245
#[inline]
239246
pub fn to_const_int(self) -> ConstInt {
240247
assert!(self.layout.ty.is_integral());

‎compiler/rustc_const_eval/src/interpret/operator.rs

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
6161
}
6262

6363
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
64+
fn three_way_compare<T: Ord>(&self, lhs: T, rhs: T) -> (ImmTy<'tcx, M::Provenance>, bool) {
65+
let res = Ord::cmp(&lhs, &rhs);
66+
return (ImmTy::from_ordering(res, *self.tcx), false);
67+
}
68+
6469
fn binary_char_op(
6570
&self,
6671
bin_op: mir::BinOp,
@@ -69,6 +74,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
6974
) -> (ImmTy<'tcx, M::Provenance>, bool) {
7075
use rustc_middle::mir::BinOp::*;
7176

77+
if bin_op == Cmp {
78+
return self.three_way_compare(l, r);
79+
}
80+
7281
let res = match bin_op {
7382
Eq => l == r,
7483
Ne => l != r,
@@ -231,6 +240,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
231240
let r = self.sign_extend(r, right_layout) as i128;
232241
return Ok((ImmTy::from_bool(op(&l, &r), *self.tcx), false));
233242
}
243+
if bin_op == Cmp {
244+
let l = self.sign_extend(l, left_layout) as i128;
245+
let r = self.sign_extend(r, right_layout) as i128;
246+
return Ok(self.three_way_compare(l, r));
247+
}
234248
let op: Option<fn(i128, i128) -> (i128, bool)> = match bin_op {
235249
Div if r == 0 => throw_ub!(DivisionByZero),
236250
Rem if r == 0 => throw_ub!(RemainderByZero),
@@ -270,6 +284,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
270284
}
271285
}
272286

287+
if bin_op == Cmp {
288+
return Ok(self.three_way_compare(l, r));
289+
}
290+
273291
let val = match bin_op {
274292
Eq => ImmTy::from_bool(l == r, *self.tcx),
275293
Ne => ImmTy::from_bool(l != r, *self.tcx),

‎compiler/rustc_const_eval/src/transform/validate.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -987,6 +987,15 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
987987
)
988988
}
989989
}
990+
Cmp => {
991+
for x in [a, b] {
992+
check_kinds!(
993+
x,
994+
"Cannot three-way compare non-integer type {:?}",
995+
ty::Char | ty::Uint(..) | ty::Int(..)
996+
)
997+
}
998+
}
990999
AddUnchecked | SubUnchecked | MulUnchecked | Shl | ShlUnchecked | Shr
9911000
| ShrUnchecked => {
9921001
for x in [a, b] {

‎compiler/rustc_const_eval/src/util/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ pub fn binop_left_homogeneous(op: mir::BinOp) -> bool {
1919
match op {
2020
Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
2121
| BitAnd | BitOr | Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => true,
22-
Eq | Ne | Lt | Le | Gt | Ge => false,
22+
Eq | Ne | Lt | Le | Gt | Ge | Cmp => false,
2323
}
2424
}
2525

@@ -30,7 +30,7 @@ pub fn binop_right_homogeneous(op: mir::BinOp) -> bool {
3030
use rustc_middle::mir::BinOp::*;
3131
match op {
3232
Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
33-
| BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge => true,
33+
| BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge | Cmp => true,
3434
Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => false,
3535
}
3636
}

‎compiler/rustc_hir/src/lang_items.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ language_item_table! {
225225
Unpin, sym::unpin, unpin_trait, Target::Trait, GenericRequirement::None;
226226
Pin, sym::pin, pin_type, Target::Struct, GenericRequirement::None;
227227

228+
OrderingEnum, sym::Ordering, ordering_enum, Target::Enum, GenericRequirement::Exact(0);
228229
PartialEq, sym::eq, eq_trait, Target::Trait, GenericRequirement::Exact(1);
229230
PartialOrd, sym::partial_ord, partial_ord_trait, Target::Trait, GenericRequirement::Exact(1);
230231
CVoid, sym::c_void, c_void, Target::Enum, GenericRequirement::None;

‎compiler/rustc_hir_analysis/src/check/intrinsic.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
107107
| sym::cttz
108108
| sym::bswap
109109
| sym::bitreverse
110+
| sym::three_way_compare
110111
| sym::discriminant_value
111112
| sym::type_id
112113
| sym::likely
@@ -418,6 +419,10 @@ pub fn check_intrinsic_type(
418419
| sym::bswap
419420
| sym::bitreverse => (1, 0, vec![param(0)], param(0)),
420421

422+
sym::three_way_compare => {
423+
(1, 0, vec![param(0), param(0)], tcx.ty_ordering_enum(Some(span)))
424+
}
425+
421426
sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
422427
(1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
423428
}

‎compiler/rustc_middle/src/mir/syntax.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1444,6 +1444,8 @@ pub enum BinOp {
14441444
Ge,
14451445
/// The `>` operator (greater than)
14461446
Gt,
1447+
/// The `<=>` operator (three-way comparison, like `Ord::cmp`)
1448+
Cmp,
14471449
/// The `ptr.offset` operator
14481450
Offset,
14491451
}

‎compiler/rustc_middle/src/mir/tcx.rs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,11 @@ impl<'tcx> BinOp {
276276
&BinOp::Eq | &BinOp::Lt | &BinOp::Le | &BinOp::Ne | &BinOp::Ge | &BinOp::Gt => {
277277
tcx.types.bool
278278
}
279+
&BinOp::Cmp => {
280+
// these should be integer-like types of the same size.
281+
assert_eq!(lhs_ty, rhs_ty);
282+
tcx.ty_ordering_enum(None)
283+
}
279284
}
280285
}
281286
}
@@ -312,7 +317,8 @@ impl BinOp {
312317
BinOp::Gt => hir::BinOpKind::Gt,
313318
BinOp::Le => hir::BinOpKind::Le,
314319
BinOp::Ge => hir::BinOpKind::Ge,
315-
BinOp::AddUnchecked
320+
BinOp::Cmp
321+
| BinOp::AddUnchecked
316322
| BinOp::SubUnchecked
317323
| BinOp::MulUnchecked
318324
| BinOp::ShlUnchecked

‎compiler/rustc_middle/src/ty/context.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -905,6 +905,13 @@ impl<'tcx> TyCtxt<'tcx> {
905905
self.get_lang_items(())
906906
}
907907

908+
/// Gets a `Ty` representing the [`LangItem::OrderingEnum`]
909+
#[track_caller]
910+
pub fn ty_ordering_enum(self, span: Option<Span>) -> Ty<'tcx> {
911+
let ordering_enum = self.require_lang_item(hir::LangItem::OrderingEnum, span);
912+
self.type_of(ordering_enum).no_bound_vars().unwrap()
913+
}
914+
908915
/// Obtain the given diagnostic item's `DefId`. Use `is_diagnostic_item` if you just want to
909916
/// compare against another `DefId`, since `is_diagnostic_item` is cheaper.
910917
pub fn get_diagnostic_item(self, name: Symbol) -> Option<DefId> {

‎compiler/rustc_mir_transform/src/lower_intrinsics.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
9090
sym::wrapping_add
9191
| sym::wrapping_sub
9292
| sym::wrapping_mul
93+
| sym::three_way_compare
9394
| sym::unchecked_add
9495
| sym::unchecked_sub
9596
| sym::unchecked_mul
@@ -109,6 +110,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
109110
sym::wrapping_add => BinOp::Add,
110111
sym::wrapping_sub => BinOp::Sub,
111112
sym::wrapping_mul => BinOp::Mul,
113+
sym::three_way_compare => BinOp::Cmp,
112114
sym::unchecked_add => BinOp::AddUnchecked,
113115
sym::unchecked_sub => BinOp::SubUnchecked,
114116
sym::unchecked_mul => BinOp::MulUnchecked,

‎compiler/rustc_mir_transform/src/promote_consts.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,6 +525,7 @@ impl<'tcx> Validator<'_, 'tcx> {
525525
| BinOp::Lt
526526
| BinOp::Ge
527527
| BinOp::Gt
528+
| BinOp::Cmp
528529
| BinOp::Offset
529530
| BinOp::Add
530531
| BinOp::AddUnchecked

‎compiler/rustc_smir/src/rustc_smir/convert/mir.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -493,6 +493,7 @@ impl<'tcx> Stable<'tcx> for mir::BinOp {
493493
BinOp::Ne => stable_mir::mir::BinOp::Ne,
494494
BinOp::Ge => stable_mir::mir::BinOp::Ge,
495495
BinOp::Gt => stable_mir::mir::BinOp::Gt,
496+
BinOp::Cmp => stable_mir::mir::BinOp::Cmp,
496497
BinOp::Offset => stable_mir::mir::BinOp::Offset,
497498
}
498499
}

‎compiler/rustc_span/src/symbol.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1785,6 +1785,7 @@ symbols! {
17851785
thread,
17861786
thread_local,
17871787
thread_local_macro,
1788+
three_way_compare,
17881789
thumb2,
17891790
thumb_mode: "thumb-mode",
17901791
tmm_reg,

‎compiler/rustc_ty_utils/src/consts.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ fn check_binop(op: mir::BinOp) -> bool {
8282
match op {
8383
Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
8484
| BitAnd | BitOr | Shl | ShlUnchecked | Shr | ShrUnchecked | Eq | Lt | Le | Ne | Ge
85-
| Gt => true,
85+
| Gt | Cmp => true,
8686
Offset => false,
8787
}
8888
}

‎compiler/stable_mir/src/mir/body.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,7 @@ pub enum BinOp {
329329
Ne,
330330
Ge,
331331
Gt,
332+
Cmp,
332333
Offset,
333334
}
334335

@@ -368,6 +369,9 @@ impl BinOp {
368369
assert!(lhs_kind.is_primitive() || lhs_kind.is_raw_ptr() || lhs_kind.is_fn_ptr());
369370
Ty::bool_ty()
370371
}
372+
BinOp::Cmp => {
373+
unimplemented!("Should cmp::Ordering be a RigidTy?");
374+
}
371375
}
372376
}
373377
}

‎library/core/src/cmp.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,7 @@ pub struct AssertParamIsEq<T: Eq + ?Sized> {
376376
/// ```
377377
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
378378
#[stable(feature = "rust1", since = "1.0.0")]
379+
#[cfg_attr(not(bootstrap), lang = "Ordering")]
379380
#[repr(i8)]
380381
pub enum Ordering {
381382
/// An ordering where a compared value is less than another.
@@ -1563,12 +1564,19 @@ mod impls {
15631564
impl Ord for $t {
15641565
#[inline]
15651566
fn cmp(&self, other: &$t) -> Ordering {
1567+
#[cfg(bootstrap)]
1568+
{
15661569
// The order here is important to generate more optimal assembly.
15671570
// See <https://github.com/rust-lang/rust/issues/63758> for more info.
15681571
if *self < *other { Less }
15691572
else if *self == *other { Equal }
15701573
else { Greater }
15711574
}
1575+
#[cfg(not(bootstrap))]
1576+
{
1577+
crate::intrinsics::three_way_compare(*self, *other)
1578+
}
1579+
}
15721580
}
15731581
)*)
15741582
}

‎library/core/src/intrinsics.rs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2146,6 +2146,18 @@ extern "rust-intrinsic" {
21462146
#[rustc_nounwind]
21472147
pub fn bitreverse<T: Copy>(x: T) -> T;
21482148

2149+
/// Does a three-way comparison between the two integer arguments.
2150+
///
2151+
/// This is included as an intrinsic as it's useful to let it be one thing
2152+
/// in MIR, rather than the multiple checks and switches that make its IR
2153+
/// large and difficult to optimize.
2154+
///
2155+
/// The stabilized version of this intrinsic is [`Ord::cmp`].
2156+
#[cfg(not(bootstrap))]
2157+
#[rustc_const_unstable(feature = "const_three_way_compare", issue = "none")]
2158+
#[rustc_safe_intrinsic]
2159+
pub fn three_way_compare<T: Copy>(lhs: T, rhs: T) -> crate::cmp::Ordering;
2160+
21492161
/// Performs checked integer addition.
21502162
///
21512163
/// Note that, unlike most intrinsics, this is safe to call;

‎library/core/tests/intrinsics.rs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,3 +99,22 @@ fn test_const_deallocate_at_runtime() {
9999
const_deallocate(core::ptr::null_mut(), 1, 1); // nop
100100
}
101101
}
102+
103+
#[cfg(not(bootstrap))]
104+
#[test]
105+
fn test_three_way_compare_in_const_contexts() {
106+
use core::cmp::Ordering::*;
107+
use core::intrinsics::three_way_compare;
108+
109+
const {
110+
assert!(Less as i8 == three_way_compare(123_u16, 456) as _);
111+
assert!(Equal as i8 == three_way_compare(456_u16, 456) as _);
112+
assert!(Greater as i8 == three_way_compare(789_u16, 456) as _);
113+
assert!(Less as i8 == three_way_compare('A', 'B') as _);
114+
assert!(Equal as i8 == three_way_compare('B', 'B') as _);
115+
assert!(Greater as i8 == three_way_compare('C', 'B') as _);
116+
assert!(Less as i8 == three_way_compare(-123_i16, 456) as _);
117+
assert!(Equal as i8 == three_way_compare(456_i16, 456) as _);
118+
assert!(Greater as i8 == three_way_compare(123_i16, -456) as _);
119+
}
120+
}

‎library/core/tests/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#![feature(const_pointer_is_aligned)]
2222
#![feature(const_ptr_as_ref)]
2323
#![feature(const_ptr_write)]
24+
#![cfg_attr(not(bootstrap), feature(const_three_way_compare))]
2425
#![feature(const_trait_impl)]
2526
#![feature(const_likely)]
2627
#![feature(const_location_fields)]

‎tests/assembly/x86_64-cmp.rs

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
//@ revisions: DEBUG OPTIM
2+
//@ [DEBUG] compile-flags: -C opt-level=0
3+
//@ [OPTIM] compile-flags: -C opt-level=3
4+
//@ assembly-output: emit-asm
5+
//@ compile-flags: --crate-type=lib -C llvm-args=-x86-asm-syntax=intel
6+
//@ only-x86_64
7+
//@ ignore-sgx
8+
9+
#![feature(core_intrinsics)]
10+
11+
use std::intrinsics::three_way_compare;
12+
13+
#[no_mangle]
14+
// CHECK-LABEL: signed_cmp:
15+
pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
16+
// DEBUG: cmp
17+
// DEBUG: setg
18+
// DEBUG: and
19+
// DEBUG: cmp
20+
// DEBUG: setl
21+
// DEBUG: and
22+
// DEBUG: sub
23+
24+
// OPTIM: xor
25+
// OPTIM: cmp
26+
// OPTIM: setne
27+
// OPTIM: mov
28+
// OPTIM: cmovge
29+
// OPTIM: ret
30+
three_way_compare(a, b)
31+
}
32+
33+
#[no_mangle]
34+
// CHECK-LABEL: unsigned_cmp:
35+
pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
36+
// DEBUG: cmp
37+
// DEBUG: seta
38+
// DEBUG: and
39+
// DEBUG: cmp
40+
// DEBUG: setb
41+
// DEBUG: and
42+
// DEBUG: sub
43+
44+
// OPTIM: xor
45+
// OPTIM: cmp
46+
// OPTIM: setne
47+
// OPTIM: mov
48+
// OPTIM: cmovae
49+
// OPTIM: ret
50+
three_way_compare(a, b)
51+
}

‎tests/codegen/intrinsics/three_way_compare.rs

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
//@ revisions: DEBUG OPTIM
2+
//@ [DEBUG] compile-flags: -C opt-level=0
3+
//@ [OPTIM] compile-flags: -C opt-level=3
4+
//@ compile-flags: -C no-prepopulate-passes
5+
6+
#![crate_type = "lib"]
7+
#![feature(core_intrinsics)]
8+
9+
use std::intrinsics::three_way_compare;
10+
11+
#[no_mangle]
12+
// CHECK-LABEL: @signed_cmp
13+
// DEBUG-SAME: (i16 %a, i16 %b)
14+
// OPTIM-SAME: (i16 noundef %a, i16 noundef %b)
15+
pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
16+
// DEBUG: %[[GT:.+]] = icmp sgt i16 %a, %b
17+
// DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
18+
// DEBUG: %[[LT:.+]] = icmp slt i16 %a, %b
19+
// DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
20+
// DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
21+
22+
// OPTIM: %[[LT:.+]] = icmp slt i16 %a, %b
23+
// OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
24+
// OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
25+
// OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
26+
// OPTIM: ret i8 %[[CGEL]]
27+
three_way_compare(a, b)
28+
}
29+
30+
#[no_mangle]
31+
// CHECK-LABEL: @unsigned_cmp
32+
// DEBUG-SAME: (i16 %a, i16 %b)
33+
// OPTIM-SAME: (i16 noundef %a, i16 noundef %b)
34+
pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
35+
// DEBUG: %[[GT:.+]] = icmp ugt i16 %a, %b
36+
// DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
37+
// DEBUG: %[[LT:.+]] = icmp ult i16 %a, %b
38+
// DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
39+
// DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
40+
41+
// OPTIM: %[[LT:.+]] = icmp ult i16 %a, %b
42+
// OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
43+
// OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
44+
// OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
45+
// OPTIM: ret i8 %[[CGEL]]
46+
three_way_compare(a, b)
47+
}

‎tests/mir-opt/lower_intrinsics.rs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,3 +229,18 @@ pub unsafe fn ptr_offset(p: *const i32, d: isize) -> *const i32 {
229229

230230
core::intrinsics::offset(p, d)
231231
}
232+
233+
// EMIT_MIR lower_intrinsics.three_way_compare_char.LowerIntrinsics.diff
234+
pub fn three_way_compare_char(a: char, b: char) {
235+
let _x = core::intrinsics::three_way_compare(a, b);
236+
}
237+
238+
// EMIT_MIR lower_intrinsics.three_way_compare_signed.LowerIntrinsics.diff
239+
pub fn three_way_compare_signed(a: i16, b: i16) {
240+
core::intrinsics::three_way_compare(a, b);
241+
}
242+
243+
// EMIT_MIR lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.diff
244+
pub fn three_way_compare_unsigned(a: u32, b: u32) {
245+
let _x = core::intrinsics::three_way_compare(a, b);
246+
}

‎tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-abort.diff

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
- // MIR for `three_way_compare_char` before LowerIntrinsics
2+
+ // MIR for `three_way_compare_char` after LowerIntrinsics
3+
4+
fn three_way_compare_char(_1: char, _2: char) -> () {
5+
debug a => _1;
6+
debug b => _2;
7+
let mut _0: ();
8+
let _3: std::cmp::Ordering;
9+
let mut _4: char;
10+
let mut _5: char;
11+
scope 1 {
12+
debug _x => _3;
13+
}
14+
15+
bb0: {
16+
StorageLive(_3);
17+
StorageLive(_4);
18+
_4 = _1;
19+
StorageLive(_5);
20+
_5 = _2;
21+
- _3 = three_way_compare::<char>(move _4, move _5) -> [return: bb1, unwind unreachable];
22+
+ _3 = Cmp(move _4, move _5);
23+
+ goto -> bb1;
24+
}
25+
26+
bb1: {
27+
StorageDead(_5);
28+
StorageDead(_4);
29+
_0 = const ();
30+
StorageDead(_3);
31+
return;
32+
}
33+
}
34+

‎tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-unwind.diff

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
- // MIR for `three_way_compare_char` before LowerIntrinsics
2+
+ // MIR for `three_way_compare_char` after LowerIntrinsics
3+
4+
fn three_way_compare_char(_1: char, _2: char) -> () {
5+
debug a => _1;
6+
debug b => _2;
7+
let mut _0: ();
8+
let _3: std::cmp::Ordering;
9+
let mut _4: char;
10+
let mut _5: char;
11+
scope 1 {
12+
debug _x => _3;
13+
}
14+
15+
bb0: {
16+
StorageLive(_3);
17+
StorageLive(_4);
18+
_4 = _1;
19+
StorageLive(_5);
20+
_5 = _2;
21+
- _3 = three_way_compare::<char>(move _4, move _5) -> [return: bb1, unwind continue];
22+
+ _3 = Cmp(move _4, move _5);
23+
+ goto -> bb1;
24+
}
25+
26+
bb1: {
27+
StorageDead(_5);
28+
StorageDead(_4);
29+
_0 = const ();
30+
StorageDead(_3);
31+
return;
32+
}
33+
}
34+

‎tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-abort.diff

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
- // MIR for `three_way_compare_signed` before LowerIntrinsics
2+
+ // MIR for `three_way_compare_signed` after LowerIntrinsics
3+
4+
fn three_way_compare_signed(_1: i16, _2: i16) -> () {
5+
debug a => _1;
6+
debug b => _2;
7+
let mut _0: ();
8+
let _3: std::cmp::Ordering;
9+
let mut _4: i16;
10+
let mut _5: i16;
11+
12+
bb0: {
13+
StorageLive(_3);
14+
StorageLive(_4);
15+
_4 = _1;
16+
StorageLive(_5);
17+
_5 = _2;
18+
- _3 = three_way_compare::<i16>(move _4, move _5) -> [return: bb1, unwind unreachable];
19+
+ _3 = Cmp(move _4, move _5);
20+
+ goto -> bb1;
21+
}
22+
23+
bb1: {
24+
StorageDead(_5);
25+
StorageDead(_4);
26+
StorageDead(_3);
27+
_0 = const ();
28+
return;
29+
}
30+
}
31+

‎tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-unwind.diff

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
- // MIR for `three_way_compare_signed` before LowerIntrinsics
2+
+ // MIR for `three_way_compare_signed` after LowerIntrinsics
3+
4+
fn three_way_compare_signed(_1: i16, _2: i16) -> () {
5+
debug a => _1;
6+
debug b => _2;
7+
let mut _0: ();
8+
let _3: std::cmp::Ordering;
9+
let mut _4: i16;
10+
let mut _5: i16;
11+
12+
bb0: {
13+
StorageLive(_3);
14+
StorageLive(_4);
15+
_4 = _1;
16+
StorageLive(_5);
17+
_5 = _2;
18+
- _3 = three_way_compare::<i16>(move _4, move _5) -> [return: bb1, unwind continue];
19+
+ _3 = Cmp(move _4, move _5);
20+
+ goto -> bb1;
21+
}
22+
23+
bb1: {
24+
StorageDead(_5);
25+
StorageDead(_4);
26+
StorageDead(_3);
27+
_0 = const ();
28+
return;
29+
}
30+
}
31+

‎tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-abort.diff

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
- // MIR for `three_way_compare_unsigned` before LowerIntrinsics
2+
+ // MIR for `three_way_compare_unsigned` after LowerIntrinsics
3+
4+
fn three_way_compare_unsigned(_1: u32, _2: u32) -> () {
5+
debug a => _1;
6+
debug b => _2;
7+
let mut _0: ();
8+
let _3: std::cmp::Ordering;
9+
let mut _4: u32;
10+
let mut _5: u32;
11+
scope 1 {
12+
debug _x => _3;
13+
}
14+
15+
bb0: {
16+
StorageLive(_3);
17+
StorageLive(_4);
18+
_4 = _1;
19+
StorageLive(_5);
20+
_5 = _2;
21+
- _3 = three_way_compare::<u32>(move _4, move _5) -> [return: bb1, unwind unreachable];
22+
+ _3 = Cmp(move _4, move _5);
23+
+ goto -> bb1;
24+
}
25+
26+
bb1: {
27+
StorageDead(_5);
28+
StorageDead(_4);
29+
_0 = const ();
30+
StorageDead(_3);
31+
return;
32+
}
33+
}
34+

‎tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-unwind.diff

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
- // MIR for `three_way_compare_unsigned` before LowerIntrinsics
2+
+ // MIR for `three_way_compare_unsigned` after LowerIntrinsics
3+
4+
fn three_way_compare_unsigned(_1: u32, _2: u32) -> () {
5+
debug a => _1;
6+
debug b => _2;
7+
let mut _0: ();
8+
let _3: std::cmp::Ordering;
9+
let mut _4: u32;
10+
let mut _5: u32;
11+
scope 1 {
12+
debug _x => _3;
13+
}
14+
15+
bb0: {
16+
StorageLive(_3);
17+
StorageLive(_4);
18+
_4 = _1;
19+
StorageLive(_5);
20+
_5 = _2;
21+
- _3 = three_way_compare::<u32>(move _4, move _5) -> [return: bb1, unwind continue];
22+
+ _3 = Cmp(move _4, move _5);
23+
+ goto -> bb1;
24+
}
25+
26+
bb1: {
27+
StorageDead(_5);
28+
StorageDead(_4);
29+
_0 = const ();
30+
StorageDead(_3);
31+
return;
32+
}
33+
}
34+

‎tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir

Lines changed: 32 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -6,37 +6,33 @@ fn <impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp(_1: &MultiField, _2: &M
66
let mut _0: std::option::Option<std::cmp::Ordering>;
77
let mut _3: &char;
88
let mut _4: &char;
9-
let mut _10: std::option::Option<std::cmp::Ordering>;
9+
let mut _8: std::option::Option<std::cmp::Ordering>;
10+
let mut _9: i8;
11+
let mut _10: &i16;
1012
let mut _11: &i16;
11-
let mut _12: &i16;
12-
let _18: std::option::Option<std::cmp::Ordering>;
1313
scope 1 {
14-
debug cmp => _18;
14+
debug cmp => _8;
1515
}
1616
scope 2 (inlined std::cmp::impls::<impl PartialOrd for char>::partial_cmp) {
1717
debug self => _3;
1818
debug other => _4;
19-
let mut _9: std::cmp::Ordering;
19+
let mut _7: std::cmp::Ordering;
2020
scope 3 (inlined std::cmp::impls::<impl Ord for char>::cmp) {
2121
debug self => _3;
2222
debug other => _4;
2323
let mut _5: char;
2424
let mut _6: char;
25-
let mut _7: bool;
26-
let mut _8: bool;
2725
}
2826
}
2927
scope 4 (inlined std::cmp::impls::<impl PartialOrd for i16>::partial_cmp) {
30-
debug self => _11;
31-
debug other => _12;
32-
let mut _17: std::cmp::Ordering;
28+
debug self => _10;
29+
debug other => _11;
30+
let mut _14: std::cmp::Ordering;
3331
scope 5 (inlined std::cmp::impls::<impl Ord for i16>::cmp) {
34-
debug self => _11;
35-
debug other => _12;
32+
debug self => _10;
33+
debug other => _11;
34+
let mut _12: i16;
3635
let mut _13: i16;
37-
let mut _14: i16;
38-
let mut _15: bool;
39-
let mut _16: bool;
4036
}
4137
}
4238

@@ -45,115 +41,46 @@ fn <impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp(_1: &MultiField, _2: &M
4541
_3 = &((*_1).0: char);
4642
StorageLive(_4);
4743
_4 = &((*_2).0: char);
48-
StorageLive(_9);
4944
StorageLive(_5);
50-
StorageLive(_6);
51-
StorageLive(_7);
5245
_5 = ((*_1).0: char);
46+
StorageLive(_6);
5347
_6 = ((*_2).0: char);
54-
_7 = Lt(_5, _6);
55-
switchInt(move _7) -> [0: bb1, otherwise: bb10];
56-
}
57-
58-
bb1: {
59-
StorageLive(_8);
60-
_8 = Eq(_5, _6);
61-
switchInt(move _8) -> [0: bb2, otherwise: bb3];
62-
}
63-
64-
bb2: {
65-
_9 = const Greater;
66-
StorageDead(_8);
67-
StorageDead(_7);
48+
_7 = Cmp(move _5, move _6);
6849
StorageDead(_6);
6950
StorageDead(_5);
70-
_10 = Option::<std::cmp::Ordering>::Some(move _9);
71-
StorageDead(_9);
51+
_8 = Option::<std::cmp::Ordering>::Some(_7);
7252
StorageDead(_4);
7353
StorageDead(_3);
74-
goto -> bb11;
54+
_9 = discriminant(_7);
55+
switchInt(move _9) -> [0: bb1, otherwise: bb2];
7556
}
7657

77-
bb3: {
78-
StorageDead(_8);
79-
StorageDead(_7);
80-
StorageDead(_6);
81-
StorageDead(_5);
82-
StorageDead(_9);
83-
StorageDead(_4);
84-
StorageDead(_3);
58+
bb1: {
59+
StorageLive(_10);
60+
_10 = &((*_1).1: i16);
8561
StorageLive(_11);
86-
_11 = &((*_1).1: i16);
62+
_11 = &((*_2).1: i16);
63+
StorageLive(_14);
8764
StorageLive(_12);
88-
_12 = &((*_2).1: i16);
89-
StorageLive(_17);
65+
_12 = ((*_1).1: i16);
9066
StorageLive(_13);
91-
StorageLive(_14);
92-
StorageLive(_15);
93-
_13 = ((*_1).1: i16);
94-
_14 = ((*_2).1: i16);
95-
_15 = Lt(_13, _14);
96-
switchInt(move _15) -> [0: bb4, otherwise: bb8];
97-
}
98-
99-
bb4: {
100-
StorageLive(_16);
101-
_16 = Eq(_13, _14);
102-
switchInt(move _16) -> [0: bb5, otherwise: bb6];
103-
}
104-
105-
bb5: {
106-
_17 = const Greater;
107-
goto -> bb7;
108-
}
109-
110-
bb6: {
111-
_17 = const Equal;
112-
goto -> bb7;
113-
}
114-
115-
bb7: {
116-
StorageDead(_16);
117-
goto -> bb9;
118-
}
119-
120-
bb8: {
121-
_17 = const Less;
122-
goto -> bb9;
123-
}
124-
125-
bb9: {
126-
StorageDead(_15);
127-
StorageDead(_14);
67+
_13 = ((*_2).1: i16);
68+
_14 = Cmp(move _12, move _13);
12869
StorageDead(_13);
129-
_0 = Option::<std::cmp::Ordering>::Some(move _17);
130-
StorageDead(_17);
13170
StorageDead(_12);
71+
_0 = Option::<std::cmp::Ordering>::Some(move _14);
72+
StorageDead(_14);
13273
StorageDead(_11);
133-
goto -> bb12;
74+
StorageDead(_10);
75+
goto -> bb3;
13476
}
13577

136-
bb10: {
137-
_9 = const Less;
138-
StorageDead(_7);
139-
StorageDead(_6);
140-
StorageDead(_5);
141-
_10 = Option::<std::cmp::Ordering>::Some(move _9);
142-
StorageDead(_9);
143-
StorageDead(_4);
144-
StorageDead(_3);
145-
goto -> bb11;
146-
}
147-
148-
bb11: {
149-
StorageLive(_18);
150-
_18 = _10;
151-
_0 = _10;
152-
StorageDead(_18);
153-
goto -> bb12;
78+
bb2: {
79+
_0 = _8;
80+
goto -> bb3;
15481
}
15582

156-
bb12: {
83+
bb3: {
15784
return;
15885
}
15986
}

0 commit comments

Comments
 (0)
Please sign in to comment.