Skip to content

Commit e08b80c

Browse files
borsbors
committed
Auto merge of rust-lang#128371 - andjo403:rangeAttribute, r=nikic
Add range attribute to scalar function results and arguments as LLVM 19 adds the range attribute this starts to use it for better optimization. hade been interesting to see a perf run with the rust-lang#127513 closes rust-lang#50156 cc rust-lang#49572 shall be fixed but not possible to see as there is asserts that already trigger the optimization.
2 parents 1d8f135 + cfadfab commit e08b80c

File tree

12 files changed

+159
-24
lines changed

12 files changed

+159
-24
lines changed

Diff for: ‎compiler/rustc_codegen_llvm/src/abi.rs

+48-13
Original file line numberDiff line numberDiff line change
@@ -430,9 +430,32 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
430430
i += 1;
431431
i - 1
432432
};
433+
434+
let apply_range_attr = |idx: AttributePlace, scalar: rustc_target::abi::Scalar| {
435+
if cx.sess().opts.optimize != config::OptLevel::No
436+
&& llvm_util::get_version() >= (19, 0, 0)
437+
&& matches!(scalar.primitive(), Int(..))
438+
// If the value is a boolean, the range is 0..2 and that ultimately
439+
// become 0..0 when the type becomes i1, which would be rejected
440+
// by the LLVM verifier.
441+
&& !scalar.is_bool()
442+
// LLVM also rejects full range.
443+
&& !scalar.is_always_valid(cx)
444+
{
445+
attributes::apply_to_llfn(
446+
llfn,
447+
idx,
448+
&[llvm::CreateRangeAttr(cx.llcx, scalar.size(cx), scalar.valid_range(cx))],
449+
);
450+
}
451+
};
452+
433453
match &self.ret.mode {
434454
PassMode::Direct(attrs) => {
435455
attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
456+
if let abi::Abi::Scalar(scalar) = self.ret.layout.abi {
457+
apply_range_attr(llvm::AttributePlace::ReturnValue, scalar);
458+
}
436459
}
437460
PassMode::Indirect { attrs, meta_attrs: _, on_stack } => {
438461
assert!(!on_stack);
@@ -471,8 +494,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
471494
);
472495
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]);
473496
}
474-
PassMode::Direct(attrs)
475-
| PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
497+
PassMode::Direct(attrs) => {
498+
let i = apply(attrs);
499+
if let abi::Abi::Scalar(scalar) = arg.layout.abi {
500+
apply_range_attr(llvm::AttributePlace::Argument(i), scalar);
501+
}
502+
}
503+
PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
476504
apply(attrs);
477505
}
478506
PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
@@ -481,8 +509,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
481509
apply(meta_attrs);
482510
}
483511
PassMode::Pair(a, b) => {
484-
apply(a);
485-
apply(b);
512+
let i = apply(a);
513+
let ii = apply(b);
514+
if let abi::Abi::ScalarPair(scalar_a, scalar_b) = arg.layout.abi {
515+
apply_range_attr(llvm::AttributePlace::Argument(i), scalar_a);
516+
apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b);
517+
}
486518
}
487519
PassMode::Cast { cast, pad_i32 } => {
488520
if *pad_i32 {
@@ -537,15 +569,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
537569
}
538570
_ => {}
539571
}
540-
if let abi::Abi::Scalar(scalar) = self.ret.layout.abi {
541-
// If the value is a boolean, the range is 0..2 and that ultimately
542-
// become 0..0 when the type becomes i1, which would be rejected
543-
// by the LLVM verifier.
544-
if let Int(..) = scalar.primitive() {
545-
if !scalar.is_bool() && !scalar.is_always_valid(bx) {
546-
bx.range_metadata(callsite, scalar.valid_range(bx));
547-
}
548-
}
572+
if bx.cx.sess().opts.optimize != config::OptLevel::No
573+
&& llvm_util::get_version() < (19, 0, 0)
574+
&& let abi::Abi::Scalar(scalar) = self.ret.layout.abi
575+
&& matches!(scalar.primitive(), Int(..))
576+
// If the value is a boolean, the range is 0..2 and that ultimately
577+
// become 0..0 when the type becomes i1, which would be rejected
578+
// by the LLVM verifier.
579+
&& !scalar.is_bool()
580+
// LLVM also rejects full range.
581+
&& !scalar.is_always_valid(bx)
582+
{
583+
bx.range_metadata(callsite, scalar.valid_range(bx));
549584
}
550585
for arg in self.args.iter() {
551586
match &arg.mode {

Diff for: ‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

+6
Original file line numberDiff line numberDiff line change
@@ -1575,6 +1575,12 @@ extern "C" {
15751575
pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute;
15761576
pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute;
15771577
pub fn LLVMRustCreateMemoryEffectsAttr(C: &Context, effects: MemoryEffects) -> &Attribute;
1578+
pub fn LLVMRustCreateRangeAttribute(
1579+
C: &Context,
1580+
num_bits: c_uint,
1581+
lower_words: *const u64,
1582+
upper_words: *const u64,
1583+
) -> &Attribute;
15781584

15791585
// Operations on functions
15801586
pub fn LLVMRustGetOrInsertFunction<'a>(

Diff for: ‎compiler/rustc_codegen_llvm/src/llvm/mod.rs

+16-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use std::string::FromUtf8Error;
88
use libc::c_uint;
99
use rustc_data_structures::small_c_str::SmallCStr;
1010
use rustc_llvm::RustString;
11-
use rustc_target::abi::Align;
11+
use rustc_target::abi::{Align, Size, WrappingRange};
1212

1313
pub use self::AtomicRmwBinOp::*;
1414
pub use self::CallConv::*;
@@ -105,6 +105,21 @@ pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribu
105105
unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) }
106106
}
107107

108+
pub fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute {
109+
let lower = range.start;
110+
let upper = range.end.wrapping_add(1);
111+
let lower_words = [lower as u64, (lower >> 64) as u64];
112+
let upper_words = [upper as u64, (upper >> 64) as u64];
113+
unsafe {
114+
LLVMRustCreateRangeAttribute(
115+
llcx,
116+
size.bits().try_into().unwrap(),
117+
lower_words.as_ptr(),
118+
upper_words.as_ptr(),
119+
)
120+
}
121+
}
122+
108123
#[derive(Copy, Clone)]
109124
pub enum AttributePlace {
110125
ReturnValue,

Diff for: ‎compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

+12
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,18 @@ LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) {
397397
std::nullopt));
398398
}
399399

400+
extern "C" LLVMAttributeRef
401+
LLVMRustCreateRangeAttribute(LLVMContextRef C, unsigned NumBits,
402+
const uint64_t LowerWords[],
403+
const uint64_t UpperWords[]) {
404+
#if LLVM_VERSION_GE(19, 0)
405+
return LLVMCreateConstantRangeAttribute(C, Attribute::Range, NumBits,
406+
LowerWords, UpperWords);
407+
#else
408+
report_fatal_error("LLVM 19.0 is required for Range Attribute");
409+
#endif
410+
}
411+
400412
// These values **must** match ffi::AllocKindFlags.
401413
// It _happens_ to match the LLVM values of llvm::AllocFnKind,
402414
// but that's happenstance and we do explicit conversions before

Diff for: ‎tests/codegen/call-metadata.rs

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// scalar value.
33

44
//@ compile-flags: -O -C no-prepopulate-passes
5+
//@ ignore-llvm-version: 19 - 99
56

67
#![crate_type = "lib"]
78

Diff for: ‎tests/codegen/cast-optimized.rs

-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ pub fn u32_index(c: u32) -> [bool; 22] {
2020
// CHECK-LABEL: @char_as_u32_index
2121
#[no_mangle]
2222
pub fn char_as_u32_index(c: char) -> [bool; 22] {
23-
// CHECK: %[[B:.+]] = icmp ult i32 %c, 1114112
24-
// CHECK: call void @llvm.assume(i1 %[[B]])
2523
let c = c as u32;
2624

2725
let mut array = [false; 22];

Diff for: ‎tests/codegen/common_prim_int_ptr.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ pub fn insert_box(x: Box<()>) -> Result<usize, Box<()>> {
2828

2929
// CHECK-LABEL: @extract_int
3030
// CHECK-NOT: nonnull
31-
// CHECK-SAME: (i{{[0-9]+}} {{[^,]+}} [[DISCRIMINANT:%[0-9]+]], ptr {{[^,]+}} [[PAYLOAD:%[0-9]+]])
31+
// CHECK-SAME: (i{{[0-9]+}} {{[^%]+}} [[DISCRIMINANT:%[0-9]+]], ptr {{[^,]+}} [[PAYLOAD:%[0-9]+]])
3232
#[no_mangle]
3333
pub unsafe fn extract_int(x: Result<usize, Box<()>>) -> usize {
3434
// CHECK: [[TEMP:%.+]] = ptrtoint ptr [[PAYLOAD]] to [[USIZE:i[0-9]+]]
@@ -40,7 +40,7 @@ pub unsafe fn extract_int(x: Result<usize, Box<()>>) -> usize {
4040
}
4141

4242
// CHECK-LABEL: @extract_box
43-
// CHECK-SAME: (i{{[0-9]+}} {{[^,]+}} [[DISCRIMINANT:%[0-9]+]], ptr {{[^,]+}} [[PAYLOAD:%[0-9]+]])
43+
// CHECK-SAME: (i{{[0-9]+}} {{[^%]+}} [[DISCRIMINANT:%[0-9]+]], ptr {{[^,]+}} [[PAYLOAD:%[0-9]+]])
4444
#[no_mangle]
4545
pub unsafe fn extract_box(x: Result<usize, Box<i32>>) -> Box<i32> {
4646
// CHECK: ret ptr [[PAYLOAD]]

Diff for: ‎tests/codegen/enum/enum-match.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ pub enum Enum1 {
3434

3535
// CHECK: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match1{{.*}}
3636
// CHECK-NEXT: start:
37-
// CHECK-NEXT: %1 = add i8 %0, -2
37+
// CHECK-NEXT: %1 = add{{( nsw)?}} i8 %0, -2
3838
// CHECK-NEXT: %2 = zext i8 %1 to i64
3939
// CHECK-NEXT: %3 = icmp ult i8 %1, 2
4040
// CHECK-NEXT: %4 = add nuw nsw i64 %2, 1

Diff for: ‎tests/codegen/function-arguments.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ pub fn maybeuninit_enum_bool(x: MaybeUninit<MyBool>) -> MaybeUninit<MyBool> {
5050
x
5151
}
5252

53-
// CHECK: noundef i32 @char(i32 noundef %x)
53+
// CHECK: noundef{{( range\(i32 0, 1114112\))?}} i32 @char(i32 noundef{{( range\(i32 0, 1114112\))?}} %x)
5454
#[no_mangle]
5555
pub fn char(x: char) -> char {
5656
x
@@ -68,7 +68,7 @@ pub fn int(x: u64) -> u64 {
6868
x
6969
}
7070

71-
// CHECK: noundef i64 @nonzero_int(i64 noundef %x)
71+
// CHECK: noundef{{( range\(i64 1, 0\))?}} i64 @nonzero_int(i64 noundef{{( range\(i64 1, 0\))?}} %x)
7272
#[no_mangle]
7373
pub fn nonzero_int(x: NonZero<u64>) -> NonZero<u64> {
7474
x
@@ -250,7 +250,7 @@ pub fn return_slice(x: &[u16]) -> &[u16] {
250250
x
251251
}
252252

253-
// CHECK: { i16, i16 } @enum_id_1(i16 noundef %x.0, i16 %x.1)
253+
// CHECK: { i16, i16 } @enum_id_1(i16 noundef{{( range\(i16 0, 3\))?}} %x.0, i16 %x.1)
254254
#[no_mangle]
255255
pub fn enum_id_1(x: Option<Result<u16, u16>>) -> Option<Result<u16, u16>> {
256256
x

Diff for: ‎tests/codegen/issues/issue-68667-unwrap-combinators.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
// MIR inlining now optimizes this code.
66

77
// CHECK-LABEL: @unwrap_combinators
8-
// CHECK: icmp
8+
// CHECK: {{icmp|trunc}}
99
// CHECK-NEXT: icmp
1010
// CHECK-NEXT: select i1
1111
// CHECK-NEXT: ret i1

Diff for: ‎tests/codegen/range-attribute.rs

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
// Checks that range metadata gets emitted on functions result and arguments
2+
// with scalar value.
3+
4+
//@ compile-flags: -O -C no-prepopulate-passes
5+
//@ min-llvm-version: 19
6+
7+
#![crate_type = "lib"]
8+
9+
use std::num::NonZero;
10+
11+
// Hack to get the correct size for usize
12+
// CHECK: @helper([[USIZE:i[0-9]+]] noundef %_1)
13+
#[no_mangle]
14+
pub fn helper(_: usize) {}
15+
16+
// CHECK: noundef range(i128 1, 0) i128 @nonzero_int(i128 noundef range(i128 1, 0) %x)
17+
#[no_mangle]
18+
pub fn nonzero_int(x: NonZero<u128>) -> NonZero<u128> {
19+
x
20+
}
21+
22+
// CHECK: noundef range(i8 0, 3) i8 @optional_bool(i8 noundef range(i8 0, 3) %x)
23+
#[no_mangle]
24+
pub fn optional_bool(x: Option<bool>) -> Option<bool> {
25+
x
26+
}
27+
28+
pub enum Enum0 {
29+
A(bool),
30+
B,
31+
C,
32+
}
33+
34+
// CHECK: noundef range(i8 0, 4) i8 @enum0_value(i8 noundef range(i8 0, 4) %x)
35+
#[no_mangle]
36+
pub fn enum0_value(x: Enum0) -> Enum0 {
37+
x
38+
}
39+
40+
pub enum Enum1 {
41+
A(u64),
42+
B(u64),
43+
C(u64),
44+
}
45+
46+
// CHECK: { [[ENUM1_TYP:i[0-9]+]], i64 } @enum1_value([[ENUM1_TYP]] noundef range([[ENUM1_TYP]] 0, 3) %x.0, i64 noundef %x.1)
47+
#[no_mangle]
48+
pub fn enum1_value(x: Enum1) -> Enum1 {
49+
x
50+
}
51+
52+
pub enum Enum2 {
53+
A(Enum0),
54+
B(Enum0),
55+
C(Enum0),
56+
}
57+
58+
// CHECK: { i8, i8 } @enum2_value(i8 noundef range(i8 0, 3) %x.0, i8 noundef %x.1)
59+
#[no_mangle]
60+
pub fn enum2_value(x: Enum2) -> Enum2 {
61+
x
62+
}
63+
64+
// CHECK: noundef [[USIZE]] @takes_slice(ptr noalias noundef nonnull readonly align 4 %x.0, [[USIZE]] noundef %x.1)
65+
#[no_mangle]
66+
pub fn takes_slice(x: &[i32]) -> usize {
67+
x.len()
68+
}

Diff for: ‎tests/codegen/repr/transparent.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ pub enum Bool {
7474
FileNotFound,
7575
}
7676

77-
// CHECK: define{{( dso_local)?}} noundef{{( zeroext)?}} i8 @test_Gpz(i8 noundef{{( zeroext)?}} %_1)
77+
// CHECK: define{{( dso_local)?}} noundef{{( zeroext)?( range\(i8 0, 3\))?}} i8 @test_Gpz(i8 noundef{{( zeroext)?( range\(i8 0, 3\))?}} %_1)
7878
#[no_mangle]
7979
pub extern "C" fn test_Gpz(_: GenericPlusZst<Bool>) -> GenericPlusZst<Bool> {
8080
loop {}

0 commit comments

Comments
 (0)