Rollup merge of #124003 - WaffleLapkin:dellvmization, r=scottmcm,RalfJung,antoyo

Dellvmize some intrinsics (use `u32` instead of `Self` in some integer intrinsics)

This implements rust-lang/compiler-team#693 minus what was implemented in #123226.

Note: I decided to _not_ change `shl`/... builder methods, as it just doesn't seem worth it.

r? ``@scottmcm``

Author: matthiaskrgr

compiler/rustc_codegen_cranelift/example/example.rs

@@ -149,7 +149,7 @@ pub fn array_as_slice(arr: &[u8; 3]) -> &[u8] {
     arr
 }
 
-pub unsafe fn use_ctlz_nonzero(a: u16) -> u16 {
+pub unsafe fn use_ctlz_nonzero(a: u16) -> u32 {
     intrinsics::ctlz_nonzero(a)
 }
 

compiler/rustc_codegen_cranelift/example/mini_core.rs

@@ -627,7 +627,7 @@ pub mod intrinsics {
         pub fn min_align_of_val<T: ?::Sized>(val: *const T) -> usize;
         pub fn copy<T>(src: *const T, dst: *mut T, count: usize);
         pub fn transmute<T, U>(e: T) -> U;
-        pub fn ctlz_nonzero<T>(x: T) -> T;
+        pub fn ctlz_nonzero<T>(x: T) -> u32;
         #[rustc_safe_intrinsic]
         pub fn needs_drop<T: ?::Sized>() -> bool;
         #[rustc_safe_intrinsic]

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -26,6 +26,7 @@ use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{sym, Symbol};
 
 pub(crate) use self::llvm::codegen_llvm_intrinsic_call;
+use crate::cast::clif_intcast;
 use crate::prelude::*;
 
 fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! {
@@ -627,7 +628,8 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
             // FIXME trap on `ctlz_nonzero` with zero arg.
             let res = fx.bcx.ins().clz(val);
-            let res = CValue::by_val(res, arg.layout());
+            let res = clif_intcast(fx, res, types::I32, false);
+            let res = CValue::by_val(res, ret.layout());
             ret.write_cvalue(fx, res);
         }
         sym::cttz | sym::cttz_nonzero => {
@@ -636,15 +638,17 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
             // FIXME trap on `cttz_nonzero` with zero arg.
             let res = fx.bcx.ins().ctz(val);
-            let res = CValue::by_val(res, arg.layout());
+            let res = clif_intcast(fx, res, types::I32, false);
+            let res = CValue::by_val(res, ret.layout());
             ret.write_cvalue(fx, res);
         }
         sym::ctpop => {
             intrinsic_args!(fx, args => (arg); intrinsic);
             let val = arg.load_scalar(fx);
 
             let res = fx.bcx.ins().popcnt(val);
-            let res = CValue::by_val(res, arg.layout());
+            let res = clif_intcast(fx, res, types::I32, false);
+            let res = CValue::by_val(res, ret.layout());
             ret.write_cvalue(fx, res);
         }
         sym::bitreverse => {

compiler/rustc_codegen_gcc/example/example.rs

@@ -153,9 +153,10 @@ fn array_as_slice(arr: &[u8; 3]) -> &[u8] {
     arr
 }
 
-unsafe fn use_ctlz_nonzero(a: u16) -> u16 {
-    intrinsics::ctlz_nonzero(a)
-}
+// FIXME: fix the intrinsic implementation to work with the new ->u32 signature
+// unsafe fn use_ctlz_nonzero(a: u16) -> u32 {
+//     intrinsics::ctlz_nonzero(a)
+// }
 
 fn ptr_as_usize(ptr: *const u8) -> usize {
     ptr as usize

compiler/rustc_codegen_gcc/example/mini_core.rs

@@ -593,7 +593,7 @@ pub mod intrinsics {
         pub fn min_align_of_val<T: ?Sized>(val: *const T) -> usize;
         pub fn copy<T>(src: *const T, dst: *mut T, count: usize);
         pub fn transmute<T, U>(e: T) -> U;
-        pub fn ctlz_nonzero<T>(x: T) -> T;
+        pub fn ctlz_nonzero<T>(x: T) -> u32;
         #[rustc_safe_intrinsic]
         pub fn needs_drop<T: ?Sized>() -> bool;
         #[rustc_safe_intrinsic]

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -315,25 +315,32 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     Some((width, signed)) => match name {
                         sym::ctlz | sym::cttz => {
                             let y = self.const_bool(false);
-                            self.call_intrinsic(
+                            let ret = self.call_intrinsic(
                                 &format!("llvm.{name}.i{width}"),
                                 &[args[0].immediate(), y],
-                            )
+                            );
+
+                            self.intcast(ret, llret_ty, false)
                         }
                         sym::ctlz_nonzero => {
                             let y = self.const_bool(true);
                             let llvm_name = &format!("llvm.ctlz.i{width}");
-                            self.call_intrinsic(llvm_name, &[args[0].immediate(), y])
+                            let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
+                            self.intcast(ret, llret_ty, false)
                         }
                         sym::cttz_nonzero => {
                             let y = self.const_bool(true);
                             let llvm_name = &format!("llvm.cttz.i{width}");
-                            self.call_intrinsic(llvm_name, &[args[0].immediate(), y])
+                            let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
+                            self.intcast(ret, llret_ty, false)
+                        }
+                        sym::ctpop => {
+                            let ret = self.call_intrinsic(
+                                &format!("llvm.ctpop.i{width}"),
+                                &[args[0].immediate()],
+                            );
+                            self.intcast(ret, llret_ty, false)
                         }
-                        sym::ctpop => self.call_intrinsic(
-                            &format!("llvm.ctpop.i{width}"),
-                            &[args[0].immediate()],
-                        ),
                         sym::bswap => {
                             if width == 8 {
                                 args[0].immediate() // byte swap a u8/i8 is just a no-op
@@ -355,6 +362,10 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             // rotate = funnel shift with first two args the same
                             let llvm_name =
                                 &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width);
+
+                            // llvm expects shift to be the same type as the values, but rust always uses `u32`
+                            let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
+
                             self.call_intrinsic(llvm_name, &[val, val, raw_shift])
                         }
                         sym::saturating_add | sym::saturating_sub => {

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -173,7 +173,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let ty = instance_args.type_at(0);
                 let layout = self.layout_of(ty)?;
                 let val = self.read_scalar(&args[0])?;
-                let out_val = self.numeric_intrinsic(intrinsic_name, val, layout)?;
+
+                let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?;
                 self.write_scalar(out_val, dest)?;
             }
             sym::saturating_add | sym::saturating_sub => {
@@ -200,21 +201,24 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             sym::rotate_left | sym::rotate_right => {
                 // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
                 // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
-                let layout = self.layout_of(instance_args.type_at(0))?;
+                let layout_val = self.layout_of(instance_args.type_at(0))?;
                 let val = self.read_scalar(&args[0])?;
-                let val_bits = val.to_bits(layout.size)?;
+                let val_bits = val.to_bits(layout_val.size)?;
+
+                let layout_raw_shift = self.layout_of(self.tcx.types.u32)?;
                 let raw_shift = self.read_scalar(&args[1])?;
-                let raw_shift_bits = raw_shift.to_bits(layout.size)?;
-                let width_bits = u128::from(layout.size.bits());
+                let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?;
+
+                let width_bits = u128::from(layout_val.size.bits());
                 let shift_bits = raw_shift_bits % width_bits;
                 let inv_shift_bits = (width_bits - shift_bits) % width_bits;
                 let result_bits = if intrinsic_name == sym::rotate_left {
                     (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
                 } else {
                     (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
                 };
-                let truncated_bits = self.truncate(result_bits, layout);
-                let result = Scalar::from_uint(truncated_bits, layout.size);
+                let truncated_bits = self.truncate(result_bits, layout_val);
+                let result = Scalar::from_uint(truncated_bits, layout_val.size);
                 self.write_scalar(result, dest)?;
             }
             sym::copy => {
@@ -472,6 +476,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         name: Symbol,
         val: Scalar<M::Provenance>,
         layout: TyAndLayout<'tcx>,
+        ret_layout: TyAndLayout<'tcx>,
     ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
         assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
         let bits = val.to_bits(layout.size)?;
@@ -483,11 +488,17 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
             sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
             sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
-            sym::bswap => (bits << extra).swap_bytes(),
-            sym::bitreverse => (bits << extra).reverse_bits(),
+            sym::bswap => {
+                assert_eq!(layout, ret_layout);
+                (bits << extra).swap_bytes()
+            }
+            sym::bitreverse => {
+                assert_eq!(layout, ret_layout);
+                (bits << extra).reverse_bits()
+            }
             _ => bug!("not a numeric intrinsic: {}", name),
         };
-        Ok(Scalar::from_uint(bits_out, layout.size))
+        Ok(Scalar::from_uint(bits_out, ret_layout.size))
     }
 
     pub fn exact_div(

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -412,13 +412,11 @@ pub fn check_intrinsic_type(
                 (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], Ty::new_unit(tcx))
             }
 
-            sym::ctpop
-            | sym::ctlz
-            | sym::ctlz_nonzero
-            | sym::cttz
-            | sym::cttz_nonzero
-            | sym::bswap
-            | sym::bitreverse => (1, 0, vec![param(0)], param(0)),
+            sym::ctpop | sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
+                (1, 0, vec![param(0)], tcx.types.u32)
+            }
+
+            sym::bswap | sym::bitreverse => (1, 0, vec![param(0)], param(0)),
 
             sym::three_way_compare => {
                 (1, 0, vec![param(0), param(0)], tcx.ty_ordering_enum(Some(span)))
@@ -461,7 +459,7 @@ pub fn check_intrinsic_type(
                 (1, 0, vec![param(0), param(0)], param(0))
             }
             sym::unchecked_shl | sym::unchecked_shr => (2, 0, vec![param(0), param(1)], param(0)),
-            sym::rotate_left | sym::rotate_right => (1, 0, vec![param(0), param(0)], param(0)),
+            sym::rotate_left | sym::rotate_right => (1, 0, vec![param(0), tcx.types.u32], param(0)),
             sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul => {
                 (1, 0, vec![param(0), param(0)], param(0))
             }

library/core/src/intrinsics.rs

@@ -1987,6 +1987,13 @@ extern "rust-intrinsic" {
     /// The stabilized versions of this intrinsic are available on the integer
     /// primitives via the `count_ones` method. For example,
     /// [`u32::count_ones`]
+    #[cfg(not(bootstrap))]
+    #[rustc_const_stable(feature = "const_ctpop", since = "1.40.0")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn ctpop<T: Copy>(x: T) -> u32;
+
+    #[cfg(bootstrap)]
     #[rustc_const_stable(feature = "const_ctpop", since = "1.40.0")]
     #[rustc_safe_intrinsic]
     #[rustc_nounwind]
@@ -2028,6 +2035,13 @@ extern "rust-intrinsic" {
     /// let num_leading = ctlz(x);
     /// assert_eq!(num_leading, 16);
     /// ```
+    #[cfg(not(bootstrap))]
+    #[rustc_const_stable(feature = "const_ctlz", since = "1.40.0")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn ctlz<T: Copy>(x: T) -> u32;
+
+    #[cfg(bootstrap)]
     #[rustc_const_stable(feature = "const_ctlz", since = "1.40.0")]
     #[rustc_safe_intrinsic]
     #[rustc_nounwind]
@@ -2050,6 +2064,12 @@ extern "rust-intrinsic" {
     /// let num_leading = unsafe { ctlz_nonzero(x) };
     /// assert_eq!(num_leading, 3);
     /// ```
+    #[cfg(not(bootstrap))]
+    #[rustc_const_stable(feature = "constctlz", since = "1.50.0")]
+    #[rustc_nounwind]
+    pub fn ctlz_nonzero<T: Copy>(x: T) -> u32;
+
+    #[cfg(bootstrap)]
     #[rustc_const_stable(feature = "constctlz", since = "1.50.0")]
     #[rustc_nounwind]
     pub fn ctlz_nonzero<T: Copy>(x: T) -> T;
@@ -2090,6 +2110,13 @@ extern "rust-intrinsic" {
     /// let num_trailing = cttz(x);
     /// assert_eq!(num_trailing, 16);
     /// ```
+    #[cfg(not(bootstrap))]
+    #[rustc_const_stable(feature = "const_cttz", since = "1.40.0")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn cttz<T: Copy>(x: T) -> u32;
+
+    #[cfg(bootstrap)]
     #[rustc_const_stable(feature = "const_cttz", since = "1.40.0")]
     #[rustc_safe_intrinsic]
     #[rustc_nounwind]
@@ -2112,6 +2139,12 @@ extern "rust-intrinsic" {
     /// let num_trailing = unsafe { cttz_nonzero(x) };
     /// assert_eq!(num_trailing, 3);
     /// ```
+    #[cfg(not(bootstrap))]
+    #[rustc_const_stable(feature = "const_cttz_nonzero", since = "1.53.0")]
+    #[rustc_nounwind]
+    pub fn cttz_nonzero<T: Copy>(x: T) -> u32;
+
+    #[cfg(bootstrap)]
     #[rustc_const_stable(feature = "const_cttz_nonzero", since = "1.53.0")]
     #[rustc_nounwind]
     pub fn cttz_nonzero<T: Copy>(x: T) -> T;
@@ -2288,6 +2321,13 @@ extern "rust-intrinsic" {
     /// The stabilized versions of this intrinsic are available on the integer
     /// primitives via the `rotate_left` method. For example,
     /// [`u32::rotate_left`]
+    #[cfg(not(bootstrap))]
+    #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn rotate_left<T: Copy>(x: T, shift: u32) -> T;
+
+    #[cfg(bootstrap)]
     #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")]
     #[rustc_safe_intrinsic]
     #[rustc_nounwind]
@@ -2303,6 +2343,13 @@ extern "rust-intrinsic" {
     /// The stabilized versions of this intrinsic are available on the integer
     /// primitives via the `rotate_right` method. For example,
     /// [`u32::rotate_right`]
+    #[cfg(not(bootstrap))]
+    #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn rotate_right<T: Copy>(x: T, shift: u32) -> T;
+
+    #[cfg(bootstrap)]
     #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")]
     #[rustc_safe_intrinsic]
     #[rustc_nounwind]

library/core/src/num/nonzero.rs

@@ -527,7 +527,12 @@ macro_rules! nonzero_integer {
             #[inline]
             pub const fn leading_zeros(self) -> u32 {
                 // SAFETY: since `self` cannot be zero, it is safe to call `ctlz_nonzero`.
-                unsafe { intrinsics::ctlz_nonzero(self.get() as $UnsignedPrimitive) as u32 }
+                unsafe {
+                    #[cfg(not(bootstrap))]
+                    return intrinsics::ctlz_nonzero(self.get() as $UnsignedPrimitive);
+                    #[cfg(bootstrap)]
+                    return intrinsics::ctlz_nonzero(self.get() as $UnsignedPrimitive) as u32;
+                }
             }
 
             /// Returns the number of trailing zeros in the binary representation
@@ -551,7 +556,12 @@ macro_rules! nonzero_integer {
             #[inline]
             pub const fn trailing_zeros(self) -> u32 {
                 // SAFETY: since `self` cannot be zero, it is safe to call `cttz_nonzero`.
-                unsafe { intrinsics::cttz_nonzero(self.get() as $UnsignedPrimitive) as u32 }
+                unsafe {
+                    #[cfg(not(bootstrap))]
+                    return intrinsics::cttz_nonzero(self.get() as $UnsignedPrimitive);
+                    #[cfg(bootstrap)]
+                    return intrinsics::cttz_nonzero(self.get() as $UnsignedPrimitive) as u32;
+                }
             }
 
             /// Returns the number of ones in the binary representation of `self`.