Skip to content

Commit 79a9a71

Browse files
borsbors
committed
Auto merge of #3075 - eduardosm:x86-addcarry-subborrow, r=RalfJung
Move `llvm.x86.*` shims into `shims::x86` and implement `_addcarry_u32` and `_subborrow_u{32,64}` This PR moves all `llvm.x86.*` shims into `shims::x86` and adds `llvm.x86.addcarry.32`, `llvm.x86.subborrow.32` and `llvm.x86.subborrow.64`. Additionally, it fixes the input carry semantics of `llvm.x86.addcarry.32`. The input carry is an 8-bit value that is interpreted as 1 when it is non-zero. https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/2021-8/addcarry-u32-addcarry-u64.html
2 parents 9484a4b + ce3a919 commit 79a9a71

File tree

7 files changed

+1948
-1789
lines changed

7 files changed

+1948
-1789
lines changed

‎src/shims/foreign_items.rs

+7-32
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use rustc_target::{
2222
};
2323

2424
use super::backtrace::EvalContextExt as _;
25-
use crate::helpers::{convert::Truncate, target_os_is_unix};
25+
use crate::helpers::target_os_is_unix;
2626
use crate::*;
2727

2828
/// Returned by `emulate_foreign_item_by_name`.
@@ -981,30 +981,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
981981
throw_unsup_format!("unsupported `llvm.prefetch` type argument: {}", ty);
982982
}
983983
}
984-
"llvm.x86.addcarry.64" if this.tcx.sess.target.arch == "x86_64" => {
985-
// Computes u8+u64+u64, returning tuple (u8,u64) comprising the output carry and truncated sum.
986-
let [c_in, a, b] = this.check_shim(abi, Abi::Unadjusted, link_name, args)?;
987-
let c_in = this.read_scalar(c_in)?.to_u8()?;
988-
let a = this.read_scalar(a)?.to_u64()?;
989-
let b = this.read_scalar(b)?.to_u64()?;
990-
991-
#[allow(clippy::arithmetic_side_effects)]
992-
// adding two u64 and a u8 cannot wrap in a u128
993-
let wide_sum = u128::from(c_in) + u128::from(a) + u128::from(b);
994-
#[allow(clippy::arithmetic_side_effects)] // it's a u128, we can shift by 64
995-
let (c_out, sum) = ((wide_sum >> 64).truncate::<u8>(), wide_sum.truncate::<u64>());
996-
997-
let c_out_field = this.project_field(dest, 0)?;
998-
this.write_scalar(Scalar::from_u8(c_out), &c_out_field)?;
999-
let sum_field = this.project_field(dest, 1)?;
1000-
this.write_scalar(Scalar::from_u64(sum), &sum_field)?;
1001-
}
1002-
"llvm.x86.sse2.pause"
1003-
if this.tcx.sess.target.arch == "x86" || this.tcx.sess.target.arch == "x86_64" =>
1004-
{
1005-
let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
1006-
this.yield_active_thread();
1007-
}
984+
// FIXME: Move these to an `arm` submodule.
1008985
"llvm.aarch64.isb" if this.tcx.sess.target.arch == "aarch64" => {
1009986
let [arg] = this.check_shim(abi, Abi::Unadjusted, link_name, args)?;
1010987
let arg = this.read_scalar(arg)?.to_i32()?;
@@ -1055,13 +1032,11 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
10551032
}
10561033
}
10571034

1058-
name if name.starts_with("llvm.x86.sse.") => {
1059-
return shims::x86::sse::EvalContextExt::emulate_x86_sse_intrinsic(
1060-
this, link_name, abi, args, dest,
1061-
);
1062-
}
1063-
name if name.starts_with("llvm.x86.sse2.") => {
1064-
return shims::x86::sse2::EvalContextExt::emulate_x86_sse2_intrinsic(
1035+
name if name.starts_with("llvm.x86.")
1036+
&& (this.tcx.sess.target.arch == "x86"
1037+
|| this.tcx.sess.target.arch == "x86_64") =>
1038+
{
1039+
return shims::x86::EvalContextExt::emulate_x86_intrinsic(
10651040
this, link_name, abi, args, dest,
10661041
);
10671042
}

‎src/shims/x86/mod.rs

+89-2
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,98 @@
11
use rustc_middle::mir;
2+
use rustc_span::Symbol;
23
use rustc_target::abi::Size;
4+
use rustc_target::spec::abi::Abi;
35

46
use crate::*;
57
use helpers::bool_to_simd_element;
8+
use shims::foreign_items::EmulateByNameResult;
69

7-
pub(super) mod sse;
8-
pub(super) mod sse2;
10+
mod sse;
11+
mod sse2;
12+
13+
impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {}
14+
pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
15+
crate::MiriInterpCxExt<'mir, 'tcx>
16+
{
17+
fn emulate_x86_intrinsic(
18+
&mut self,
19+
link_name: Symbol,
20+
abi: Abi,
21+
args: &[OpTy<'tcx, Provenance>],
22+
dest: &PlaceTy<'tcx, Provenance>,
23+
) -> InterpResult<'tcx, EmulateByNameResult<'mir, 'tcx>> {
24+
let this = self.eval_context_mut();
25+
// Prefix should have already been checked.
26+
let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.").unwrap();
27+
match unprefixed_name {
28+
// Used to implement the `_addcarry_u32` and `_addcarry_u64` functions.
29+
// Computes a + b with input and output carry. The input carry is an 8-bit
30+
// value, which is interpreted as 1 if it is non-zero. The output carry is
31+
// an 8-bit value that will be 0 or 1.
32+
// https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/2021-8/addcarry-u32-addcarry-u64.html
33+
"addcarry.32" | "addcarry.64" => {
34+
if unprefixed_name == "addcarry.64" && this.tcx.sess.target.arch != "x86_64" {
35+
return Ok(EmulateByNameResult::NotSupported);
36+
}
37+
38+
let [c_in, a, b] = this.check_shim(abi, Abi::Unadjusted, link_name, args)?;
39+
let c_in = this.read_scalar(c_in)?.to_u8()? != 0;
40+
let a = this.read_immediate(a)?;
41+
let b = this.read_immediate(b)?;
42+
43+
let (sum, overflow1) = this.overflowing_binary_op(mir::BinOp::Add, &a, &b)?;
44+
let (sum, overflow2) = this.overflowing_binary_op(
45+
mir::BinOp::Add,
46+
&sum,
47+
&ImmTy::from_uint(c_in, a.layout),
48+
)?;
49+
let c_out = overflow1 | overflow2;
50+
51+
this.write_scalar(Scalar::from_u8(c_out.into()), &this.project_field(dest, 0)?)?;
52+
this.write_immediate(*sum, &this.project_field(dest, 1)?)?;
53+
}
54+
// Used to implement the `_subborrow_u32` and `_subborrow_u64` functions.
55+
// Computes a - b with input and output borrow. The input borrow is an 8-bit
56+
// value, which is interpreted as 1 if it is non-zero. The output borrow is
57+
// an 8-bit value that will be 0 or 1.
58+
// https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/2021-8/subborrow-u32-subborrow-u64.html
59+
"subborrow.32" | "subborrow.64" => {
60+
if unprefixed_name == "subborrow.64" && this.tcx.sess.target.arch != "x86_64" {
61+
return Ok(EmulateByNameResult::NotSupported);
62+
}
63+
64+
let [b_in, a, b] = this.check_shim(abi, Abi::Unadjusted, link_name, args)?;
65+
let b_in = this.read_scalar(b_in)?.to_u8()? != 0;
66+
let a = this.read_immediate(a)?;
67+
let b = this.read_immediate(b)?;
68+
69+
let (sub, overflow1) = this.overflowing_binary_op(mir::BinOp::Sub, &a, &b)?;
70+
let (sub, overflow2) = this.overflowing_binary_op(
71+
mir::BinOp::Sub,
72+
&sub,
73+
&ImmTy::from_uint(b_in, a.layout),
74+
)?;
75+
let b_out = overflow1 | overflow2;
76+
77+
this.write_scalar(Scalar::from_u8(b_out.into()), &this.project_field(dest, 0)?)?;
78+
this.write_immediate(*sub, &this.project_field(dest, 1)?)?;
79+
}
80+
81+
name if name.starts_with("sse.") => {
82+
return sse::EvalContextExt::emulate_x86_sse_intrinsic(
83+
this, link_name, abi, args, dest,
84+
);
85+
}
86+
name if name.starts_with("sse2.") => {
87+
return sse2::EvalContextExt::emulate_x86_sse2_intrinsic(
88+
this, link_name, abi, args, dest,
89+
);
90+
}
91+
_ => return Ok(EmulateByNameResult::NotSupported),
92+
}
93+
Ok(EmulateByNameResult::NeedsJumping)
94+
}
95+
}
996

1097
/// Floating point comparison operation
1198
///

‎src/shims/x86/sse.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ use crate::*;
1010
use shims::foreign_items::EmulateByNameResult;
1111

1212
impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {}
13-
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
13+
pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
14+
crate::MiriInterpCxExt<'mir, 'tcx>
15+
{
1416
fn emulate_x86_sse_intrinsic(
1517
&mut self,
1618
link_name: Symbol,

‎src/shims/x86/sse2.rs

+9-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ use crate::*;
1313
use shims::foreign_items::EmulateByNameResult;
1414

1515
impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {}
16-
pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
16+
pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
17+
crate::MiriInterpCxExt<'mir, 'tcx>
18+
{
1719
fn emulate_x86_sse2_intrinsic(
1820
&mut self,
1921
link_name: Symbol,
@@ -753,6 +755,12 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
753755

754756
this.write_scalar(Scalar::from_u32(res.try_into().unwrap()), dest)?;
755757
}
758+
// Used to implement the `_mm_pause` function.
759+
// The intrinsic is used to hint the processor that the code is in a spin-loop.
760+
"pause" => {
761+
let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
762+
this.yield_active_thread();
763+
}
756764
_ => return Ok(EmulateByNameResult::NotSupported),
757765
}
758766
Ok(EmulateByNameResult::NeedsJumping)

0 commit comments

Comments
 (0)