Skip to content

Commit d89f307

Browse files
bjorn3bjorn3
committed
Move simd intrinsics to intrinsics/simd.rs
1 parent 67a0fda commit d89f307

File tree

2 files changed

+221
-196
lines changed

2 files changed

+221
-196
lines changed

‎src/intrinsics/mod.rs

+50-196
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
pub mod llvm;
2+
mod simd;
23

34
use crate::prelude::*;
45

@@ -70,7 +71,7 @@ macro intrinsic_match {
7071
}
7172
}
7273

73-
macro_rules! call_intrinsic_match {
74+
macro call_intrinsic_match {
7475
($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $destination:expr, $args:expr, $(
7576
$name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
7677
)*) => {
@@ -102,32 +103,28 @@ macro_rules! call_intrinsic_match {
102103
}
103104
}
104105

105-
macro_rules! atomic_binop_return_old {
106-
($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) => {
107-
let clif_ty = $fx.clif_type($T).unwrap();
108-
let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
109-
let new = $fx.bcx.ins().$op(old, $src);
110-
$fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
111-
$ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
112-
};
106+
macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) {
107+
let clif_ty = $fx.clif_type($T).unwrap();
108+
let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
109+
let new = $fx.bcx.ins().$op(old, $src);
110+
$fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
111+
$ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
113112
}
114113

115-
macro_rules! atomic_minmax {
116-
($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) => {
117-
// Read old
118-
let clif_ty = $fx.clif_type($T).unwrap();
119-
let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
114+
macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) {
115+
// Read old
116+
let clif_ty = $fx.clif_type($T).unwrap();
117+
let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
120118

121-
// Compare
122-
let is_eq = codegen_icmp($fx, IntCC::SignedGreaterThan, old, $src);
123-
let new = $fx.bcx.ins().select(is_eq, old, $src);
119+
// Compare
120+
let is_eq = codegen_icmp($fx, IntCC::SignedGreaterThan, old, $src);
121+
let new = $fx.bcx.ins().select(is_eq, old, $src);
124122

125-
// Write new
126-
$fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
123+
// Write new
124+
$fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
127125

128-
let ret_val = CValue::by_val(old, $ret.layout());
129-
$ret.write_cvalue($fx, ret_val);
130-
};
126+
let ret_val = CValue::by_val(old, $ret.layout());
127+
$ret.write_cvalue($fx, ret_val);
131128
}
132129

133130
fn lane_type_and_count<'tcx>(
@@ -206,7 +203,7 @@ fn bool_to_zero_or_max_uint<'tcx>(
206203
CValue::by_val(res, layout)
207204
}
208205

209-
macro_rules! simd_cmp {
206+
macro simd_cmp {
210207
($fx:expr, $intrinsic:expr, $cc:ident($x:ident, $y:ident) -> $ret:ident) => {
211208
simd_for_each_lane(
212209
$fx,
@@ -222,7 +219,7 @@ macro_rules! simd_cmp {
222219
bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
223220
},
224221
);
225-
};
222+
},
226223
($fx:expr, $intrinsic:expr, $cc_u:ident|$cc_s:ident($x:ident, $y:ident) -> $ret:ident) => {
227224
simd_for_each_lane(
228225
$fx,
@@ -239,10 +236,10 @@ macro_rules! simd_cmp {
239236
bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
240237
},
241238
);
242-
};
239+
},
243240
}
244241

245-
macro_rules! simd_int_binop {
242+
macro simd_int_binop {
246243
($fx:expr, $intrinsic:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
247244
simd_for_each_lane(
248245
$fx,
@@ -258,7 +255,7 @@ macro_rules! simd_int_binop {
258255
CValue::by_val(res_lane, ret_lane_layout)
259256
},
260257
);
261-
};
258+
},
262259
($fx:expr, $intrinsic:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => {
263260
simd_for_each_lane(
264261
$fx,
@@ -275,10 +272,10 @@ macro_rules! simd_int_binop {
275272
CValue::by_val(res_lane, ret_lane_layout)
276273
},
277274
);
278-
};
275+
},
279276
}
280277

281-
macro_rules! simd_int_flt_binop {
278+
macro simd_int_flt_binop {
282279
($fx:expr, $intrinsic:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
283280
simd_for_each_lane(
284281
$fx,
@@ -295,7 +292,7 @@ macro_rules! simd_int_flt_binop {
295292
CValue::by_val(res_lane, ret_lane_layout)
296293
},
297294
);
298-
};
295+
},
299296
($fx:expr, $intrinsic:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
300297
simd_for_each_lane(
301298
$fx,
@@ -313,26 +310,24 @@ macro_rules! simd_int_flt_binop {
313310
CValue::by_val(res_lane, ret_lane_layout)
314311
},
315312
);
316-
};
313+
},
317314
}
318315

319-
macro_rules! simd_flt_binop {
320-
($fx:expr, $intrinsic:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
321-
simd_for_each_lane(
322-
$fx,
323-
$intrinsic,
324-
$x,
325-
$y,
326-
$ret,
327-
|fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
328-
let res_lane = match lane_layout.ty.kind {
329-
ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
330-
_ => unreachable!("{:?}", lane_layout.ty),
331-
};
332-
CValue::by_val(res_lane, ret_lane_layout)
333-
},
334-
);
335-
};
316+
macro simd_flt_binop($fx:expr, $intrinsic:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
317+
simd_for_each_lane(
318+
$fx,
319+
$intrinsic,
320+
$x,
321+
$y,
322+
$ret,
323+
|fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
324+
let res_lane = match lane_layout.ty.kind {
325+
ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
326+
_ => unreachable!("{:?}", lane_layout.ty),
327+
};
328+
CValue::by_val(res_lane, ret_lane_layout)
329+
},
330+
);
336331
}
337332

338333
pub fn codegen_intrinsic_call<'tcx>(
@@ -371,6 +366,13 @@ pub fn codegen_intrinsic_call<'tcx>(
371366
}
372367
};
373368

369+
if intrinsic.starts_with("simd_") {
370+
self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span);
371+
let ret_ebb = fx.get_ebb(destination.expect("SIMD intrinsics don't diverge").1);
372+
fx.bcx.ins().jump(ret_ebb, &[]);
373+
return;
374+
}
375+
374376
let usize_layout = fx.layout_of(fx.tcx.types.usize);
375377

376378
call_intrinsic_match! {
@@ -944,154 +946,6 @@ pub fn codegen_intrinsic_call<'tcx>(
944946
ret.write_cvalue(fx, val);
945947
};
946948

947-
simd_cast, (c a) {
948-
let (lane_layout, lane_count) = lane_type_and_count(fx, a.layout(), intrinsic);
949-
let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx, ret.layout(), intrinsic);
950-
assert_eq!(lane_count, ret_lane_count);
951-
952-
let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
953-
954-
let from_signed = type_sign(lane_layout.ty);
955-
let to_signed = type_sign(ret_lane_layout.ty);
956-
957-
for lane in 0..lane_count {
958-
let lane = mir::Field::new(lane.try_into().unwrap());
959-
960-
let a_lane = a.value_field(fx, lane).load_scalar(fx);
961-
let res = clif_int_or_float_cast(fx, a_lane, from_signed, ret_lane_ty, to_signed);
962-
ret.place_field(fx, lane).write_cvalue(fx, CValue::by_val(res, ret_lane_layout));
963-
}
964-
};
965-
966-
simd_eq, (c x, c y) {
967-
simd_cmp!(fx, intrinsic, Equal(x, y) -> ret);
968-
};
969-
simd_ne, (c x, c y) {
970-
simd_cmp!(fx, intrinsic, NotEqual(x, y) -> ret);
971-
};
972-
simd_lt, (c x, c y) {
973-
simd_cmp!(fx, intrinsic, UnsignedLessThan|SignedLessThan(x, y) -> ret);
974-
};
975-
simd_le, (c x, c y) {
976-
simd_cmp!(fx, intrinsic, UnsignedLessThanOrEqual|SignedLessThanOrEqual(x, y) -> ret);
977-
};
978-
simd_gt, (c x, c y) {
979-
simd_cmp!(fx, intrinsic, UnsignedGreaterThan|SignedGreaterThan(x, y) -> ret);
980-
};
981-
simd_ge, (c x, c y) {
982-
simd_cmp!(fx, intrinsic, UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual(x, y) -> ret);
983-
};
984-
985-
// simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
986-
_ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) {
987-
let n: u32 = intrinsic["simd_shuffle".len()..].parse().unwrap();
988-
989-
assert_eq!(x.layout(), y.layout());
990-
let layout = x.layout();
991-
992-
let (lane_type, lane_count) = lane_type_and_count(fx, layout, intrinsic);
993-
let (ret_lane_type, ret_lane_count) = lane_type_and_count(fx, ret.layout(), intrinsic);
994-
995-
assert_eq!(lane_type, ret_lane_type);
996-
assert_eq!(n, ret_lane_count);
997-
998-
let total_len = lane_count * 2;
999-
1000-
let indexes = {
1001-
use rustc::mir::interpret::*;
1002-
let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
1003-
1004-
let idx_bytes = match idx_const.val {
1005-
ty::ConstKind::Value(ConstValue::ByRef { alloc, offset }) => {
1006-
let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
1007-
let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
1008-
alloc.get_bytes(fx, ptr, size).unwrap()
1009-
}
1010-
_ => unreachable!("{:?}", idx_const),
1011-
};
1012-
1013-
(0..ret_lane_count).map(|i| {
1014-
let i = usize::try_from(i).unwrap();
1015-
let idx = rustc::mir::interpret::read_target_uint(
1016-
fx.tcx.data_layout.endian,
1017-
&idx_bytes[4*i.. 4*i + 4],
1018-
).expect("read_target_uint");
1019-
u32::try_from(idx).expect("try_from u32")
1020-
}).collect::<Vec<u32>>()
1021-
};
1022-
1023-
for &idx in &indexes {
1024-
assert!(idx < total_len, "idx {} out of range 0..{}", idx, total_len);
1025-
}
1026-
1027-
for (out_idx, in_idx) in indexes.into_iter().enumerate() {
1028-
let in_lane = if in_idx < lane_count {
1029-
x.value_field(fx, mir::Field::new(in_idx.try_into().unwrap()))
1030-
} else {
1031-
y.value_field(fx, mir::Field::new((in_idx - lane_count).try_into().unwrap()))
1032-
};
1033-
let out_lane = ret.place_field(fx, mir::Field::new(out_idx));
1034-
out_lane.write_cvalue(fx, in_lane);
1035-
}
1036-
};
1037-
1038-
simd_extract, (c v, o idx) {
1039-
let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
1040-
idx_const
1041-
} else {
1042-
fx.tcx.sess.span_warn(
1043-
fx.mir.span,
1044-
"`#[rustc_arg_required_const(..)]` is not yet supported. Calling this function will panic.",
1045-
);
1046-
crate::trap::trap_panic(fx, "`#[rustc_arg_required_const(..)]` is not yet supported.");
1047-
return;
1048-
};
1049-
1050-
let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).expect(&format!("kind not scalar: {:?}", idx_const));
1051-
let (_lane_type, lane_count) = lane_type_and_count(fx, v.layout(), intrinsic);
1052-
if idx >= lane_count.into() {
1053-
fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
1054-
}
1055-
1056-
let ret_lane = v.value_field(fx, mir::Field::new(idx.try_into().unwrap()));
1057-
ret.write_cvalue(fx, ret_lane);
1058-
};
1059-
1060-
simd_add, (c x, c y) {
1061-
simd_int_flt_binop!(fx, intrinsic, iadd|fadd(x, y) -> ret);
1062-
};
1063-
simd_sub, (c x, c y) {
1064-
simd_int_flt_binop!(fx, intrinsic, isub|fsub(x, y) -> ret);
1065-
};
1066-
simd_mul, (c x, c y) {
1067-
simd_int_flt_binop!(fx, intrinsic, imul|fmul(x, y) -> ret);
1068-
};
1069-
simd_div, (c x, c y) {
1070-
simd_int_flt_binop!(fx, intrinsic, udiv|sdiv|fdiv(x, y) -> ret);
1071-
};
1072-
simd_shl, (c x, c y) {
1073-
simd_int_binop!(fx, intrinsic, ishl(x, y) -> ret);
1074-
};
1075-
simd_shr, (c x, c y) {
1076-
simd_int_binop!(fx, intrinsic, ushr|sshr(x, y) -> ret);
1077-
};
1078-
simd_and, (c x, c y) {
1079-
simd_int_binop!(fx, intrinsic, band(x, y) -> ret);
1080-
};
1081-
simd_or, (c x, c y) {
1082-
simd_int_binop!(fx, intrinsic, bor(x, y) -> ret);
1083-
};
1084-
simd_xor, (c x, c y) {
1085-
simd_int_binop!(fx, intrinsic, bxor(x, y) -> ret);
1086-
};
1087-
1088-
simd_fmin, (c x, c y) {
1089-
simd_flt_binop!(fx, intrinsic, fmin(x, y) -> ret);
1090-
};
1091-
simd_fmax, (c x, c y) {
1092-
simd_flt_binop!(fx, intrinsic, fmax(x, y) -> ret);
1093-
};
1094-
1095949
try, (v f, v data, v _local_ptr) {
1096950
// FIXME once unwinding is supported, change this to actually catch panics
1097951
let f_sig = fx.bcx.func.import_signature(Signature {

0 commit comments

Comments
 (0)