1 //! Emulate LLVM intrinsics
3 use crate::intrinsics::*;
6 use rustc_middle::ty::subst::SubstsRef;
8 pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
9 fx: &mut FunctionCx<'_, '_, 'tcx>,
11 substs: SubstsRef<'tcx>,
12 args: &[mir::Operand<'tcx>],
13 destination: Option<(CPlace<'tcx>, BasicBlock)>,
15 let ret = destination.unwrap().0;
18 fx, intrinsic, substs, args,
20 fx.tcx.sess.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
21 crate::trap::trap_unimplemented(fx, intrinsic);
24 // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
25 "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd", (c a) {
26 let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
27 let lane_ty = fx.clif_type(lane_ty).unwrap();
28 assert!(lane_count <= 32);
30 let mut res = fx.bcx.ins().iconst(types::I32, 0);
32 for lane in (0..lane_count).rev() {
33 let a_lane = a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx);
36 let a_lane = match lane_ty {
37 types::F32 => fx.bcx.ins().bitcast(types::I32, a_lane),
38 types::F64 => fx.bcx.ins().bitcast(types::I64, a_lane),
42 // extract sign bit of an int
43 let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_ty.bits() - 1));
45 // shift sign bit into result
46 let a_lane_sign = clif_intcast(fx, a_lane_sign, types::I32, false);
47 res = fx.bcx.ins().ishl_imm(res, 1);
48 res = fx.bcx.ins().bor(res, a_lane_sign);
51 let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
52 ret.write_cvalue(fx, res);
54 "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) {
55 let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
56 let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) {
58 1 => FloatCC::LessThan,
59 2 => FloatCC::LessThanOrEqual,
61 unimplemented!("Compares corresponding elements in `a` and `b` to see if neither is `NaN`.");
64 unimplemented!("Compares corresponding elements in `a` and `b` to see if either is `NaN`.");
66 4 => FloatCC::NotEqual,
68 unimplemented!("not less than");
71 unimplemented!("not less than or equal");
73 kind => unreachable!("kind {:?}", kind),
76 simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
77 let res_lane = match lane_layout.ty.kind() {
78 ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
79 _ => unreachable!("{:?}", lane_layout.ty),
81 bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
84 "llvm.x86.sse2.psrli.d", (c a, o imm8) {
85 let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
86 simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
87 let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
88 imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
89 _ => fx.bcx.ins().iconst(types::I32, 0),
91 CValue::by_val(res_lane, res_lane_layout)
94 "llvm.x86.sse2.pslli.d", (c a, o imm8) {
95 let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
96 simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
97 let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
98 imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
99 _ => fx.bcx.ins().iconst(types::I32, 0),
101 CValue::by_val(res_lane, res_lane_layout)
104 "llvm.x86.sse2.storeu.dq", (v mem_addr, c a) {
105 // FIXME correctly handle the unalignment
106 let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
107 dest.write_cvalue(fx, a);
109 "llvm.x86.addcarry.64", (v c_in, c a, c b) {
119 "llvm.x86.subborrow.64", (v b_in, c a, c b) {
131 if let Some((_, dest)) = destination {
132 let ret_block = fx.get_block(dest);
133 fx.bcx.ins().jump(ret_block, &[]);
135 trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
139 // llvm.x86.avx2.vperm2i128
140 // llvm.x86.ssse3.pshuf.b.128
141 // llvm.x86.avx2.pshuf.b
142 // llvm.x86.avx2.psrli.w
143 // llvm.x86.sse2.psrli.w
145 fn llvm_add_sub<'tcx>(
146 fx: &mut FunctionCx<'_, '_, 'tcx>,
156 "llvm.x86.addcarry.64/llvm.x86.subborrow.64 second operand must be u64"
161 "llvm.x86.addcarry.64/llvm.x86.subborrow.64 third operand must be u64"
164 // c + carry -> c + first intermediate carry or borrow respectively
165 let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b);
166 let c = int0.value_field(fx, mir::Field::new(0));
167 let cb0 = int0.value_field(fx, mir::Field::new(1)).load_scalar(fx);
169 // c + carry -> c + second intermediate carry or borrow respectively
170 let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in);
171 let cb_in_as_u64 = CValue::by_val(cb_in_as_u64, fx.layout_of(fx.tcx.types.u64));
172 let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_u64);
173 let (c, cb1) = int1.load_scalar_pair(fx);
175 // carry0 | carry1 -> carry or borrow respectively
176 let cb_out = fx.bcx.ins().bor(cb0, cb1);
178 let layout = fx.layout_of(fx.tcx.mk_tup([fx.tcx.types.u8, fx.tcx.types.u64].iter()));
179 let val = CValue::by_val_pair(cb_out, c, layout);
180 ret.write_cvalue(fx, val);