1 //! Emulate LLVM intrinsics
3 use crate::intrinsics::*;
6 use rustc_middle::ty::subst::SubstsRef;
8 pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
9 fx: &mut FunctionCx<'_, '_, 'tcx>,
11 _substs: SubstsRef<'tcx>,
12 args: &[mir::Operand<'tcx>],
14 target: Option<BasicBlock>,
17 "llvm.x86.sse2.pause" | "llvm.aarch64.isb" => {
21 // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
22 "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd" => {
23 intrinsic_args!(fx, args => (a); intrinsic);
25 let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
26 let lane_ty = fx.clif_type(lane_ty).unwrap();
27 assert!(lane_count <= 32);
29 let mut res = fx.bcx.ins().iconst(types::I32, 0);
31 for lane in (0..lane_count).rev() {
32 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
35 let a_lane = match lane_ty {
36 types::F32 => fx.bcx.ins().bitcast(types::I32, a_lane),
37 types::F64 => fx.bcx.ins().bitcast(types::I64, a_lane),
41 // extract sign bit of an int
42 let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_ty.bits() - 1));
44 // shift sign bit into result
45 let a_lane_sign = clif_intcast(fx, a_lane_sign, types::I32, false);
46 res = fx.bcx.ins().ishl_imm(res, 1);
47 res = fx.bcx.ins().bor(res, a_lane_sign);
50 let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
51 ret.write_cvalue(fx, res);
53 "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
54 let (x, y, kind) = match args {
55 [x, y, kind] => (x, y, kind),
56 _ => bug!("wrong number of args for intrinsic {intrinsic}"),
58 let x = codegen_operand(fx, x);
59 let y = codegen_operand(fx, y);
60 let kind = crate::constant::mir_operand_get_const_val(fx, kind)
61 .expect("llvm.x86.sse2.cmp.* kind not const");
63 let flt_cc = match kind
64 .try_to_bits(Size::from_bytes(1))
65 .unwrap_or_else(|| panic!("kind not scalar: {:?}", kind))
68 1 => FloatCC::LessThan,
69 2 => FloatCC::LessThanOrEqual,
70 7 => FloatCC::Ordered,
71 3 => FloatCC::Unordered,
72 4 => FloatCC::NotEqual,
73 5 => FloatCC::UnorderedOrGreaterThanOrEqual,
74 6 => FloatCC::UnorderedOrGreaterThan,
75 kind => unreachable!("kind {:?}", kind),
78 simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
79 let res_lane = match lane_ty.kind() {
80 ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
81 _ => unreachable!("{:?}", lane_ty),
83 bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane)
86 "llvm.x86.sse2.psrli.d" => {
87 let (a, imm8) = match args {
88 [a, imm8] => (a, imm8),
89 _ => bug!("wrong number of args for intrinsic {intrinsic}"),
91 let a = codegen_operand(fx, a);
92 let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
93 .expect("llvm.x86.sse2.psrli.d imm8 not const");
95 simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
96 .try_to_bits(Size::from_bytes(4))
97 .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
99 imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
100 _ => fx.bcx.ins().iconst(types::I32, 0),
103 "llvm.x86.sse2.pslli.d" => {
104 let (a, imm8) = match args {
105 [a, imm8] => (a, imm8),
106 _ => bug!("wrong number of args for intrinsic {intrinsic}"),
108 let a = codegen_operand(fx, a);
109 let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
110 .expect("llvm.x86.sse2.psrli.d imm8 not const");
112 simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
113 .try_to_bits(Size::from_bytes(4))
114 .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
116 imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
117 _ => fx.bcx.ins().iconst(types::I32, 0),
120 "llvm.x86.sse2.storeu.dq" => {
121 intrinsic_args!(fx, args => (mem_addr, a); intrinsic);
122 let mem_addr = mem_addr.load_scalar(fx);
124 // FIXME correctly handle the unalignment
125 let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
126 dest.write_cvalue(fx, a);
128 "llvm.x86.addcarry.64" => {
129 intrinsic_args!(fx, args => (c_in, a, b); intrinsic);
130 let c_in = c_in.load_scalar(fx);
132 llvm_add_sub(fx, BinOp::Add, ret, c_in, a, b);
134 "llvm.x86.subborrow.64" => {
135 intrinsic_args!(fx, args => (b_in, a, b); intrinsic);
136 let b_in = b_in.load_scalar(fx);
138 llvm_add_sub(fx, BinOp::Sub, ret, b_in, a, b);
143 .warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
144 crate::trap::trap_unimplemented(fx, intrinsic);
149 let dest = target.expect("all llvm intrinsics used by stdlib should return");
150 let ret_block = fx.get_block(dest);
151 fx.bcx.ins().jump(ret_block, &[]);
154 // llvm.x86.avx2.vperm2i128
155 // llvm.x86.ssse3.pshuf.b.128
156 // llvm.x86.avx2.pshuf.b
157 // llvm.x86.avx2.psrli.w
158 // llvm.x86.sse2.psrli.w
160 fn llvm_add_sub<'tcx>(
161 fx: &mut FunctionCx<'_, '_, 'tcx>,
171 "llvm.x86.addcarry.64/llvm.x86.subborrow.64 second operand must be u64"
176 "llvm.x86.addcarry.64/llvm.x86.subborrow.64 third operand must be u64"
179 // c + carry -> c + first intermediate carry or borrow respectively
180 let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b);
181 let c = int0.value_field(fx, mir::Field::new(0));
182 let cb0 = int0.value_field(fx, mir::Field::new(1)).load_scalar(fx);
184 // c + carry -> c + second intermediate carry or borrow respectively
185 let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in);
186 let cb_in_as_u64 = CValue::by_val(cb_in_as_u64, fx.layout_of(fx.tcx.types.u64));
187 let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_u64);
188 let (c, cb1) = int1.load_scalar_pair(fx);
190 // carry0 | carry1 -> carry or borrow respectively
191 let cb_out = fx.bcx.ins().bor(cb0, cb1);
193 let layout = fx.layout_of(fx.tcx.mk_tup([fx.tcx.types.u8, fx.tcx.types.u64].iter()));
194 let val = CValue::by_val_pair(cb_out, c, layout);
195 ret.write_cvalue(fx, val);