1 //! Codegen `extern "platform-intrinsic"` intrinsics.
3 use rustc_middle::ty::subst::SubstsRef;
4 use rustc_span::Symbol;
9 fn report_simd_type_validation_error(
10 fx: &mut FunctionCx<'_, '_, '_>,
15 fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
16 // Prevent verifier error
17 crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
20 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
21 fx: &mut FunctionCx<'_, '_, 'tcx>,
23 _substs: SubstsRef<'tcx>,
24 args: &[mir::Operand<'tcx>],
31 fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
35 if !a.layout().ty.is_simd() {
36 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
40 simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
41 let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
43 let from_signed = type_sign(lane_ty);
44 let to_signed = type_sign(ret_lane_ty);
46 clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
50 simd_eq | simd_ne | simd_lt | simd_le | simd_gt | simd_ge, (c x, c y) {
51 if !x.layout().ty.is_simd() {
52 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
56 // FIXME use vector instructions when possible
57 simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
58 let res_lane = match (lane_ty.kind(), intrinsic) {
59 (ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
60 (ty::Uint(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
61 (ty::Uint(_), sym::simd_lt) => {
62 fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
64 (ty::Uint(_), sym::simd_le) => {
65 fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane)
67 (ty::Uint(_), sym::simd_gt) => {
68 fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane)
70 (ty::Uint(_), sym::simd_ge) => {
71 fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane)
74 (ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
75 (ty::Int(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
76 (ty::Int(_), sym::simd_lt) => fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane),
77 (ty::Int(_), sym::simd_le) => {
78 fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
80 (ty::Int(_), sym::simd_gt) => {
81 fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane)
83 (ty::Int(_), sym::simd_ge) => {
84 fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
87 (ty::Float(_), sym::simd_eq) => fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane),
88 (ty::Float(_), sym::simd_ne) => fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane),
89 (ty::Float(_), sym::simd_lt) => fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane),
90 (ty::Float(_), sym::simd_le) => {
91 fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
93 (ty::Float(_), sym::simd_gt) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane),
94 (ty::Float(_), sym::simd_ge) => {
95 fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
101 let ty = fx.clif_type(res_lane_ty).unwrap();
103 let res_lane = fx.bcx.ins().bint(ty, res_lane);
104 fx.bcx.ins().ineg(res_lane)
108 // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
109 _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
110 if !x.layout().ty.is_simd() {
111 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
115 // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
116 // If there is no suffix, use the index array length.
117 let n: u16 = if intrinsic == sym::simd_shuffle {
118 // Make sure this is actually an array, since typeck only checks the length-suffixed
119 // version of this intrinsic.
120 let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
121 match idx_ty.kind() {
122 ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
123 len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
124 span_bug!(span, "could not evaluate shuffle index array length")
125 }).try_into().unwrap()
128 fx.tcx.sess.span_err(
131 "simd_shuffle index must be an array of `u32`, got `{}`",
135 // Prevent verifier error
136 crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
141 intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
144 assert_eq!(x.layout(), y.layout());
145 let layout = x.layout();
147 let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
148 let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
150 assert_eq!(lane_ty, ret_lane_ty);
151 assert_eq!(u64::from(n), ret_lane_count);
153 let total_len = lane_count * 2;
156 use rustc_middle::mir::interpret::*;
157 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
159 let idx_bytes = match idx_const {
160 ConstValue::ByRef { alloc, offset } => {
161 let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
162 alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap()
164 _ => unreachable!("{:?}", idx_const),
167 (0..ret_lane_count).map(|i| {
168 let i = usize::try_from(i).unwrap();
169 let idx = rustc_middle::mir::interpret::read_target_uint(
170 fx.tcx.data_layout.endian,
171 &idx_bytes[4*i.. 4*i + 4],
172 ).expect("read_target_uint");
173 u16::try_from(idx).expect("try_from u32")
174 }).collect::<Vec<u16>>()
177 for &idx in &indexes {
178 assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
181 for (out_idx, in_idx) in indexes.into_iter().enumerate() {
182 let in_lane = if u64::from(in_idx) < lane_count {
183 x.value_lane(fx, in_idx.into())
185 y.value_lane(fx, u64::from(in_idx) - lane_count)
187 let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap());
188 out_lane.write_cvalue(fx, in_lane);
192 simd_insert, (c base, o idx, c val) {
194 let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
197 fx.tcx.sess.span_fatal(
199 "Index argument for `simd_insert` is not a constant",
203 let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
204 let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
205 if idx >= lane_count.into() {
206 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
209 ret.write_cvalue(fx, base);
210 let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
211 ret_lane.write_cvalue(fx, val);
214 simd_extract, (c v, o idx) {
215 if !v.layout().ty.is_simd() {
216 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
220 let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
223 fx.tcx.sess.span_warn(
225 "Index argument for `simd_extract` is not a constant",
227 let res = crate::trap::trap_unimplemented_ret_value(
230 "Index argument for `simd_extract` is not a constant",
232 ret.write_cvalue(fx, res);
236 let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
237 let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
238 if idx >= lane_count.into() {
239 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
242 let ret_lane = v.value_lane(fx, idx.try_into().unwrap());
243 ret.write_cvalue(fx, ret_lane);
247 if !a.layout().ty.is_simd() {
248 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
252 simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
253 match lane_ty.kind() {
254 ty::Int(_) => fx.bcx.ins().ineg(lane),
255 ty::Float(_) => fx.bcx.ins().fneg(lane),
261 simd_add | simd_sub | simd_mul | simd_div | simd_rem
262 | simd_shl | simd_shr | simd_and | simd_or | simd_xor, (c x, c y) {
263 if !x.layout().ty.is_simd() {
264 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
268 // FIXME use vector instructions when possible
269 simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| match (
273 (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
274 (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
275 (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
276 (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
277 (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
279 (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
280 (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
281 (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
282 (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
283 (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
285 (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
286 (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
287 (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
288 (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
289 (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
291 vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
292 vec![AbiParam::new(types::F32)],
295 (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
297 vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
298 vec![AbiParam::new(types::F64)],
302 (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
303 (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
304 (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
305 (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
306 (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
308 (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
309 (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
310 (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
311 (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
312 (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
318 simd_fma, (c a, c b, c c) {
319 if !a.layout().ty.is_simd() {
320 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
323 assert_eq!(a.layout(), b.layout());
324 assert_eq!(a.layout(), c.layout());
325 assert_eq!(a.layout(), ret.layout());
327 let layout = a.layout();
328 let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
330 for lane in 0..lane_count {
331 let a_lane = a.value_lane(fx, lane);
332 let b_lane = b.value_lane(fx, lane);
333 let c_lane = c.value_lane(fx, lane);
335 let res_lane = match lane_ty.kind() {
336 ty::Float(FloatTy::F32) => fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty),
337 ty::Float(FloatTy::F64) => fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty),
341 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
345 simd_fmin | simd_fmax, (c x, c y) {
346 if !x.layout().ty.is_simd() {
347 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
351 // FIXME use vector instructions when possible
352 simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
353 match lane_ty.kind() {
355 _ => unreachable!("{:?}", lane_ty),
358 sym::simd_fmin => crate::num::codegen_float_min(fx, x_lane, y_lane),
359 sym::simd_fmax => crate::num::codegen_float_max(fx, x_lane, y_lane),
366 if !a.layout().ty.is_simd() {
367 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
371 simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
372 match lane_ty.kind() {
373 ty::Float(FloatTy::F32) => fx.lib_call(
375 vec![AbiParam::new(types::F32)],
376 vec![AbiParam::new(types::F32)],
379 ty::Float(FloatTy::F64) => fx.lib_call(
381 vec![AbiParam::new(types::F64)],
382 vec![AbiParam::new(types::F64)],
385 _ => unreachable!("{:?}", lane_ty),
390 simd_fabs | simd_fsqrt | simd_ceil | simd_floor | simd_trunc, (c a) {
391 if !a.layout().ty.is_simd() {
392 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
396 simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
397 match lane_ty.kind() {
399 _ => unreachable!("{:?}", lane_ty),
402 sym::simd_fabs => fx.bcx.ins().fabs(lane),
403 sym::simd_fsqrt => fx.bcx.ins().sqrt(lane),
404 sym::simd_ceil => fx.bcx.ins().ceil(lane),
405 sym::simd_floor => fx.bcx.ins().floor(lane),
406 sym::simd_trunc => fx.bcx.ins().trunc(lane),
412 simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
413 // FIXME there must be no acc param for integer vectors
414 if !v.layout().ty.is_simd() {
415 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
419 simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
420 if lane_ty.is_floating_point() {
421 fx.bcx.ins().fadd(a, b)
423 fx.bcx.ins().iadd(a, b)
428 simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
429 // FIXME there must be no acc param for integer vectors
430 if !v.layout().ty.is_simd() {
431 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
435 simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
436 if lane_ty.is_floating_point() {
437 fx.bcx.ins().fmul(a, b)
439 fx.bcx.ins().imul(a, b)
444 simd_reduce_all, (c v) {
445 if !v.layout().ty.is_simd() {
446 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
450 simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
453 simd_reduce_any, (c v) {
454 if !v.layout().ty.is_simd() {
455 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
459 simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
462 simd_reduce_and, (c v) {
463 if !v.layout().ty.is_simd() {
464 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
468 simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
471 simd_reduce_or, (c v) {
472 if !v.layout().ty.is_simd() {
473 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
477 simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
480 simd_reduce_xor, (c v) {
481 if !v.layout().ty.is_simd() {
482 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
486 simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
489 simd_reduce_min, (c v) {
490 if !v.layout().ty.is_simd() {
491 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
495 simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
496 let lt = match ty.kind() {
497 ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
498 ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
499 ty::Float(_) => return crate::num::codegen_float_min(fx, a, b),
502 fx.bcx.ins().select(lt, a, b)
506 simd_reduce_max, (c v) {
507 if !v.layout().ty.is_simd() {
508 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
512 simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
513 let gt = match ty.kind() {
514 ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
515 ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
516 ty::Float(_) => return crate::num::codegen_float_max(fx, a, b),
519 fx.bcx.ins().select(gt, a, b)
523 simd_select, (c m, c a, c b) {
524 if !m.layout().ty.is_simd() {
525 report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
528 if !a.layout().ty.is_simd() {
529 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
532 assert_eq!(a.layout(), b.layout());
534 let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
535 let lane_layout = fx.layout_of(lane_ty);
537 for lane in 0..lane_count {
538 let m_lane = m.value_lane(fx, lane).load_scalar(fx);
539 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
540 let b_lane = b.value_lane(fx, lane).load_scalar(fx);
542 let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
543 let res_lane = CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
545 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);