src/intrinsics/simd.rs

   1 //! Codegen `extern "platform-intrinsic"` intrinsics.
   2
   3 use super::*;
   4 use crate::prelude::*;
   5
   6 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
   7     fx: &mut FunctionCx<'_, '_, 'tcx>,
   8     instance: Instance<'tcx>,
   9     args: &[mir::Operand<'tcx>],
  10     ret: CPlace<'tcx>,
  11     span: Span,
  12 ) {
  13     let def_id = instance.def_id();
  14     let substs = instance.substs;
  15
  16     let intrinsic = fx.tcx.item_name(def_id);
  17
  18     intrinsic_match! {
  19         fx, intrinsic, substs, args,
  20         _ => {
  21             fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
  22         };
  23
  24         simd_cast, (c a) {
  25             validate_simd_type!(fx, intrinsic, span, a.layout().ty);
  26             simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
  27                 let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
  28
  29                 let from_signed = type_sign(lane_layout.ty);
  30                 let to_signed = type_sign(ret_lane_layout.ty);
  31
  32                 let ret_lane = clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed);
  33                 CValue::by_val(ret_lane, ret_lane_layout)
  34             });
  35         };
  36
  37         simd_eq, (c x, c y) {
  38             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  39             simd_cmp!(fx, Equal|Equal(x, y) -> ret);
  40         };
  41         simd_ne, (c x, c y) {
  42             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  43             simd_cmp!(fx, NotEqual|NotEqual(x, y) -> ret);
  44         };
  45         simd_lt, (c x, c y) {
  46             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  47             simd_cmp!(fx, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret);
  48         };
  49         simd_le, (c x, c y) {
  50             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  51             simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret);
  52         };
  53         simd_gt, (c x, c y) {
  54             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  55             simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret);
  56         };
  57         simd_ge, (c x, c y) {
  58             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  59             simd_cmp!(
  60                 fx,
  61                 UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual
  62                 (x, y) -> ret
  63             );
  64         };
  65
  66         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
  67         _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
  68             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  69
  70             let n: u16 = intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap();
  71
  72             assert_eq!(x.layout(), y.layout());
  73             let layout = x.layout();
  74
  75             let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
  76             let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
  77
  78             assert_eq!(lane_ty, ret_lane_ty);
  79             assert_eq!(u64::from(n), ret_lane_count);
  80
  81             let total_len = lane_count * 2;
  82
  83             let indexes = {
  84                 use rustc_middle::mir::interpret::*;
  85                 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
  86
  87                 let idx_bytes = match idx_const {
  88                     ConstValue::ByRef { alloc, offset } => {
  89                         let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
  90                         let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
  91                         alloc.get_bytes(fx, ptr, size).unwrap()
  92                     }
  93                     _ => unreachable!("{:?}", idx_const),
  94                 };
  95
  96                 (0..ret_lane_count).map(|i| {
  97                     let i = usize::try_from(i).unwrap();
  98                     let idx = rustc_middle::mir::interpret::read_target_uint(
  99                         fx.tcx.data_layout.endian,
 100                         &idx_bytes[4*i.. 4*i + 4],
 101                     ).expect("read_target_uint");
 102                     u16::try_from(idx).expect("try_from u32")
 103                 }).collect::<Vec<u16>>()
 104             };
 105
 106             for &idx in &indexes {
 107                 assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
 108             }
 109
 110             for (out_idx, in_idx) in indexes.into_iter().enumerate() {
 111                 let in_lane = if u64::from(in_idx) < lane_count {
 112                     x.value_field(fx, mir::Field::new(in_idx.into()))
 113                 } else {
 114                     y.value_field(fx, mir::Field::new(usize::from(in_idx) - usize::try_from(lane_count).unwrap()))
 115                 };
 116                 let out_lane = ret.place_field(fx, mir::Field::new(out_idx));
 117                 out_lane.write_cvalue(fx, in_lane);
 118             }
 119         };
 120
 121         simd_insert, (c base, o idx, c val) {
 122             // FIXME validate
 123             let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
 124                 idx_const
 125             } else {
 126                 fx.tcx.sess.span_fatal(
 127                     span,
 128                     "Index argument for `simd_insert` is not a constant",
 129                 );
 130             };
 131
 132             let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
 133             let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
 134             if idx >= lane_count.into() {
 135                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
 136             }
 137
 138             ret.write_cvalue(fx, base);
 139             let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
 140             ret_lane.write_cvalue(fx, val);
 141         };
 142
 143         simd_extract, (c v, o idx) {
 144             validate_simd_type!(fx, intrinsic, span, v.layout().ty);
 145             let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
 146                 idx_const
 147             } else {
 148                 fx.tcx.sess.span_warn(
 149                     span,
 150                     "Index argument for `simd_extract` is not a constant",
 151                 );
 152                 let res = crate::trap::trap_unimplemented_ret_value(
 153                     fx,
 154                     ret.layout(),
 155                     "Index argument for `simd_extract` is not a constant",
 156                 );
 157                 ret.write_cvalue(fx, res);
 158                 return;
 159             };
 160
 161             let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
 162             let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
 163             if idx >= lane_count.into() {
 164                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
 165             }
 166
 167             let ret_lane = v.value_field(fx, mir::Field::new(idx.try_into().unwrap()));
 168             ret.write_cvalue(fx, ret_lane);
 169         };
 170
 171         simd_add, (c x, c y) {
 172             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 173             simd_int_flt_binop!(fx, iadd|fadd(x, y) -> ret);
 174         };
 175         simd_sub, (c x, c y) {
 176             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 177             simd_int_flt_binop!(fx, isub|fsub(x, y) -> ret);
 178         };
 179         simd_mul, (c x, c y) {
 180             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 181             simd_int_flt_binop!(fx, imul|fmul(x, y) -> ret);
 182         };
 183         simd_div, (c x, c y) {
 184             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 185             simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret);
 186         };
 187         simd_shl, (c x, c y) {
 188             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 189             simd_int_binop!(fx, ishl(x, y) -> ret);
 190         };
 191         simd_shr, (c x, c y) {
 192             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 193             simd_int_binop!(fx, ushr|sshr(x, y) -> ret);
 194         };
 195         simd_and, (c x, c y) {
 196             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 197             simd_int_binop!(fx, band(x, y) -> ret);
 198         };
 199         simd_or, (c x, c y) {
 200             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 201             simd_int_binop!(fx, bor(x, y) -> ret);
 202         };
 203         simd_xor, (c x, c y) {
 204             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 205             simd_int_binop!(fx, bxor(x, y) -> ret);
 206         };
 207
 208         simd_fma, (c a, c b, c c) {
 209             validate_simd_type!(fx, intrinsic, span, a.layout().ty);
 210             assert_eq!(a.layout(), b.layout());
 211             assert_eq!(a.layout(), c.layout());
 212             let layout = a.layout();
 213
 214             let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
 215             let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
 216             assert_eq!(lane_count, ret_lane_count);
 217             let ret_lane_layout = fx.layout_of(ret_lane_ty);
 218
 219             for lane in 0..lane_count {
 220                 let lane = mir::Field::new(lane.try_into().unwrap());
 221                 let a_lane = a.value_field(fx, lane).load_scalar(fx);
 222                 let b_lane = b.value_field(fx, lane).load_scalar(fx);
 223                 let c_lane = c.value_field(fx, lane).load_scalar(fx);
 224
 225                 let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane);
 226                 let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout);
 227
 228                 ret.place_field(fx, lane).write_cvalue(fx, res_lane);
 229             }
 230         };
 231
 232         simd_fmin, (c x, c y) {
 233             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 234             simd_flt_binop!(fx, fmin(x, y) -> ret);
 235         };
 236         simd_fmax, (c x, c y) {
 237             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 238             simd_flt_binop!(fx, fmax(x, y) -> ret);
 239         };
 240
 241         simd_reduce_add_ordered | simd_reduce_add_unordered, (c v) {
 242             validate_simd_type!(fx, intrinsic, span, v.layout().ty);
 243             simd_reduce(fx, v, ret, |fx, lane_layout, a, b| {
 244                 if lane_layout.ty.is_floating_point() {
 245                     fx.bcx.ins().fadd(a, b)
 246                 } else {
 247                     fx.bcx.ins().iadd(a, b)
 248                 }
 249             });
 250         };
 251
 252         simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v) {
 253             validate_simd_type!(fx, intrinsic, span, v.layout().ty);
 254             simd_reduce(fx, v, ret, |fx, lane_layout, a, b| {
 255                 if lane_layout.ty.is_floating_point() {
 256                     fx.bcx.ins().fmul(a, b)
 257                 } else {
 258                     fx.bcx.ins().imul(a, b)
 259                 }
 260             });
 261         };
 262
 263         simd_reduce_all, (c v) {
 264             validate_simd_type!(fx, intrinsic, span, v.layout().ty);
 265             simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().band(a, b));
 266         };
 267
 268         simd_reduce_any, (c v) {
 269             validate_simd_type!(fx, intrinsic, span, v.layout().ty);
 270             simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().bor(a, b));
 271         };
 272
 273         // simd_fabs
 274         // simd_saturating_add
 275         // simd_bitmask
 276         // simd_select
 277         // simd_rem
 278         // simd_neg
 279         // simd_trunc
 280         // simd_floor
 281     }
 282 }