compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

   1 //! Codegen `extern "platform-intrinsic"` intrinsics.
   2
   3 use super::*;
   4 use crate::prelude::*;
   5
   6 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
   7     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
   8     instance: Instance<'tcx>,
   9     args: &[mir::Operand<'tcx>],
  10     ret: CPlace<'tcx>,
  11     span: Span,
  12 ) {
  13     let def_id = instance.def_id();
  14     let substs = instance.substs;
  15
  16     let intrinsic = fx.tcx.item_name(def_id).as_str();
  17     let intrinsic = &intrinsic[..];
  18
  19     intrinsic_match! {
  20         fx, intrinsic, substs, args,
  21         _ => {
  22             fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
  23         };
  24
  25         simd_cast, (c a) {
  26             validate_simd_type!(fx, intrinsic, span, a.layout().ty);
  27             simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
  28                 let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
  29
  30                 let from_signed = type_sign(lane_layout.ty);
  31                 let to_signed = type_sign(ret_lane_layout.ty);
  32
  33                 let ret_lane = clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed);
  34                 CValue::by_val(ret_lane, ret_lane_layout)
  35             });
  36         };
  37
  38         // FIXME support float comparisons
  39         simd_eq, (c x, c y) {
  40             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  41             simd_cmp!(fx, Equal(x, y) -> ret);
  42         };
  43         simd_ne, (c x, c y) {
  44             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  45             simd_cmp!(fx, NotEqual(x, y) -> ret);
  46         };
  47         simd_lt, (c x, c y) {
  48             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  49             simd_cmp!(fx, UnsignedLessThan|SignedLessThan(x, y) -> ret);
  50         };
  51         simd_le, (c x, c y) {
  52             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  53             simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual(x, y) -> ret);
  54         };
  55         simd_gt, (c x, c y) {
  56             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  57             simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan(x, y) -> ret);
  58         };
  59         simd_ge, (c x, c y) {
  60             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  61             simd_cmp!(fx, UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual(x, y) -> ret);
  62         };
  63
  64         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
  65         _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) {
  66             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
  67
  68             let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap();
  69
  70             assert_eq!(x.layout(), y.layout());
  71             let layout = x.layout();
  72
  73             let (lane_type, lane_count) = lane_type_and_count(fx.tcx, layout);
  74             let (ret_lane_type, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
  75
  76             assert_eq!(lane_type, ret_lane_type);
  77             assert_eq!(n, ret_lane_count);
  78
  79             let total_len = lane_count * 2;
  80
  81             let indexes = {
  82                 use rustc_middle::mir::interpret::*;
  83                 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
  84
  85                 let idx_bytes = match idx_const.val {
  86                     ty::ConstKind::Value(ConstValue::ByRef { alloc, offset }) => {
  87                         let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
  88                         let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
  89                         alloc.get_bytes(fx, ptr, size).unwrap()
  90                     }
  91                     _ => unreachable!("{:?}", idx_const),
  92                 };
  93
  94                 (0..ret_lane_count).map(|i| {
  95                     let i = usize::try_from(i).unwrap();
  96                     let idx = rustc_middle::mir::interpret::read_target_uint(
  97                         fx.tcx.data_layout.endian,
  98                         &idx_bytes[4*i.. 4*i + 4],
  99                     ).expect("read_target_uint");
 100                     u16::try_from(idx).expect("try_from u32")
 101                 }).collect::<Vec<u16>>()
 102             };
 103
 104             for &idx in &indexes {
 105                 assert!(idx < total_len, "idx {} out of range 0..{}", idx, total_len);
 106             }
 107
 108             for (out_idx, in_idx) in indexes.into_iter().enumerate() {
 109                 let in_lane = if in_idx < lane_count {
 110                     x.value_field(fx, mir::Field::new(in_idx.try_into().unwrap()))
 111                 } else {
 112                     y.value_field(fx, mir::Field::new((in_idx - lane_count).try_into().unwrap()))
 113                 };
 114                 let out_lane = ret.place_field(fx, mir::Field::new(out_idx));
 115                 out_lane.write_cvalue(fx, in_lane);
 116             }
 117         };
 118
 119         simd_insert, (c base, o idx, c val) {
 120             // FIXME validate
 121             let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
 122                 idx_const
 123             } else {
 124                 fx.tcx.sess.span_fatal(
 125                     span,
 126                     "Index argument for `simd_insert` is not a constant",
 127                 );
 128             };
 129
 130             let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
 131             let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, base.layout());
 132             if idx >= lane_count.into() {
 133                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
 134             }
 135
 136             ret.write_cvalue(fx, base);
 137             let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
 138             ret_lane.write_cvalue(fx, val);
 139         };
 140
 141         simd_extract, (c v, o idx) {
 142             validate_simd_type!(fx, intrinsic, span, v.layout().ty);
 143             let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
 144                 idx_const
 145             } else {
 146                 fx.tcx.sess.span_fatal(
 147                     span,
 148                     "Index argument for `simd_extract` is not a constant",
 149                 );
 150             };
 151
 152             let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
 153             let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, v.layout());
 154             if idx >= lane_count.into() {
 155                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
 156             }
 157
 158             let ret_lane = v.value_field(fx, mir::Field::new(idx.try_into().unwrap()));
 159             ret.write_cvalue(fx, ret_lane);
 160         };
 161
 162         simd_add, (c x, c y) {
 163             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 164             simd_int_flt_binop!(fx, iadd|fadd(x, y) -> ret);
 165         };
 166         simd_sub, (c x, c y) {
 167             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 168             simd_int_flt_binop!(fx, isub|fsub(x, y) -> ret);
 169         };
 170         simd_mul, (c x, c y) {
 171             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 172             simd_int_flt_binop!(fx, imul|fmul(x, y) -> ret);
 173         };
 174         simd_div, (c x, c y) {
 175             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 176             simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret);
 177         };
 178         simd_shl, (c x, c y) {
 179             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 180             simd_int_binop!(fx, ishl(x, y) -> ret);
 181         };
 182         simd_shr, (c x, c y) {
 183             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 184             simd_int_binop!(fx, ushr|sshr(x, y) -> ret);
 185         };
 186         simd_and, (c x, c y) {
 187             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 188             simd_int_binop!(fx, band(x, y) -> ret);
 189         };
 190         simd_or, (c x, c y) {
 191             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 192             simd_int_binop!(fx, bor(x, y) -> ret);
 193         };
 194         simd_xor, (c x, c y) {
 195             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 196             simd_int_binop!(fx, bxor(x, y) -> ret);
 197         };
 198
 199         simd_fma, (c a, c b, c c) {
 200             validate_simd_type!(fx, intrinsic, span, a.layout().ty);
 201             assert_eq!(a.layout(), b.layout());
 202             assert_eq!(a.layout(), c.layout());
 203             let layout = a.layout();
 204
 205             let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
 206             let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
 207             assert_eq!(lane_count, ret_lane_count);
 208
 209             for lane in 0..lane_count {
 210                 let lane = mir::Field::new(lane.try_into().unwrap());
 211                 let a_lane = a.value_field(fx, lane).load_scalar(fx);
 212                 let b_lane = b.value_field(fx, lane).load_scalar(fx);
 213                 let c_lane = c.value_field(fx, lane).load_scalar(fx);
 214
 215                 let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane);
 216                 let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout);
 217
 218                 ret.place_field(fx, lane).write_cvalue(fx, res_lane);
 219             }
 220         };
 221
 222         simd_fmin, (c x, c y) {
 223             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 224             simd_flt_binop!(fx, fmin(x, y) -> ret);
 225         };
 226         simd_fmax, (c x, c y) {
 227             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 228             simd_flt_binop!(fx, fmax(x, y) -> ret);
 229         };
 230
 231         // simd_fabs
 232         // simd_saturating_add
 233         // simd_bitmask
 234         // simd_select
 235         // simd_reduce_add_{,un}ordered
 236         // simd_rem
 237     }
 238 }