compiler/rustc_codegen_gcc/src/intrinsic/simd.rs

   1 use std::cmp::Ordering;
   2
   3 use gccjit::{BinaryOp, RValue, Type, ToRValue};
   4 use rustc_codegen_ssa::base::compare_simd_types;
   5 use rustc_codegen_ssa::common::TypeKind;
   6 use rustc_codegen_ssa::mir::operand::OperandRef;
   7 use rustc_codegen_ssa::mir::place::PlaceRef;
   8 use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods};
   9 use rustc_hir as hir;
  10 use rustc_middle::span_bug;
  11 use rustc_middle::ty::layout::HasTyCtxt;
  12 use rustc_middle::ty::{self, Ty};
  13 use rustc_span::{Span, Symbol, sym};
  14 use rustc_target::abi::Align;
  15
  16 use crate::builder::Builder;
  17 use crate::errors::{
  18     InvalidMonomorphizationInvalidFloatVector,
  19     InvalidMonomorphizationNotFloat,
  20     InvalidMonomorphizationUnrecognized,
  21     InvalidMonomorphizationExpectedSignedUnsigned,
  22     InvalidMonomorphizationUnsupportedElement,
  23     InvalidMonomorphizationInvalidBitmask,
  24     InvalidMonomorphizationSimdShuffle,
  25     InvalidMonomorphizationExpectedSimd,
  26     InvalidMonomorphizationMaskType,
  27     InvalidMonomorphizationReturnLength,
  28     InvalidMonomorphizationReturnLengthInputType,
  29     InvalidMonomorphizationReturnElement,
  30     InvalidMonomorphizationReturnType,
  31     InvalidMonomorphizationInsertedType,
  32     InvalidMonomorphizationReturnIntegerType,
  33     InvalidMonomorphizationMismatchedLengths,
  34     InvalidMonomorphizationUnsupportedCast,
  35     InvalidMonomorphizationUnsupportedOperation
  36 };
  37 use crate::intrinsic;
  38
  39 pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>, name: Symbol, callee_ty: Ty<'tcx>, args: &[OperandRef<'tcx, RValue<'gcc>>], ret_ty: Ty<'tcx>, llret_ty: Type<'gcc>, span: Span) -> Result<RValue<'gcc>, ()> {
  40     // macros for error handling:
  41     macro_rules! return_error {
  42         ($err:expr) => {
  43             {
  44                 bx.sess().emit_err($err);
  45                 return Err(());
  46             }
  47         }
  48     }
  49     macro_rules! require {
  50         ($cond:expr, $err:expr) => {
  51             if !$cond {
  52                 return_error!($err);
  53             }
  54         }
  55     }
  56     macro_rules! require_simd {
  57         ($ty: expr, $position: expr) => {
  58             require!($ty.is_simd(), InvalidMonomorphizationExpectedSimd { span, name, position: $position, found_ty: $ty })
  59         };
  60     }
  61
  62     let tcx = bx.tcx();
  63     let sig =
  64         tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx));
  65     let arg_tys = sig.inputs();
  66
  67     if name == sym::simd_select_bitmask {
  68         require_simd!(arg_tys[1], "argument");
  69         let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
  70
  71         let expected_int_bits = (len.max(8) - 1).next_power_of_two();
  72         let expected_bytes = len / 8 + ((len % 8 > 0) as u64);
  73
  74         let mask_ty = arg_tys[0];
  75         let mut mask = match mask_ty.kind() {
  76             ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
  77             ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
  78             ty::Array(elem, len)
  79                 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
  80                     && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all())
  81                         == Some(expected_bytes) =>
  82             {
  83                 let place = PlaceRef::alloca(bx, args[0].layout);
  84                 args[0].val.store(bx, place);
  85                 let int_ty = bx.type_ix(expected_bytes * 8);
  86                 let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty));
  87                 bx.load(int_ty, ptr, Align::ONE)
  88             }
  89             _ => return_error!(
  90                 InvalidMonomorphizationInvalidBitmask { span, name, ty: mask_ty, expected_int_bits, expected_bytes }
  91             ),
  92         };
  93
  94         let arg1 = args[1].immediate();
  95         let arg1_type = arg1.get_type();
  96         let arg1_vector_type = arg1_type.unqualified().dyncast_vector().expect("vector type");
  97         let arg1_element_type = arg1_vector_type.get_element_type();
  98
  99         let mut elements = vec![];
 100         let one = bx.context.new_rvalue_one(mask.get_type());
 101         for _ in 0..len {
 102             let element = bx.context.new_cast(None, mask & one, arg1_element_type);
 103             elements.push(element);
 104             mask = mask >> one;
 105         }
 106         let vector_mask = bx.context.new_rvalue_from_vector(None, arg1_type, &elements);
 107
 108         return Ok(bx.vector_select(vector_mask, arg1, args[2].immediate()));
 109     }
 110
 111     // every intrinsic below takes a SIMD vector as its first argument
 112     require_simd!(arg_tys[0], "input");
 113     let in_ty = arg_tys[0];
 114
 115     let comparison = match name {
 116         sym::simd_eq => Some(hir::BinOpKind::Eq),
 117         sym::simd_ne => Some(hir::BinOpKind::Ne),
 118         sym::simd_lt => Some(hir::BinOpKind::Lt),
 119         sym::simd_le => Some(hir::BinOpKind::Le),
 120         sym::simd_gt => Some(hir::BinOpKind::Gt),
 121         sym::simd_ge => Some(hir::BinOpKind::Ge),
 122         _ => None,
 123     };
 124
 125     let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx());
 126     if let Some(cmp_op) = comparison {
 127         require_simd!(ret_ty, "return");
 128
 129         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
 130         require!(
 131             in_len == out_len,
 132             InvalidMonomorphizationReturnLengthInputType { span, name, in_len, in_ty, ret_ty, out_len }
 133         );
 134         require!(
 135             bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
 136             InvalidMonomorphizationReturnIntegerType {span, name, ret_ty, out_ty}
 137         );
 138
 139         return Ok(compare_simd_types(
 140             bx,
 141             args[0].immediate(),
 142             args[1].immediate(),
 143             in_elem,
 144             llret_ty,
 145             cmp_op,
 146         ));
 147     }
 148
 149     if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") {
 150         let n: u64 =
 151             if stripped.is_empty() {
 152                 // Make sure this is actually an array, since typeck only checks the length-suffixed
 153                 // version of this intrinsic.
 154                 match args[2].layout.ty.kind() {
 155                     ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
 156                         len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
 157                             span_bug!(span, "could not evaluate shuffle index array length")
 158                         })
 159                     }
 160                     _ => return_error!(
 161                         InvalidMonomorphizationSimdShuffle { span, name, ty: args[2].layout.ty }
 162                     ),
 163                 }
 164             }
 165             else {
 166                 stripped.parse().unwrap_or_else(|_| {
 167                     span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")
 168                 })
 169             };
 170
 171         require_simd!(ret_ty, "return");
 172
 173         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
 174         require!(
 175             out_len == n,
 176             InvalidMonomorphizationReturnLength { span, name, in_len: n, ret_ty, out_len }
 177         );
 178         require!(
 179             in_elem == out_ty,
 180             InvalidMonomorphizationReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
 181         );
 182
 183         let vector = args[2].immediate();
 184
 185         return Ok(bx.shuffle_vector(
 186             args[0].immediate(),
 187             args[1].immediate(),
 188             vector,
 189         ));
 190     }
 191
 192     #[cfg(feature="master")]
 193     if name == sym::simd_insert {
 194         require!(
 195             in_elem == arg_tys[2],
 196             InvalidMonomorphizationInsertedType { span, name, in_elem, in_ty, out_ty: arg_tys[2] }
 197         );
 198         let vector = args[0].immediate();
 199         let index = args[1].immediate();
 200         let value = args[2].immediate();
 201         // TODO(antoyo): use a recursive unqualified() here.
 202         let vector_type = vector.get_type().unqualified().dyncast_vector().expect("vector type");
 203         let element_type = vector_type.get_element_type();
 204         // NOTE: we cannot cast to an array and assign to its element here because the value might
 205         // not be an l-value. So, call a builtin to set the element.
 206         // TODO(antoyo): perhaps we could create a new vector or maybe there's a GIMPLE instruction for that?
 207         // TODO(antoyo): don't use target specific builtins here.
 208         let func_name =
 209             match in_len {
 210                 2 => {
 211                     if element_type == bx.i64_type {
 212                         "__builtin_ia32_vec_set_v2di"
 213                     }
 214                     else {
 215                         unimplemented!();
 216                     }
 217                 },
 218                 4 => {
 219                     if element_type == bx.i32_type {
 220                         "__builtin_ia32_vec_set_v4si"
 221                     }
 222                     else {
 223                         unimplemented!();
 224                     }
 225                 },
 226                 8 => {
 227                     if element_type == bx.i16_type {
 228                         "__builtin_ia32_vec_set_v8hi"
 229                     }
 230                     else {
 231                         unimplemented!();
 232                     }
 233                 },
 234                 _ => unimplemented!("Len: {}", in_len),
 235             };
 236         let builtin = bx.context.get_target_builtin_function(func_name);
 237         let param1_type = builtin.get_param(0).to_rvalue().get_type();
 238         // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
 239         let vector = bx.cx.bitcast_if_needed(vector, param1_type);
 240         let result = bx.context.new_call(None, builtin, &[vector, value, bx.context.new_cast(None, index, bx.int_type)]);
 241         // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
 242         return Ok(bx.context.new_bitcast(None, result, vector.get_type()));
 243     }
 244
 245     #[cfg(feature="master")]
 246     if name == sym::simd_extract {
 247         require!(
 248             ret_ty == in_elem,
 249             InvalidMonomorphizationReturnType { span, name, in_elem, in_ty, ret_ty }
 250         );
 251         let vector = args[0].immediate();
 252         return Ok(bx.context.new_vector_access(None, vector, args[1].immediate()).to_rvalue());
 253     }
 254
 255     if name == sym::simd_select {
 256         let m_elem_ty = in_elem;
 257         let m_len = in_len;
 258         require_simd!(arg_tys[1], "argument");
 259         let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
 260         require!(
 261             m_len == v_len,
 262             InvalidMonomorphizationMismatchedLengths { span, name, m_len, v_len }
 263         );
 264         match m_elem_ty.kind() {
 265             ty::Int(_) => {}
 266             _ => return_error!(InvalidMonomorphizationMaskType { span, name, ty: m_elem_ty }),
 267         }
 268         return Ok(bx.vector_select(args[0].immediate(), args[1].immediate(), args[2].immediate()));
 269     }
 270
 271     if name == sym::simd_cast {
 272         require_simd!(ret_ty, "return");
 273         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
 274         require!(
 275             in_len == out_len,
 276             InvalidMonomorphizationReturnLengthInputType { span, name, in_len, in_ty, ret_ty, out_len }
 277         );
 278         // casting cares about nominal type, not just structural type
 279         if in_elem == out_elem {
 280             return Ok(args[0].immediate());
 281         }
 282
 283         enum Style {
 284             Float,
 285             Int(/* is signed? */ bool),
 286             Unsupported,
 287         }
 288
 289         let (in_style, in_width) = match in_elem.kind() {
 290             // vectors of pointer-sized integers should've been
 291             // disallowed before here, so this unwrap is safe.
 292             ty::Int(i) => (
 293                 Style::Int(true),
 294                 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
 295             ),
 296             ty::Uint(u) => (
 297                 Style::Int(false),
 298                 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
 299             ),
 300             ty::Float(f) => (Style::Float, f.bit_width()),
 301             _ => (Style::Unsupported, 0),
 302         };
 303         let (out_style, out_width) = match out_elem.kind() {
 304             ty::Int(i) => (
 305                 Style::Int(true),
 306                 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
 307             ),
 308             ty::Uint(u) => (
 309                 Style::Int(false),
 310                 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
 311             ),
 312             ty::Float(f) => (Style::Float, f.bit_width()),
 313             _ => (Style::Unsupported, 0),
 314         };
 315
 316         let extend = |in_type, out_type| {
 317             let vector_type = bx.context.new_vector_type(out_type, 8);
 318             let vector = args[0].immediate();
 319             let array_type = bx.context.new_array_type(None, in_type, 8);
 320             // TODO(antoyo): switch to using new_vector_access or __builtin_convertvector for vector casting.
 321             let array = bx.context.new_bitcast(None, vector, array_type);
 322
 323             let cast_vec_element = |index| {
 324                 let index = bx.context.new_rvalue_from_int(bx.int_type, index);
 325                 bx.context.new_cast(None, bx.context.new_array_access(None, array, index).to_rvalue(), out_type)
 326             };
 327
 328             bx.context.new_rvalue_from_vector(None, vector_type, &[
 329                 cast_vec_element(0),
 330                 cast_vec_element(1),
 331                 cast_vec_element(2),
 332                 cast_vec_element(3),
 333                 cast_vec_element(4),
 334                 cast_vec_element(5),
 335                 cast_vec_element(6),
 336                 cast_vec_element(7),
 337             ])
 338         };
 339
 340         match (in_style, out_style) {
 341             (Style::Int(in_is_signed), Style::Int(_)) => {
 342                 return Ok(match in_width.cmp(&out_width) {
 343                     Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
 344                     Ordering::Equal => args[0].immediate(),
 345                     Ordering::Less => {
 346                         if in_is_signed {
 347                             match (in_width, out_width) {
 348                                 // FIXME(antoyo): the function _mm_cvtepi8_epi16 should directly
 349                                 // call an intrinsic equivalent to __builtin_ia32_pmovsxbw128 so that
 350                                 // we can generate a call to it.
 351                                 (8, 16) => extend(bx.i8_type, bx.i16_type),
 352                                 (8, 32) => extend(bx.i8_type, bx.i32_type),
 353                                 (8, 64) => extend(bx.i8_type, bx.i64_type),
 354                                 (16, 32) => extend(bx.i16_type, bx.i32_type),
 355                                 (32, 64) => extend(bx.i32_type, bx.i64_type),
 356                                 (16, 64) => extend(bx.i16_type, bx.i64_type),
 357                                 _ => unimplemented!("in: {}, out: {}", in_width, out_width),
 358                             }
 359                         } else {
 360                             match (in_width, out_width) {
 361                                 (8, 16) => extend(bx.u8_type, bx.u16_type),
 362                                 (8, 32) => extend(bx.u8_type, bx.u32_type),
 363                                 (8, 64) => extend(bx.u8_type, bx.u64_type),
 364                                 (16, 32) => extend(bx.u16_type, bx.u32_type),
 365                                 (16, 64) => extend(bx.u16_type, bx.u64_type),
 366                                 (32, 64) => extend(bx.u32_type, bx.u64_type),
 367                                 _ => unimplemented!("in: {}, out: {}", in_width, out_width),
 368                             }
 369                         }
 370                     }
 371                 });
 372             }
 373             (Style::Int(_), Style::Float) => {
 374                 // TODO: add support for internal functions in libgccjit to get access to IFN_VEC_CONVERT which is
 375                 // doing like __builtin_convertvector?
 376                 // Or maybe provide convert_vector as an API since it might not easy to get the
 377                 // types of internal functions.
 378                 unimplemented!();
 379             }
 380             (Style::Float, Style::Int(_)) => {
 381                 unimplemented!();
 382             }
 383             (Style::Float, Style::Float) => {
 384                 unimplemented!();
 385             }
 386             _ => { /* Unsupported. Fallthrough. */ }
 387         }
 388         return_error!(
 389             InvalidMonomorphizationUnsupportedCast { span, name, in_ty, in_elem, ret_ty, out_elem }
 390         );
 391     }
 392
 393     macro_rules! arith_binary {
 394         ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
 395             $(if name == sym::$name {
 396                 match in_elem.kind() {
 397                     $($(ty::$p(_))|* => {
 398                         return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
 399                     })*
 400                     _ => {},
 401                 }
 402                 return_error!(InvalidMonomorphizationUnsupportedOperation { span, name, in_ty, in_elem })
 403             })*
 404         }
 405     }
 406
 407     fn simd_simple_float_intrinsic<'gcc, 'tcx>(
 408         name: Symbol,
 409         in_elem: Ty<'_>,
 410         in_ty: Ty<'_>,
 411         in_len: u64,
 412         bx: &mut Builder<'_, 'gcc, 'tcx>,
 413         span: Span,
 414         args: &[OperandRef<'tcx, RValue<'gcc>>],
 415     ) -> Result<RValue<'gcc>, ()> {
 416         macro_rules! return_error {
 417             ($err:expr) => {
 418                 {
 419                     bx.sess().emit_err($err);
 420                     return Err(());
 421                 }
 422             }
 423         }
 424         let (elem_ty_str, elem_ty) =
 425             if let ty::Float(f) = in_elem.kind() {
 426                 let elem_ty = bx.cx.type_float_from_ty(*f);
 427                 match f.bit_width() {
 428                     32 => ("f32", elem_ty),
 429                     64 => ("f64", elem_ty),
 430                     _ => {
 431                         return_error!(InvalidMonomorphizationInvalidFloatVector { span, name, elem_ty: f.name_str(), vec_ty: in_ty });
 432                     }
 433                 }
 434             }
 435             else {
 436                 return_error!(InvalidMonomorphizationNotFloat { span, name, ty: in_ty });
 437             };
 438
 439         let vec_ty = bx.cx.type_vector(elem_ty, in_len);
 440
 441         let (intr_name, fn_ty) =
 442             match name {
 443                 sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
 444                 sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), // TODO(antoyo): pand with 170141183420855150465331762880109871103
 445                 sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
 446                 sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
 447                 sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
 448                 sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
 449                 sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
 450                 sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
 451                 sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
 452                 sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
 453                 sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
 454                 sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
 455                 sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
 456                 sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
 457                 sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
 458                 sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
 459                 _ => return_error!(InvalidMonomorphizationUnrecognized { span, name })
 460             };
 461         let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
 462         let function = intrinsic::llvm::intrinsic(llvm_name, &bx.cx);
 463         let function: RValue<'gcc> = unsafe { std::mem::transmute(function) };
 464         let c = bx.call(fn_ty, None, function, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
 465         Ok(c)
 466     }
 467
 468     if std::matches!(
 469         name,
 470         sym::simd_ceil
 471             | sym::simd_fabs
 472             | sym::simd_fcos
 473             | sym::simd_fexp2
 474             | sym::simd_fexp
 475             | sym::simd_flog10
 476             | sym::simd_flog2
 477             | sym::simd_flog
 478             | sym::simd_floor
 479             | sym::simd_fma
 480             | sym::simd_fpow
 481             | sym::simd_fpowi
 482             | sym::simd_fsin
 483             | sym::simd_fsqrt
 484             | sym::simd_round
 485             | sym::simd_trunc
 486     ) {
 487         return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
 488     }
 489
 490     arith_binary! {
 491         simd_add: Uint, Int => add, Float => fadd;
 492         simd_sub: Uint, Int => sub, Float => fsub;
 493         simd_mul: Uint, Int => mul, Float => fmul;
 494         simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
 495         simd_rem: Uint => urem, Int => srem, Float => frem;
 496         simd_shl: Uint, Int => shl;
 497         simd_shr: Uint => lshr, Int => ashr;
 498         simd_and: Uint, Int => and;
 499         simd_or: Uint, Int => or; // FIXME(antoyo): calling `or` might not work on vectors.
 500         simd_xor: Uint, Int => xor;
 501     }
 502
 503     macro_rules! arith_unary {
 504         ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
 505             $(if name == sym::$name {
 506                 match in_elem.kind() {
 507                     $($(ty::$p(_))|* => {
 508                         return Ok(bx.$call(args[0].immediate()))
 509                     })*
 510                     _ => {},
 511                 }
 512                 return_error!(InvalidMonomorphizationUnsupportedOperation { span, name, in_ty, in_elem })
 513             })*
 514         }
 515     }
 516
 517     arith_unary! {
 518         simd_neg: Int => neg, Float => fneg;
 519     }
 520
 521     #[cfg(feature="master")]
 522     if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
 523         let lhs = args[0].immediate();
 524         let rhs = args[1].immediate();
 525         let is_add = name == sym::simd_saturating_add;
 526         let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
 527         let (signed, elem_width, elem_ty) = match *in_elem.kind() {
 528             ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)),
 529             ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)),
 530             _ => {
 531                 return_error!(InvalidMonomorphizationExpectedSignedUnsigned {
 532                     span,
 533                     name,
 534                     elem_ty: arg_tys[0].simd_size_and_type(bx.tcx()).1,
 535                     vec_ty: arg_tys[0],
 536                 });
 537             }
 538         };
 539         let builtin_name =
 540             match (signed, is_add, in_len, elem_width) {
 541                 (true, true, 32, 8) => "__builtin_ia32_paddsb256", // TODO(antoyo): cast arguments to unsigned.
 542                 (false, true, 32, 8) => "__builtin_ia32_paddusb256",
 543                 (true, true, 16, 16) => "__builtin_ia32_paddsw256",
 544                 (false, true, 16, 16) => "__builtin_ia32_paddusw256",
 545                 (true, false, 16, 16) => "__builtin_ia32_psubsw256",
 546                 (false, false, 16, 16) => "__builtin_ia32_psubusw256",
 547                 (true, false, 32, 8) => "__builtin_ia32_psubsb256",
 548                 (false, false, 32, 8) => "__builtin_ia32_psubusb256",
 549                 _ => unimplemented!("signed: {}, is_add: {}, in_len: {}, elem_width: {}", signed, is_add, in_len, elem_width),
 550             };
 551         let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
 552
 553         let func = bx.context.get_target_builtin_function(builtin_name);
 554         let param1_type = func.get_param(0).to_rvalue().get_type();
 555         let param2_type = func.get_param(1).to_rvalue().get_type();
 556         let lhs = bx.cx.bitcast_if_needed(lhs, param1_type);
 557         let rhs = bx.cx.bitcast_if_needed(rhs, param2_type);
 558         let result = bx.context.new_call(None, func, &[lhs, rhs]);
 559         // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
 560         return Ok(bx.context.new_bitcast(None, result, vec_ty));
 561     }
 562
 563     macro_rules! arith_red {
 564         ($name:ident : $vec_op:expr, $float_reduce:ident, $ordered:expr, $op:ident,
 565          $identity:expr) => {
 566             if name == sym::$name {
 567                 require!(
 568                     ret_ty == in_elem,
 569                     InvalidMonomorphizationReturnType { span, name, in_elem, in_ty, ret_ty }
 570                 );
 571                 return match in_elem.kind() {
 572                     ty::Int(_) | ty::Uint(_) => {
 573                         let r = bx.vector_reduce_op(args[0].immediate(), $vec_op);
 574                         if $ordered {
 575                             // if overflow occurs, the result is the
 576                             // mathematical result modulo 2^n:
 577                             Ok(bx.$op(args[1].immediate(), r))
 578                         }
 579                         else {
 580                             Ok(bx.vector_reduce_op(args[0].immediate(), $vec_op))
 581                         }
 582                     }
 583                     ty::Float(_) => {
 584                         if $ordered {
 585                             // ordered arithmetic reductions take an accumulator
 586                             let acc = args[1].immediate();
 587                             Ok(bx.$float_reduce(acc, args[0].immediate()))
 588                         }
 589                         else {
 590                             Ok(bx.vector_reduce_op(args[0].immediate(), $vec_op))
 591                         }
 592                     }
 593                     _ => return_error!(InvalidMonomorphizationUnsupportedElement { span, name, in_ty, elem_ty: in_elem, ret_ty }),
 594                 };
 595             }
 596         };
 597     }
 598
 599     arith_red!(
 600         simd_reduce_add_unordered: BinaryOp::Plus,
 601         vector_reduce_fadd_fast,
 602         false,
 603         add,
 604         0.0 // TODO: Use this argument.
 605     );
 606     arith_red!(
 607         simd_reduce_mul_unordered: BinaryOp::Mult,
 608         vector_reduce_fmul_fast,
 609         false,
 610         mul,
 611         1.0
 612     );
 613
 614     macro_rules! minmax_red {
 615         ($name:ident: $reduction:ident) => {
 616             if name == sym::$name {
 617                 require!(
 618                     ret_ty == in_elem,
 619                     InvalidMonomorphizationReturnType { span, name, in_elem, in_ty, ret_ty }
 620                 );
 621                 return match in_elem.kind() {
 622                     ty::Int(_) | ty::Uint(_) | ty::Float(_) => Ok(bx.$reduction(args[0].immediate())),
 623                     _ => return_error!(InvalidMonomorphizationUnsupportedElement { span, name, in_ty, elem_ty: in_elem, ret_ty }),
 624                 };
 625             }
 626         };
 627     }
 628
 629     minmax_red!(simd_reduce_min: vector_reduce_min);
 630     minmax_red!(simd_reduce_max: vector_reduce_max);
 631
 632     macro_rules! bitwise_red {
 633         ($name:ident : $op:expr, $boolean:expr) => {
 634             if name == sym::$name {
 635                 let input = if !$boolean {
 636                     require!(
 637                         ret_ty == in_elem,
 638                         InvalidMonomorphizationReturnType { span, name, in_elem, in_ty, ret_ty }
 639                     );
 640                     args[0].immediate()
 641                 } else {
 642                     match in_elem.kind() {
 643                         ty::Int(_) | ty::Uint(_) => {}
 644                         _ => return_error!(InvalidMonomorphizationUnsupportedElement { span, name, in_ty, elem_ty: in_elem, ret_ty }),
 645                     }
 646
 647                     // boolean reductions operate on vectors of i1s:
 648                     let i1 = bx.type_i1();
 649                     let i1xn = bx.type_vector(i1, in_len as u64);
 650                     bx.trunc(args[0].immediate(), i1xn)
 651                 };
 652                 return match in_elem.kind() {
 653                     ty::Int(_) | ty::Uint(_) => {
 654                         let r = bx.vector_reduce_op(input, $op);
 655                         Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
 656                     }
 657                     _ => return_error!(
 658                         InvalidMonomorphizationUnsupportedElement { span, name, in_ty, elem_ty: in_elem, ret_ty }
 659                     ),
 660                 };
 661             }
 662         };
 663     }
 664
 665     bitwise_red!(simd_reduce_and: BinaryOp::BitwiseAnd, false);
 666     bitwise_red!(simd_reduce_or: BinaryOp::BitwiseOr, false);
 667
 668     unimplemented!("simd {}", name);
 669 }