compiler/rustc_codegen_gcc/src/intrinsic/simd.rs

   1 use std::cmp::Ordering;
   2
   3 use gccjit::{BinaryOp, RValue, Type, ToRValue};
   4 use rustc_codegen_ssa::base::compare_simd_types;
   5 use rustc_codegen_ssa::common::{TypeKind, span_invalid_monomorphization_error};
   6 use rustc_codegen_ssa::mir::operand::OperandRef;
   7 use rustc_codegen_ssa::mir::place::PlaceRef;
   8 use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods};
   9 use rustc_hir as hir;
  10 use rustc_middle::span_bug;
  11 use rustc_middle::ty::layout::HasTyCtxt;
  12 use rustc_middle::ty::{self, Ty};
  13 use rustc_span::{Span, Symbol, sym};
  14 use rustc_target::abi::Align;
  15
  16 use crate::builder::Builder;
  17 use crate::intrinsic;
  18
  19 pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>, name: Symbol, callee_ty: Ty<'tcx>, args: &[OperandRef<'tcx, RValue<'gcc>>], ret_ty: Ty<'tcx>, llret_ty: Type<'gcc>, span: Span) -> Result<RValue<'gcc>, ()> {
  20     // macros for error handling:
  21     #[allow(unused_macro_rules)]
  22     macro_rules! emit_error {
  23         ($msg: tt) => {
  24             emit_error!($msg, )
  25         };
  26         ($msg: tt, $($fmt: tt)*) => {
  27             span_invalid_monomorphization_error(
  28                 bx.sess(), span,
  29                 &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg),
  30                          name, $($fmt)*));
  31         }
  32     }
  33
  34     macro_rules! return_error {
  35         ($($fmt: tt)*) => {
  36             {
  37                 emit_error!($($fmt)*);
  38                 return Err(());
  39             }
  40         }
  41     }
  42
  43     macro_rules! require {
  44         ($cond: expr, $($fmt: tt)*) => {
  45             if !$cond {
  46                 return_error!($($fmt)*);
  47             }
  48         };
  49     }
  50
  51     macro_rules! require_simd {
  52         ($ty: expr, $position: expr) => {
  53             require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty)
  54         };
  55     }
  56
  57     let tcx = bx.tcx();
  58     let sig =
  59         tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx));
  60     let arg_tys = sig.inputs();
  61
  62     if name == sym::simd_select_bitmask {
  63         require_simd!(arg_tys[1], "argument");
  64         let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
  65
  66         let expected_int_bits = (len.max(8) - 1).next_power_of_two();
  67         let expected_bytes = len / 8 + ((len % 8 > 0) as u64);
  68
  69         let mask_ty = arg_tys[0];
  70         let mut mask = match mask_ty.kind() {
  71             ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
  72             ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
  73             ty::Array(elem, len)
  74                 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
  75                     && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all())
  76                         == Some(expected_bytes) =>
  77             {
  78                 let place = PlaceRef::alloca(bx, args[0].layout);
  79                 args[0].val.store(bx, place);
  80                 let int_ty = bx.type_ix(expected_bytes * 8);
  81                 let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty));
  82                 bx.load(int_ty, ptr, Align::ONE)
  83             }
  84             _ => return_error!(
  85                 "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`",
  86                 mask_ty,
  87                 expected_int_bits,
  88                 expected_bytes
  89             ),
  90         };
  91
  92         let arg1 = args[1].immediate();
  93         let arg1_type = arg1.get_type();
  94         let arg1_vector_type = arg1_type.unqualified().dyncast_vector().expect("vector type");
  95         let arg1_element_type = arg1_vector_type.get_element_type();
  96
  97         let mut elements = vec![];
  98         let one = bx.context.new_rvalue_one(mask.get_type());
  99         for _ in 0..len {
 100             let element = bx.context.new_cast(None, mask & one, arg1_element_type);
 101             elements.push(element);
 102             mask = mask >> one;
 103         }
 104         let vector_mask = bx.context.new_rvalue_from_vector(None, arg1_type, &elements);
 105
 106         return Ok(bx.vector_select(vector_mask, arg1, args[2].immediate()));
 107     }
 108
 109     // every intrinsic below takes a SIMD vector as its first argument
 110     require_simd!(arg_tys[0], "input");
 111     let in_ty = arg_tys[0];
 112
 113     let comparison = match name {
 114         sym::simd_eq => Some(hir::BinOpKind::Eq),
 115         sym::simd_ne => Some(hir::BinOpKind::Ne),
 116         sym::simd_lt => Some(hir::BinOpKind::Lt),
 117         sym::simd_le => Some(hir::BinOpKind::Le),
 118         sym::simd_gt => Some(hir::BinOpKind::Gt),
 119         sym::simd_ge => Some(hir::BinOpKind::Ge),
 120         _ => None,
 121     };
 122
 123     let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx());
 124     if let Some(cmp_op) = comparison {
 125         require_simd!(ret_ty, "return");
 126
 127         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
 128         require!(
 129             in_len == out_len,
 130             "expected return type with length {} (same as input type `{}`), \
 131              found `{}` with length {}",
 132             in_len,
 133             in_ty,
 134             ret_ty,
 135             out_len
 136         );
 137         require!(
 138             bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
 139             "expected return type with integer elements, found `{}` with non-integer `{}`",
 140             ret_ty,
 141             out_ty
 142         );
 143
 144         return Ok(compare_simd_types(
 145             bx,
 146             args[0].immediate(),
 147             args[1].immediate(),
 148             in_elem,
 149             llret_ty,
 150             cmp_op,
 151         ));
 152     }
 153
 154     if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") {
 155         let n: u64 =
 156             if stripped.is_empty() {
 157                 // Make sure this is actually an array, since typeck only checks the length-suffixed
 158                 // version of this intrinsic.
 159                 match args[2].layout.ty.kind() {
 160                     ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
 161                         len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
 162                             span_bug!(span, "could not evaluate shuffle index array length")
 163                         })
 164                     }
 165                     _ => return_error!(
 166                         "simd_shuffle index must be an array of `u32`, got `{}`",
 167                         args[2].layout.ty
 168                     ),
 169                 }
 170             }
 171             else {
 172                 stripped.parse().unwrap_or_else(|_| {
 173                     span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")
 174                 })
 175             };
 176
 177         require_simd!(ret_ty, "return");
 178
 179         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
 180         require!(
 181             out_len == n,
 182             "expected return type of length {}, found `{}` with length {}",
 183             n,
 184             ret_ty,
 185             out_len
 186         );
 187         require!(
 188             in_elem == out_ty,
 189             "expected return element type `{}` (element of input `{}`), \
 190              found `{}` with element type `{}`",
 191             in_elem,
 192             in_ty,
 193             ret_ty,
 194             out_ty
 195         );
 196
 197         let vector = args[2].immediate();
 198
 199         return Ok(bx.shuffle_vector(
 200             args[0].immediate(),
 201             args[1].immediate(),
 202             vector,
 203         ));
 204     }
 205
 206     #[cfg(feature="master")]
 207     if name == sym::simd_insert {
 208         require!(
 209             in_elem == arg_tys[2],
 210             "expected inserted type `{}` (element of input `{}`), found `{}`",
 211             in_elem,
 212             in_ty,
 213             arg_tys[2]
 214         );
 215         let vector = args[0].immediate();
 216         let index = args[1].immediate();
 217         let value = args[2].immediate();
 218         // TODO(antoyo): use a recursive unqualified() here.
 219         let vector_type = vector.get_type().unqualified().dyncast_vector().expect("vector type");
 220         let element_type = vector_type.get_element_type();
 221         // NOTE: we cannot cast to an array and assign to its element here because the value might
 222         // not be an l-value. So, call a builtin to set the element.
 223         // TODO(antoyo): perhaps we could create a new vector or maybe there's a GIMPLE instruction for that?
 224         // TODO(antoyo): don't use target specific builtins here.
 225         let func_name =
 226             match in_len {
 227                 2 => {
 228                     if element_type == bx.i64_type {
 229                         "__builtin_ia32_vec_set_v2di"
 230                     }
 231                     else {
 232                         unimplemented!();
 233                     }
 234                 },
 235                 4 => {
 236                     if element_type == bx.i32_type {
 237                         "__builtin_ia32_vec_set_v4si"
 238                     }
 239                     else {
 240                         unimplemented!();
 241                     }
 242                 },
 243                 8 => {
 244                     if element_type == bx.i16_type {
 245                         "__builtin_ia32_vec_set_v8hi"
 246                     }
 247                     else {
 248                         unimplemented!();
 249                     }
 250                 },
 251                 _ => unimplemented!("Len: {}", in_len),
 252             };
 253         let builtin = bx.context.get_target_builtin_function(func_name);
 254         let param1_type = builtin.get_param(0).to_rvalue().get_type();
 255         // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
 256         let vector = bx.cx.bitcast_if_needed(vector, param1_type);
 257         let result = bx.context.new_call(None, builtin, &[vector, value, bx.context.new_cast(None, index, bx.int_type)]);
 258         // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
 259         return Ok(bx.context.new_bitcast(None, result, vector.get_type()));
 260     }
 261
 262     #[cfg(feature="master")]
 263     if name == sym::simd_extract {
 264         require!(
 265             ret_ty == in_elem,
 266             "expected return type `{}` (element of input `{}`), found `{}`",
 267             in_elem,
 268             in_ty,
 269             ret_ty
 270         );
 271         let vector = args[0].immediate();
 272         return Ok(bx.context.new_vector_access(None, vector, args[1].immediate()).to_rvalue());
 273     }
 274
 275     if name == sym::simd_select {
 276         let m_elem_ty = in_elem;
 277         let m_len = in_len;
 278         require_simd!(arg_tys[1], "argument");
 279         let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
 280         require!(
 281             m_len == v_len,
 282             "mismatched lengths: mask length `{}` != other vector length `{}`",
 283             m_len,
 284             v_len
 285         );
 286         match m_elem_ty.kind() {
 287             ty::Int(_) => {}
 288             _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty),
 289         }
 290         return Ok(bx.vector_select(args[0].immediate(), args[1].immediate(), args[2].immediate()));
 291     }
 292
 293     if name == sym::simd_cast {
 294         require_simd!(ret_ty, "return");
 295         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
 296         require!(
 297             in_len == out_len,
 298             "expected return type with length {} (same as input type `{}`), \
 299                   found `{}` with length {}",
 300             in_len,
 301             in_ty,
 302             ret_ty,
 303             out_len
 304         );
 305         // casting cares about nominal type, not just structural type
 306         if in_elem == out_elem {
 307             return Ok(args[0].immediate());
 308         }
 309
 310         enum Style {
 311             Float,
 312             Int(/* is signed? */ bool),
 313             Unsupported,
 314         }
 315
 316         let (in_style, in_width) = match in_elem.kind() {
 317             // vectors of pointer-sized integers should've been
 318             // disallowed before here, so this unwrap is safe.
 319             ty::Int(i) => (
 320                 Style::Int(true),
 321                 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
 322             ),
 323             ty::Uint(u) => (
 324                 Style::Int(false),
 325                 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
 326             ),
 327             ty::Float(f) => (Style::Float, f.bit_width()),
 328             _ => (Style::Unsupported, 0),
 329         };
 330         let (out_style, out_width) = match out_elem.kind() {
 331             ty::Int(i) => (
 332                 Style::Int(true),
 333                 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
 334             ),
 335             ty::Uint(u) => (
 336                 Style::Int(false),
 337                 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
 338             ),
 339             ty::Float(f) => (Style::Float, f.bit_width()),
 340             _ => (Style::Unsupported, 0),
 341         };
 342
 343         let extend = |in_type, out_type| {
 344             let vector_type = bx.context.new_vector_type(out_type, 8);
 345             let vector = args[0].immediate();
 346             let array_type = bx.context.new_array_type(None, in_type, 8);
 347             // TODO(antoyo): switch to using new_vector_access or __builtin_convertvector for vector casting.
 348             let array = bx.context.new_bitcast(None, vector, array_type);
 349
 350             let cast_vec_element = |index| {
 351                 let index = bx.context.new_rvalue_from_int(bx.int_type, index);
 352                 bx.context.new_cast(None, bx.context.new_array_access(None, array, index).to_rvalue(), out_type)
 353             };
 354
 355             bx.context.new_rvalue_from_vector(None, vector_type, &[
 356                 cast_vec_element(0),
 357                 cast_vec_element(1),
 358                 cast_vec_element(2),
 359                 cast_vec_element(3),
 360                 cast_vec_element(4),
 361                 cast_vec_element(5),
 362                 cast_vec_element(6),
 363                 cast_vec_element(7),
 364             ])
 365         };
 366
 367         match (in_style, out_style) {
 368             (Style::Int(in_is_signed), Style::Int(_)) => {
 369                 return Ok(match in_width.cmp(&out_width) {
 370                     Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
 371                     Ordering::Equal => args[0].immediate(),
 372                     Ordering::Less => {
 373                         if in_is_signed {
 374                             match (in_width, out_width) {
 375                                 // FIXME(antoyo): the function _mm_cvtepi8_epi16 should directly
 376                                 // call an intrinsic equivalent to __builtin_ia32_pmovsxbw128 so that
 377                                 // we can generate a call to it.
 378                                 (8, 16) => extend(bx.i8_type, bx.i16_type),
 379                                 (8, 32) => extend(bx.i8_type, bx.i32_type),
 380                                 (8, 64) => extend(bx.i8_type, bx.i64_type),
 381                                 (16, 32) => extend(bx.i16_type, bx.i32_type),
 382                                 (32, 64) => extend(bx.i32_type, bx.i64_type),
 383                                 (16, 64) => extend(bx.i16_type, bx.i64_type),
 384                                 _ => unimplemented!("in: {}, out: {}", in_width, out_width),
 385                             }
 386                         } else {
 387                             match (in_width, out_width) {
 388                                 (8, 16) => extend(bx.u8_type, bx.u16_type),
 389                                 (8, 32) => extend(bx.u8_type, bx.u32_type),
 390                                 (8, 64) => extend(bx.u8_type, bx.u64_type),
 391                                 (16, 32) => extend(bx.u16_type, bx.u32_type),
 392                                 (16, 64) => extend(bx.u16_type, bx.u64_type),
 393                                 (32, 64) => extend(bx.u32_type, bx.u64_type),
 394                                 _ => unimplemented!("in: {}, out: {}", in_width, out_width),
 395                             }
 396                         }
 397                     }
 398                 });
 399             }
 400             (Style::Int(_), Style::Float) => {
 401                 // TODO: add support for internal functions in libgccjit to get access to IFN_VEC_CONVERT which is
 402                 // doing like __builtin_convertvector?
 403                 // Or maybe provide convert_vector as an API since it might not easy to get the
 404                 // types of internal functions.
 405                 unimplemented!();
 406             }
 407             (Style::Float, Style::Int(_)) => {
 408                 unimplemented!();
 409             }
 410             (Style::Float, Style::Float) => {
 411                 unimplemented!();
 412             }
 413             _ => { /* Unsupported. Fallthrough. */ }
 414         }
 415         require!(
 416             false,
 417             "unsupported cast from `{}` with element `{}` to `{}` with element `{}`",
 418             in_ty,
 419             in_elem,
 420             ret_ty,
 421             out_elem
 422         );
 423     }
 424
 425     macro_rules! arith_binary {
 426         ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
 427             $(if name == sym::$name {
 428                 match in_elem.kind() {
 429                     $($(ty::$p(_))|* => {
 430                         return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
 431                     })*
 432                     _ => {},
 433                 }
 434                 require!(false,
 435                          "unsupported operation on `{}` with element `{}`",
 436                          in_ty,
 437                          in_elem)
 438             })*
 439         }
 440     }
 441
 442     fn simd_simple_float_intrinsic<'gcc, 'tcx>(
 443         name: Symbol,
 444         in_elem: Ty<'_>,
 445         in_ty: Ty<'_>,
 446         in_len: u64,
 447         bx: &mut Builder<'_, 'gcc, 'tcx>,
 448         span: Span,
 449         args: &[OperandRef<'tcx, RValue<'gcc>>],
 450     ) -> Result<RValue<'gcc>, ()> {
 451         macro_rules! emit_error {
 452             ($msg: tt) => {
 453                 emit_error!($msg, )
 454             };
 455             ($msg: tt, $($fmt: tt)*) => {
 456                 span_invalid_monomorphization_error(
 457                     bx.sess(), span,
 458                     &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg),
 459                              name, $($fmt)*));
 460             }
 461         }
 462         macro_rules! return_error {
 463             ($($fmt: tt)*) => {
 464                 {
 465                     emit_error!($($fmt)*);
 466                     return Err(());
 467                 }
 468             }
 469         }
 470
 471         let (elem_ty_str, elem_ty) =
 472             if let ty::Float(f) = in_elem.kind() {
 473                 let elem_ty = bx.cx.type_float_from_ty(*f);
 474                 match f.bit_width() {
 475                     32 => ("f32", elem_ty),
 476                     64 => ("f64", elem_ty),
 477                     _ => {
 478                         return_error!(
 479                             "unsupported element type `{}` of floating-point vector `{}`",
 480                             f.name_str(),
 481                             in_ty
 482                         );
 483                     }
 484                 }
 485             }
 486             else {
 487                 return_error!("`{}` is not a floating-point type", in_ty);
 488             };
 489
 490         let vec_ty = bx.cx.type_vector(elem_ty, in_len);
 491
 492         let (intr_name, fn_ty) =
 493             match name {
 494                 sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
 495                 sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), // TODO(antoyo): pand with 170141183420855150465331762880109871103
 496                 sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
 497                 sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
 498                 sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
 499                 sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
 500                 sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
 501                 sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
 502                 sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
 503                 sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
 504                 sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
 505                 sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
 506                 sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
 507                 sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
 508                 sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
 509                 sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
 510                 _ => return_error!("unrecognized intrinsic `{}`", name),
 511             };
 512         let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
 513         let function = intrinsic::llvm::intrinsic(llvm_name, &bx.cx);
 514         let function: RValue<'gcc> = unsafe { std::mem::transmute(function) };
 515         let c = bx.call(fn_ty, function, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
 516         Ok(c)
 517     }
 518
 519     if std::matches!(
 520         name,
 521         sym::simd_ceil
 522             | sym::simd_fabs
 523             | sym::simd_fcos
 524             | sym::simd_fexp2
 525             | sym::simd_fexp
 526             | sym::simd_flog10
 527             | sym::simd_flog2
 528             | sym::simd_flog
 529             | sym::simd_floor
 530             | sym::simd_fma
 531             | sym::simd_fpow
 532             | sym::simd_fpowi
 533             | sym::simd_fsin
 534             | sym::simd_fsqrt
 535             | sym::simd_round
 536             | sym::simd_trunc
 537     ) {
 538         return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
 539     }
 540
 541     arith_binary! {
 542         simd_add: Uint, Int => add, Float => fadd;
 543         simd_sub: Uint, Int => sub, Float => fsub;
 544         simd_mul: Uint, Int => mul, Float => fmul;
 545         simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
 546         simd_rem: Uint => urem, Int => srem, Float => frem;
 547         simd_shl: Uint, Int => shl;
 548         simd_shr: Uint => lshr, Int => ashr;
 549         simd_and: Uint, Int => and;
 550         simd_or: Uint, Int => or; // FIXME(antoyo): calling `or` might not work on vectors.
 551         simd_xor: Uint, Int => xor;
 552     }
 553
 554     macro_rules! arith_unary {
 555         ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
 556             $(if name == sym::$name {
 557                 match in_elem.kind() {
 558                     $($(ty::$p(_))|* => {
 559                         return Ok(bx.$call(args[0].immediate()))
 560                     })*
 561                     _ => {},
 562                 }
 563                 require!(false,
 564                          "unsupported operation on `{}` with element `{}`",
 565                          in_ty,
 566                          in_elem)
 567             })*
 568         }
 569     }
 570
 571     arith_unary! {
 572         simd_neg: Int => neg, Float => fneg;
 573     }
 574
 575     #[cfg(feature="master")]
 576     if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
 577         let lhs = args[0].immediate();
 578         let rhs = args[1].immediate();
 579         let is_add = name == sym::simd_saturating_add;
 580         let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
 581         let (signed, elem_width, elem_ty) = match *in_elem.kind() {
 582             ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)),
 583             ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)),
 584             _ => {
 585                 return_error!(
 586                     "expected element type `{}` of vector type `{}` \
 587                      to be a signed or unsigned integer type",
 588                     arg_tys[0].simd_size_and_type(bx.tcx()).1,
 589                     arg_tys[0]
 590                 );
 591             }
 592         };
 593         let builtin_name =
 594             match (signed, is_add, in_len, elem_width) {
 595                 (true, true, 32, 8) => "__builtin_ia32_paddsb256", // TODO(antoyo): cast arguments to unsigned.
 596                 (false, true, 32, 8) => "__builtin_ia32_paddusb256",
 597                 (true, true, 16, 16) => "__builtin_ia32_paddsw256",
 598                 (false, true, 16, 16) => "__builtin_ia32_paddusw256",
 599                 (true, false, 16, 16) => "__builtin_ia32_psubsw256",
 600                 (false, false, 16, 16) => "__builtin_ia32_psubusw256",
 601                 (true, false, 32, 8) => "__builtin_ia32_psubsb256",
 602                 (false, false, 32, 8) => "__builtin_ia32_psubusb256",
 603                 _ => unimplemented!("signed: {}, is_add: {}, in_len: {}, elem_width: {}", signed, is_add, in_len, elem_width),
 604             };
 605         let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
 606
 607         let func = bx.context.get_target_builtin_function(builtin_name);
 608         let param1_type = func.get_param(0).to_rvalue().get_type();
 609         let param2_type = func.get_param(1).to_rvalue().get_type();
 610         let lhs = bx.cx.bitcast_if_needed(lhs, param1_type);
 611         let rhs = bx.cx.bitcast_if_needed(rhs, param2_type);
 612         let result = bx.context.new_call(None, func, &[lhs, rhs]);
 613         // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
 614         return Ok(bx.context.new_bitcast(None, result, vec_ty));
 615     }
 616
 617     macro_rules! arith_red {
 618         ($name:ident : $vec_op:expr, $float_reduce:ident, $ordered:expr, $op:ident,
 619          $identity:expr) => {
 620             if name == sym::$name {
 621                 require!(
 622                     ret_ty == in_elem,
 623                     "expected return type `{}` (element of input `{}`), found `{}`",
 624                     in_elem,
 625                     in_ty,
 626                     ret_ty
 627                 );
 628                 return match in_elem.kind() {
 629                     ty::Int(_) | ty::Uint(_) => {
 630                         let r = bx.vector_reduce_op(args[0].immediate(), $vec_op);
 631                         if $ordered {
 632                             // if overflow occurs, the result is the
 633                             // mathematical result modulo 2^n:
 634                             Ok(bx.$op(args[1].immediate(), r))
 635                         }
 636                         else {
 637                             Ok(bx.vector_reduce_op(args[0].immediate(), $vec_op))
 638                         }
 639                     }
 640                     ty::Float(_) => {
 641                         if $ordered {
 642                             // ordered arithmetic reductions take an accumulator
 643                             let acc = args[1].immediate();
 644                             Ok(bx.$float_reduce(acc, args[0].immediate()))
 645                         }
 646                         else {
 647                             Ok(bx.vector_reduce_op(args[0].immediate(), $vec_op))
 648                         }
 649                     }
 650                     _ => return_error!(
 651                         "unsupported {} from `{}` with element `{}` to `{}`",
 652                         sym::$name,
 653                         in_ty,
 654                         in_elem,
 655                         ret_ty
 656                     ),
 657                 };
 658             }
 659         };
 660     }
 661
 662     arith_red!(
 663         simd_reduce_add_unordered: BinaryOp::Plus,
 664         vector_reduce_fadd_fast,
 665         false,
 666         add,
 667         0.0 // TODO: Use this argument.
 668     );
 669     arith_red!(
 670         simd_reduce_mul_unordered: BinaryOp::Mult,
 671         vector_reduce_fmul_fast,
 672         false,
 673         mul,
 674         1.0
 675     );
 676
 677     macro_rules! minmax_red {
 678         ($name:ident: $reduction:ident) => {
 679             if name == sym::$name {
 680                 require!(
 681                     ret_ty == in_elem,
 682                     "expected return type `{}` (element of input `{}`), found `{}`",
 683                     in_elem,
 684                     in_ty,
 685                     ret_ty
 686                 );
 687                 return match in_elem.kind() {
 688                     ty::Int(_) | ty::Uint(_) | ty::Float(_) => Ok(bx.$reduction(args[0].immediate())),
 689                     _ => return_error!(
 690                         "unsupported {} from `{}` with element `{}` to `{}`",
 691                         sym::$name,
 692                         in_ty,
 693                         in_elem,
 694                         ret_ty
 695                     ),
 696                 };
 697             }
 698         };
 699     }
 700
 701     minmax_red!(simd_reduce_min: vector_reduce_min);
 702     minmax_red!(simd_reduce_max: vector_reduce_max);
 703
 704     macro_rules! bitwise_red {
 705         ($name:ident : $op:expr, $boolean:expr) => {
 706             if name == sym::$name {
 707                 let input = if !$boolean {
 708                     require!(
 709                         ret_ty == in_elem,
 710                         "expected return type `{}` (element of input `{}`), found `{}`",
 711                         in_elem,
 712                         in_ty,
 713                         ret_ty
 714                     );
 715                     args[0].immediate()
 716                 } else {
 717                     match in_elem.kind() {
 718                         ty::Int(_) | ty::Uint(_) => {}
 719                         _ => return_error!(
 720                             "unsupported {} from `{}` with element `{}` to `{}`",
 721                             sym::$name,
 722                             in_ty,
 723                             in_elem,
 724                             ret_ty
 725                         ),
 726                     }
 727
 728                     // boolean reductions operate on vectors of i1s:
 729                     let i1 = bx.type_i1();
 730                     let i1xn = bx.type_vector(i1, in_len as u64);
 731                     bx.trunc(args[0].immediate(), i1xn)
 732                 };
 733                 return match in_elem.kind() {
 734                     ty::Int(_) | ty::Uint(_) => {
 735                         let r = bx.vector_reduce_op(input, $op);
 736                         Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
 737                     }
 738                     _ => return_error!(
 739                         "unsupported {} from `{}` with element `{}` to `{}`",
 740                         sym::$name,
 741                         in_ty,
 742                         in_elem,
 743                         ret_ty
 744                     ),
 745                 };
 746             }
 747         };
 748     }
 749
 750     bitwise_red!(simd_reduce_and: BinaryOp::BitwiseAnd, false);
 751     bitwise_red!(simd_reduce_or: BinaryOp::BitwiseOr, false);
 752
 753     unimplemented!("simd {}", name);
 754 }