compiler/rustc_codegen_gcc/src/intrinsic/simd.rs

   1 use gccjit::{RValue, Type};
   2 use rustc_codegen_ssa::base::compare_simd_types;
   3 use rustc_codegen_ssa::common::{TypeKind, span_invalid_monomorphization_error};
   4 use rustc_codegen_ssa::mir::operand::OperandRef;
   5 use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods};
   6 use rustc_hir as hir;
   7 use rustc_middle::span_bug;
   8 use rustc_middle::ty::layout::HasTyCtxt;
   9 use rustc_middle::ty::{self, Ty};
  10 use rustc_span::{Span, Symbol, sym};
  11
  12 use crate::builder::Builder;
  13
  14 pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>, name: Symbol, callee_ty: Ty<'tcx>, args: &[OperandRef<'tcx, RValue<'gcc>>], ret_ty: Ty<'tcx>, llret_ty: Type<'gcc>, span: Span) -> Result<RValue<'gcc>, ()> {
  15     //println!("Generic simd: {}", name);
  16
  17     // macros for error handling:
  18     macro_rules! emit_error {
  19         ($msg: tt) => {
  20             emit_error!($msg, )
  21         };
  22         ($msg: tt, $($fmt: tt)*) => {
  23             span_invalid_monomorphization_error(
  24                 bx.sess(), span,
  25                 &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg),
  26                          name, $($fmt)*));
  27         }
  28     }
  29
  30     macro_rules! return_error {
  31         ($($fmt: tt)*) => {
  32             {
  33                 emit_error!($($fmt)*);
  34                 return Err(());
  35             }
  36         }
  37     }
  38
  39     macro_rules! require {
  40         ($cond: expr, $($fmt: tt)*) => {
  41             if !$cond {
  42                 return_error!($($fmt)*);
  43             }
  44         };
  45     }
  46
  47     macro_rules! require_simd {
  48         ($ty: expr, $position: expr) => {
  49             require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty)
  50         };
  51     }
  52
  53     let tcx = bx.tcx();
  54     let sig =
  55         tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx));
  56     let arg_tys = sig.inputs();
  57     let name_str = &*name.as_str();
  58
  59     /*if name == sym::simd_select_bitmask {
  60         let in_ty = arg_tys[0];
  61         let m_len = match in_ty.kind() {
  62             // Note that this `.unwrap()` crashes for isize/usize, that's sort
  63             // of intentional as there's not currently a use case for that.
  64             ty::Int(i) => i.bit_width().unwrap(),
  65             ty::Uint(i) => i.bit_width().unwrap(),
  66             _ => return_error!("`{}` is not an integral type", in_ty),
  67         };
  68         require_simd!(arg_tys[1], "argument");
  69         let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
  70         require!(
  71             // Allow masks for vectors with fewer than 8 elements to be
  72             // represented with a u8 or i8.
  73             m_len == v_len || (m_len == 8 && v_len < 8),
  74             "mismatched lengths: mask length `{}` != other vector length `{}`",
  75             m_len,
  76             v_len
  77         );
  78         let i1 = bx.type_i1();
  79         let im = bx.type_ix(v_len);
  80         let i1xn = bx.type_vector(i1, v_len);
  81         let m_im = bx.trunc(args[0].immediate(), im);
  82         let m_i1s = bx.bitcast(m_im, i1xn);
  83         return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
  84     }*/
  85
  86     // every intrinsic below takes a SIMD vector as its first argument
  87     require_simd!(arg_tys[0], "input");
  88     let in_ty = arg_tys[0];
  89
  90     let comparison = match name {
  91         sym::simd_eq => Some(hir::BinOpKind::Eq),
  92         sym::simd_ne => Some(hir::BinOpKind::Ne),
  93         sym::simd_lt => Some(hir::BinOpKind::Lt),
  94         sym::simd_le => Some(hir::BinOpKind::Le),
  95         sym::simd_gt => Some(hir::BinOpKind::Gt),
  96         sym::simd_ge => Some(hir::BinOpKind::Ge),
  97         _ => None,
  98     };
  99
 100     let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx());
 101     if let Some(cmp_op) = comparison {
 102         require_simd!(ret_ty, "return");
 103
 104         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
 105         require!(
 106             in_len == out_len,
 107             "expected return type with length {} (same as input type `{}`), \
 108              found `{}` with length {}",
 109             in_len,
 110             in_ty,
 111             ret_ty,
 112             out_len
 113         );
 114         require!(
 115             bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
 116             "expected return type with integer elements, found `{}` with non-integer `{}`",
 117             ret_ty,
 118             out_ty
 119         );
 120
 121         return Ok(compare_simd_types(
 122             bx,
 123             args[0].immediate(),
 124             args[1].immediate(),
 125             in_elem,
 126             llret_ty,
 127             cmp_op,
 128         ));
 129     }
 130
 131     if let Some(stripped) = name_str.strip_prefix("simd_shuffle") {
 132         let n: u64 = stripped.parse().unwrap_or_else(|_| {
 133             span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")
 134         });
 135
 136         require_simd!(ret_ty, "return");
 137
 138         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
 139         require!(
 140             out_len == n,
 141             "expected return type of length {}, found `{}` with length {}",
 142             n,
 143             ret_ty,
 144             out_len
 145         );
 146         require!(
 147             in_elem == out_ty,
 148             "expected return element type `{}` (element of input `{}`), \
 149              found `{}` with element type `{}`",
 150             in_elem,
 151             in_ty,
 152             ret_ty,
 153             out_ty
 154         );
 155
 156         //let total_len = u128::from(in_len) * 2;
 157
 158         let vector = args[2].immediate();
 159
 160         // TODO:
 161         /*let indices: Option<Vec<_>> = (0..n)
 162             .map(|i| {
 163                 let arg_idx = i;
 164                 let val = bx.const_get_vector_element(vector, i as u64);
 165                 match bx.const_to_opt_u128(val, true) {
 166                     None => {
 167                         emit_error!("shuffle index #{} is not a constant", arg_idx);
 168                         None
 169                     }
 170                     Some(idx) if idx >= total_len => {
 171                         emit_error!(
 172                             "shuffle index #{} is out of bounds (limit {})",
 173                             arg_idx,
 174                             total_len
 175                         );
 176                         None
 177                     }
 178                     Some(idx) => Some(bx.const_i32(idx as i32)),
 179                 }
 180             })
 181             .collect();
 182         let indices = match indices {
 183             Some(i) => i,
 184             None => return Ok(bx.const_null(llret_ty)),
 185         };*/
 186
 187         return Ok(bx.shuffle_vector(
 188             args[0].immediate(),
 189             args[1].immediate(),
 190             vector,
 191         ));
 192     }
 193
 194     /*if name == sym::simd_insert {
 195         require!(
 196             in_elem == arg_tys[2],
 197             "expected inserted type `{}` (element of input `{}`), found `{}`",
 198             in_elem,
 199             in_ty,
 200             arg_tys[2]
 201         );
 202         return Ok(bx.insert_element(
 203             args[0].immediate(),
 204             args[2].immediate(),
 205             args[1].immediate(),
 206         ));
 207     }
 208     if name == sym::simd_extract {
 209         require!(
 210             ret_ty == in_elem,
 211             "expected return type `{}` (element of input `{}`), found `{}`",
 212             in_elem,
 213             in_ty,
 214             ret_ty
 215         );
 216         return Ok(bx.extract_element(args[0].immediate(), args[1].immediate()));
 217     }
 218
 219     if name == sym::simd_select {
 220         let m_elem_ty = in_elem;
 221         let m_len = in_len;
 222         require_simd!(arg_tys[1], "argument");
 223         let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
 224         require!(
 225             m_len == v_len,
 226             "mismatched lengths: mask length `{}` != other vector length `{}`",
 227             m_len,
 228             v_len
 229         );
 230         match m_elem_ty.kind() {
 231             ty::Int(_) => {}
 232             _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty),
 233         }
 234         // truncate the mask to a vector of i1s
 235         let i1 = bx.type_i1();
 236         let i1xn = bx.type_vector(i1, m_len as u64);
 237         let m_i1s = bx.trunc(args[0].immediate(), i1xn);
 238         return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
 239     }
 240
 241     if name == sym::simd_bitmask {
 242         // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a
 243         // vector mask and returns an unsigned integer containing the most
 244         // significant bit (MSB) of each lane.
 245
 246         // If the vector has less than 8 lanes, an u8 is returned with zeroed
 247         // trailing bits.
 248         let expected_int_bits = in_len.max(8);
 249         match ret_ty.kind() {
 250             ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => (),
 251             _ => return_error!("bitmask `{}`, expected `u{}`", ret_ty, expected_int_bits),
 252         }
 253
 254         // Integer vector <i{in_bitwidth} x in_len>:
 255         let (i_xn, in_elem_bitwidth) = match in_elem.kind() {
 256             ty::Int(i) => (
 257                 args[0].immediate(),
 258                 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()),
 259             ),
 260             ty::Uint(i) => (
 261                 args[0].immediate(),
 262                 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()),
 263             ),
 264             _ => return_error!(
 265                 "vector argument `{}`'s element type `{}`, expected integer element type",
 266                 in_ty,
 267                 in_elem
 268             ),
 269         };
 270
 271         // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position.
 272         let shift_indices =
 273             vec![
 274                 bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
 275                 in_len as _
 276             ];
 277         let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
 278         // Truncate vector to an <i1 x N>
 279         let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len));
 280         // Bitcast <i1 x N> to iN:
 281         let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
 282         // Zero-extend iN to the bitmask type:
 283         return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
 284     }
 285
 286     fn simd_simple_float_intrinsic<'a, 'gcc, 'tcx>(
 287         name: Symbol,
 288         in_elem: &::rustc_middle::ty::TyS<'_>,
 289         in_ty: &::rustc_middle::ty::TyS<'_>,
 290         in_len: u64,
 291         bx: &mut Builder<'a, 'gcc, 'tcx>,
 292         span: Span,
 293         args: &[OperandRef<'tcx, RValue<'gcc>>],
 294     ) -> Result<RValue<'gcc>, ()> {
 295         macro_rules! emit_error {
 296             ($msg: tt) => {
 297                 emit_error!($msg, )
 298             };
 299             ($msg: tt, $($fmt: tt)*) => {
 300                 span_invalid_monomorphization_error(
 301                     bx.sess(), span,
 302                     &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg),
 303                              name, $($fmt)*));
 304             }
 305         }
 306         macro_rules! return_error {
 307             ($($fmt: tt)*) => {
 308                 {
 309                     emit_error!($($fmt)*);
 310                     return Err(());
 311                 }
 312             }
 313         }
 314
 315         let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() {
 316             let elem_ty = bx.cx.type_float_from_ty(*f);
 317             match f.bit_width() {
 318                 32 => ("f32", elem_ty),
 319                 64 => ("f64", elem_ty),
 320                 _ => {
 321                     return_error!(
 322                         "unsupported element type `{}` of floating-point vector `{}`",
 323                         f.name_str(),
 324                         in_ty
 325                     );
 326                 }
 327             }
 328         } else {
 329             return_error!("`{}` is not a floating-point type", in_ty);
 330         };
 331
 332         let vec_ty = bx.type_vector(elem_ty, in_len);
 333
 334         let (intr_name, fn_ty) = match name {
 335             sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
 336             sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
 337             sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
 338             sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
 339             sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
 340             sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
 341             sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
 342             sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
 343             sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
 344             sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
 345             sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
 346             sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
 347             sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
 348             sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
 349             sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
 350             sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
 351             _ => return_error!("unrecognized intrinsic `{}`", name),
 352         };
 353         let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
 354         let f = bx.declare_cfn(&llvm_name, fn_ty);
 355         let c = bx.call(f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
 356         Ok(c)
 357     }
 358
 359     if std::matches!(
 360         name,
 361         sym::simd_ceil
 362             | sym::simd_fabs
 363             | sym::simd_fcos
 364             | sym::simd_fexp2
 365             | sym::simd_fexp
 366             | sym::simd_flog10
 367             | sym::simd_flog2
 368             | sym::simd_flog
 369             | sym::simd_floor
 370             | sym::simd_fma
 371             | sym::simd_fpow
 372             | sym::simd_fpowi
 373             | sym::simd_fsin
 374             | sym::simd_fsqrt
 375             | sym::simd_round
 376             | sym::simd_trunc
 377     ) {
 378         return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
 379     }
 380
 381     // FIXME: use:
 382     //  https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182
 383     //  https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81
 384     fn llvm_vector_str(elem_ty: Ty<'_>, vec_len: u64, no_pointers: usize) -> String {
 385         let p0s: String = "p0".repeat(no_pointers);
 386         match *elem_ty.kind() {
 387             ty::Int(v) => format!("v{}{}i{}", vec_len, p0s, v.bit_width().unwrap()),
 388             ty::Uint(v) => format!("v{}{}i{}", vec_len, p0s, v.bit_width().unwrap()),
 389             ty::Float(v) => format!("v{}{}f{}", vec_len, p0s, v.bit_width()),
 390             _ => unreachable!(),
 391         }
 392     }
 393
 394     fn gcc_vector_ty<'gcc>(
 395         cx: &CodegenCx<'gcc, '_>,
 396         elem_ty: Ty<'_>,
 397         vec_len: u64,
 398         mut no_pointers: usize,
 399     ) -> Type<'gcc> {
 400         // FIXME: use cx.layout_of(ty).llvm_type() ?
 401         let mut elem_ty = match *elem_ty.kind() {
 402             ty::Int(v) => cx.type_int_from_ty(v),
 403             ty::Uint(v) => cx.type_uint_from_ty(v),
 404             ty::Float(v) => cx.type_float_from_ty(v),
 405             _ => unreachable!(),
 406         };
 407         while no_pointers > 0 {
 408             elem_ty = cx.type_ptr_to(elem_ty);
 409             no_pointers -= 1;
 410         }
 411         cx.type_vector(elem_ty, vec_len)
 412     }
 413
 414     if name == sym::simd_gather {
 415         // simd_gather(values: <N x T>, pointers: <N x *_ T>,
 416         //             mask: <N x i{M}>) -> <N x T>
 417         // * N: number of elements in the input vectors
 418         // * T: type of the element to load
 419         // * M: any integer width is supported, will be truncated to i1
 420
 421         // All types must be simd vector types
 422         require_simd!(in_ty, "first");
 423         require_simd!(arg_tys[1], "second");
 424         require_simd!(arg_tys[2], "third");
 425         require_simd!(ret_ty, "return");
 426
 427         // Of the same length:
 428         let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
 429         let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
 430         require!(
 431             in_len == out_len,
 432             "expected {} argument with length {} (same as input type `{}`), \
 433              found `{}` with length {}",
 434             "second",
 435             in_len,
 436             in_ty,
 437             arg_tys[1],
 438             out_len
 439         );
 440         require!(
 441             in_len == out_len2,
 442             "expected {} argument with length {} (same as input type `{}`), \
 443              found `{}` with length {}",
 444             "third",
 445             in_len,
 446             in_ty,
 447             arg_tys[2],
 448             out_len2
 449         );
 450
 451         // The return type must match the first argument type
 452         require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty);
 453
 454         // This counts how many pointers
 455         fn ptr_count(t: Ty<'_>) -> usize {
 456             match t.kind() {
 457                 ty::RawPtr(p) => 1 + ptr_count(p.ty),
 458                 _ => 0,
 459             }
 460         }
 461
 462         // Non-ptr type
 463         fn non_ptr(t: Ty<'_>) -> Ty<'_> {
 464             match t.kind() {
 465                 ty::RawPtr(p) => non_ptr(p.ty),
 466                 _ => t,
 467             }
 468         }
 469
 470         // The second argument must be a simd vector with an element type that's a pointer
 471         // to the element type of the first argument
 472         let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
 473         let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
 474         let (pointer_count, underlying_ty) = match element_ty1.kind() {
 475             ty::RawPtr(p) if p.ty == in_elem => (ptr_count(element_ty1), non_ptr(element_ty1)),
 476             _ => {
 477                 require!(
 478                     false,
 479                     "expected element type `{}` of second argument `{}` \
 480                         to be a pointer to the element type `{}` of the first \
 481                         argument `{}`, found `{}` != `*_ {}`",
 482                     element_ty1,
 483                     arg_tys[1],
 484                     in_elem,
 485                     in_ty,
 486                     element_ty1,
 487                     in_elem
 488                 );
 489                 unreachable!();
 490             }
 491         };
 492         assert!(pointer_count > 0);
 493         assert_eq!(pointer_count - 1, ptr_count(element_ty0));
 494         assert_eq!(underlying_ty, non_ptr(element_ty0));
 495
 496         // The element type of the third argument must be a signed integer type of any width:
 497         let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
 498         match element_ty2.kind() {
 499             ty::Int(_) => (),
 500             _ => {
 501                 require!(
 502                     false,
 503                     "expected element type `{}` of third argument `{}` \
 504                                  to be a signed integer type",
 505                     element_ty2,
 506                     arg_tys[2]
 507                 );
 508             }
 509         }
 510
 511         // Alignment of T, must be a constant integer value:
 512         let alignment_ty = bx.type_i32();
 513         let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
 514
 515         // Truncate the mask vector to a vector of i1s:
 516         let (mask, mask_ty) = {
 517             let i1 = bx.type_i1();
 518             let i1xn = bx.type_vector(i1, in_len);
 519             (bx.trunc(args[2].immediate(), i1xn), i1xn)
 520         };
 521
 522         // Type of the vector of pointers:
 523         let llvm_pointer_vec_ty = gcc_vector_ty(bx, underlying_ty, in_len, pointer_count);
 524         let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count);
 525
 526         // Type of the vector of elements:
 527         let llvm_elem_vec_ty = gcc_vector_ty(bx, underlying_ty, in_len, pointer_count - 1);
 528         let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1);
 529
 530         let llvm_intrinsic =
 531             format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str);
 532         let f = bx.declare_cfn(
 533             &llvm_intrinsic,
 534             bx.type_func(
 535                 &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty],
 536                 llvm_elem_vec_ty,
 537             ),
 538         );
 539         let v = bx.call(f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None);
 540         return Ok(v);
 541     }
 542
 543     if name == sym::simd_scatter {
 544         // simd_scatter(values: <N x T>, pointers: <N x *mut T>,
 545         //             mask: <N x i{M}>) -> ()
 546         // * N: number of elements in the input vectors
 547         // * T: type of the element to load
 548         // * M: any integer width is supported, will be truncated to i1
 549
 550         // All types must be simd vector types
 551         require_simd!(in_ty, "first");
 552         require_simd!(arg_tys[1], "second");
 553         require_simd!(arg_tys[2], "third");
 554
 555         // Of the same length:
 556         let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx());
 557         let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
 558         require!(
 559             in_len == element_len1,
 560             "expected {} argument with length {} (same as input type `{}`), \
 561             found `{}` with length {}",
 562             "second",
 563             in_len,
 564             in_ty,
 565             arg_tys[1],
 566             element_len1
 567         );
 568         require!(
 569             in_len == element_len2,
 570             "expected {} argument with length {} (same as input type `{}`), \
 571             found `{}` with length {}",
 572             "third",
 573             in_len,
 574             in_ty,
 575             arg_tys[2],
 576             element_len2
 577         );
 578
 579         // This counts how many pointers
 580         fn ptr_count(t: Ty<'_>) -> usize {
 581             match t.kind() {
 582                 ty::RawPtr(p) => 1 + ptr_count(p.ty),
 583                 _ => 0,
 584             }
 585         }
 586
 587         // Non-ptr type
 588         fn non_ptr(t: Ty<'_>) -> Ty<'_> {
 589             match t.kind() {
 590                 ty::RawPtr(p) => non_ptr(p.ty),
 591                 _ => t,
 592             }
 593         }
 594
 595         // The second argument must be a simd vector with an element type that's a pointer
 596         // to the element type of the first argument
 597         let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
 598         let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
 599         let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
 600         let (pointer_count, underlying_ty) = match element_ty1.kind() {
 601             ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => {
 602                 (ptr_count(element_ty1), non_ptr(element_ty1))
 603             }
 604             _ => {
 605                 require!(
 606                     false,
 607                     "expected element type `{}` of second argument `{}` \
 608                         to be a pointer to the element type `{}` of the first \
 609                         argument `{}`, found `{}` != `*mut {}`",
 610                     element_ty1,
 611                     arg_tys[1],
 612                     in_elem,
 613                     in_ty,
 614                     element_ty1,
 615                     in_elem
 616                 );
 617                 unreachable!();
 618             }
 619         };
 620         assert!(pointer_count > 0);
 621         assert_eq!(pointer_count - 1, ptr_count(element_ty0));
 622         assert_eq!(underlying_ty, non_ptr(element_ty0));
 623
 624         // The element type of the third argument must be a signed integer type of any width:
 625         match element_ty2.kind() {
 626             ty::Int(_) => (),
 627             _ => {
 628                 require!(
 629                     false,
 630                     "expected element type `{}` of third argument `{}` \
 631                          be a signed integer type",
 632                     element_ty2,
 633                     arg_tys[2]
 634                 );
 635             }
 636         }
 637
 638         // Alignment of T, must be a constant integer value:
 639         let alignment_ty = bx.type_i32();
 640         let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
 641
 642         // Truncate the mask vector to a vector of i1s:
 643         let (mask, mask_ty) = {
 644             let i1 = bx.type_i1();
 645             let i1xn = bx.type_vector(i1, in_len);
 646             (bx.trunc(args[2].immediate(), i1xn), i1xn)
 647         };
 648
 649         let ret_t = bx.type_void();
 650
 651         // Type of the vector of pointers:
 652         let llvm_pointer_vec_ty = gcc_vector_ty(bx, underlying_ty, in_len, pointer_count);
 653         let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count);
 654
 655         // Type of the vector of elements:
 656         let llvm_elem_vec_ty = gcc_vector_ty(bx, underlying_ty, in_len, pointer_count - 1);
 657         let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1);
 658
 659         let llvm_intrinsic =
 660             format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str);
 661         let f = bx.declare_cfn(
 662             &llvm_intrinsic,
 663             bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t),
 664         );
 665         let v = bx.call(f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None);
 666         return Ok(v);
 667     }
 668
 669     macro_rules! arith_red {
 670         ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
 671          $identity:expr) => {
 672             if name == sym::$name {
 673                 require!(
 674                     ret_ty == in_elem,
 675                     "expected return type `{}` (element of input `{}`), found `{}`",
 676                     in_elem,
 677                     in_ty,
 678                     ret_ty
 679                 );
 680                 return match in_elem.kind() {
 681                     ty::Int(_) | ty::Uint(_) => {
 682                         let r = bx.$integer_reduce(args[0].immediate());
 683                         if $ordered {
 684                             // if overflow occurs, the result is the
 685                             // mathematical result modulo 2^n:
 686                             Ok(bx.$op(args[1].immediate(), r))
 687                         } else {
 688                             Ok(bx.$integer_reduce(args[0].immediate()))
 689                         }
 690                     }
 691                     ty::Float(f) => {
 692                         let acc = if $ordered {
 693                             // ordered arithmetic reductions take an accumulator
 694                             args[1].immediate()
 695                         } else {
 696                             // unordered arithmetic reductions use the identity accumulator
 697                             match f.bit_width() {
 698                                 32 => bx.const_real(bx.type_f32(), $identity),
 699                                 64 => bx.const_real(bx.type_f64(), $identity),
 700                                 v => return_error!(
 701                                     r#"
 702 unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#,
 703                                     sym::$name,
 704                                     in_ty,
 705                                     in_elem,
 706                                     v,
 707                                     ret_ty
 708                                 ),
 709                             }
 710                         };
 711                         Ok(bx.$float_reduce(acc, args[0].immediate()))
 712                     }
 713                     _ => return_error!(
 714                         "unsupported {} from `{}` with element `{}` to `{}`",
 715                         sym::$name,
 716                         in_ty,
 717                         in_elem,
 718                         ret_ty
 719                     ),
 720                 };
 721             }
 722         };
 723     }
 724
 725     arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, 0.0);
 726     arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
 727     arith_red!(
 728         simd_reduce_add_unordered: vector_reduce_add,
 729         vector_reduce_fadd_fast,
 730         false,
 731         add,
 732         0.0
 733     );
 734     arith_red!(
 735         simd_reduce_mul_unordered: vector_reduce_mul,
 736         vector_reduce_fmul_fast,
 737         false,
 738         mul,
 739         1.0
 740     );
 741
 742     macro_rules! minmax_red {
 743         ($name:ident: $int_red:ident, $float_red:ident) => {
 744             if name == sym::$name {
 745                 require!(
 746                     ret_ty == in_elem,
 747                     "expected return type `{}` (element of input `{}`), found `{}`",
 748                     in_elem,
 749                     in_ty,
 750                     ret_ty
 751                 );
 752                 return match in_elem.kind() {
 753                     ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
 754                     ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
 755                     ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())),
 756                     _ => return_error!(
 757                         "unsupported {} from `{}` with element `{}` to `{}`",
 758                         sym::$name,
 759                         in_ty,
 760                         in_elem,
 761                         ret_ty
 762                     ),
 763                 };
 764             }
 765         };
 766     }
 767
 768     minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
 769     minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
 770
 771     minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast);
 772     minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast);
 773
 774     macro_rules! bitwise_red {
 775         ($name:ident : $red:ident, $boolean:expr) => {
 776             if name == sym::$name {
 777                 let input = if !$boolean {
 778                     require!(
 779                         ret_ty == in_elem,
 780                         "expected return type `{}` (element of input `{}`), found `{}`",
 781                         in_elem,
 782                         in_ty,
 783                         ret_ty
 784                     );
 785                     args[0].immediate()
 786                 } else {
 787                     match in_elem.kind() {
 788                         ty::Int(_) | ty::Uint(_) => {}
 789                         _ => return_error!(
 790                             "unsupported {} from `{}` with element `{}` to `{}`",
 791                             sym::$name,
 792                             in_ty,
 793                             in_elem,
 794                             ret_ty
 795                         ),
 796                     }
 797
 798                     // boolean reductions operate on vectors of i1s:
 799                     let i1 = bx.type_i1();
 800                     let i1xn = bx.type_vector(i1, in_len as u64);
 801                     bx.trunc(args[0].immediate(), i1xn)
 802                 };
 803                 return match in_elem.kind() {
 804                     ty::Int(_) | ty::Uint(_) => {
 805                         let r = bx.$red(input);
 806                         Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
 807                     }
 808                     _ => return_error!(
 809                         "unsupported {} from `{}` with element `{}` to `{}`",
 810                         sym::$name,
 811                         in_ty,
 812                         in_elem,
 813                         ret_ty
 814                     ),
 815                 };
 816             }
 817         };
 818     }
 819
 820     bitwise_red!(simd_reduce_and: vector_reduce_and, false);
 821     bitwise_red!(simd_reduce_or: vector_reduce_or, false);
 822     bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
 823     bitwise_red!(simd_reduce_all: vector_reduce_and, true);
 824     bitwise_red!(simd_reduce_any: vector_reduce_or, true);
 825
 826     if name == sym::simd_cast {
 827         require_simd!(ret_ty, "return");
 828         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
 829         require!(
 830             in_len == out_len,
 831             "expected return type with length {} (same as input type `{}`), \
 832                   found `{}` with length {}",
 833             in_len,
 834             in_ty,
 835             ret_ty,
 836             out_len
 837         );
 838         // casting cares about nominal type, not just structural type
 839         if in_elem == out_elem {
 840             return Ok(args[0].immediate());
 841         }
 842
 843         enum Style {
 844             Float,
 845             Int(/* is signed? */ bool),
 846             Unsupported,
 847         }
 848
 849         let (in_style, in_width) = match in_elem.kind() {
 850             // vectors of pointer-sized integers should've been
 851             // disallowed before here, so this unwrap is safe.
 852             ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()),
 853             ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()),
 854             ty::Float(f) => (Style::Float, f.bit_width()),
 855             _ => (Style::Unsupported, 0),
 856         };
 857         let (out_style, out_width) = match out_elem.kind() {
 858             ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()),
 859             ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()),
 860             ty::Float(f) => (Style::Float, f.bit_width()),
 861             _ => (Style::Unsupported, 0),
 862         };
 863
 864         match (in_style, out_style) {
 865             (Style::Int(in_is_signed), Style::Int(_)) => {
 866                 return Ok(match in_width.cmp(&out_width) {
 867                     Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
 868                     Ordering::Equal => args[0].immediate(),
 869                     Ordering::Less => {
 870                         if in_is_signed {
 871                             bx.sext(args[0].immediate(), llret_ty)
 872                         } else {
 873                             bx.zext(args[0].immediate(), llret_ty)
 874                         }
 875                     }
 876                 });
 877             }
 878             (Style::Int(in_is_signed), Style::Float) => {
 879                 return Ok(if in_is_signed {
 880                     bx.sitofp(args[0].immediate(), llret_ty)
 881                 } else {
 882                     bx.uitofp(args[0].immediate(), llret_ty)
 883                 });
 884             }
 885             (Style::Float, Style::Int(out_is_signed)) => {
 886                 return Ok(if out_is_signed {
 887                     bx.fptosi(args[0].immediate(), llret_ty)
 888                 } else {
 889                     bx.fptoui(args[0].immediate(), llret_ty)
 890                 });
 891             }
 892             (Style::Float, Style::Float) => {
 893                 return Ok(match in_width.cmp(&out_width) {
 894                     Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
 895                     Ordering::Equal => args[0].immediate(),
 896                     Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
 897                 });
 898             }
 899             _ => { /* Unsupported. Fallthrough. */ }
 900         }
 901         require!(
 902             false,
 903             "unsupported cast from `{}` with element `{}` to `{}` with element `{}`",
 904             in_ty,
 905             in_elem,
 906             ret_ty,
 907             out_elem
 908         );
 909     }*/
 910
 911     macro_rules! arith_binary {
 912         ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
 913             $(if name == sym::$name {
 914                 match in_elem.kind() {
 915                     $($(ty::$p(_))|* => {
 916                         return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
 917                     })*
 918                     _ => {},
 919                 }
 920                 require!(false,
 921                          "unsupported operation on `{}` with element `{}`",
 922                          in_ty,
 923                          in_elem)
 924             })*
 925         }
 926     }
 927
 928     arith_binary! {
 929         simd_add: Uint, Int => add, Float => fadd;
 930         simd_sub: Uint, Int => sub, Float => fsub;
 931         simd_mul: Uint, Int => mul, Float => fmul;
 932         simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
 933         simd_rem: Uint => urem, Int => srem, Float => frem;
 934         simd_shl: Uint, Int => shl;
 935         simd_shr: Uint => lshr, Int => ashr;
 936         simd_and: Uint, Int => and;
 937         simd_or: Uint, Int => or; // FIXME: calling or might not work on vectors.
 938         simd_xor: Uint, Int => xor;
 939         /*simd_fmax: Float => maxnum;
 940         simd_fmin: Float => minnum;*/
 941     }
 942
 943     /*macro_rules! arith_unary {
 944         ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
 945             $(if name == sym::$name {
 946                 match in_elem.kind() {
 947                     $($(ty::$p(_))|* => {
 948                         return Ok(bx.$call(args[0].immediate()))
 949                     })*
 950                     _ => {},
 951                 }
 952                 require!(false,
 953                          "unsupported operation on `{}` with element `{}`",
 954                          in_ty,
 955                          in_elem)
 956             })*
 957         }
 958     }
 959
 960     arith_unary! {
 961         simd_neg: Int => neg, Float => fneg;
 962     }
 963
 964     if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
 965         let lhs = args[0].immediate();
 966         let rhs = args[1].immediate();
 967         let is_add = name == sym::simd_saturating_add;
 968         let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
 969         let (signed, elem_width, elem_ty) = match *in_elem.kind() {
 970             ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)),
 971             ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)),
 972             _ => {
 973                 return_error!(
 974                     "expected element type `{}` of vector type `{}` \
 975                      to be a signed or unsigned integer type",
 976                     arg_tys[0].simd_size_and_type(bx.tcx()).1,
 977                     arg_tys[0]
 978                 );
 979             }
 980         };
 981         let llvm_intrinsic = &format!(
 982             "llvm.{}{}.sat.v{}i{}",
 983             if signed { 's' } else { 'u' },
 984             if is_add { "add" } else { "sub" },
 985             in_len,
 986             elem_width
 987         );
 988         let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
 989
 990         let f = bx.declare_cfn(
 991             &llvm_intrinsic,
 992             bx.type_func(&[vec_ty, vec_ty], vec_ty),
 993         );
 994         let v = bx.call(f, &[lhs, rhs], None);
 995         return Ok(v);
 996     }*/
 997
 998     unimplemented!("simd {}", name);
 999
1000     //span_bug!(span, "unknown SIMD intrinsic");
1001 }