src/librustc_trans/trans/intrinsic.rs

   1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 #![allow(non_upper_case_globals)]
  12
  13 use arena::TypedArena;
  14 use intrinsics::{self, Intrinsic};
  15 use libc;
  16 use llvm;
  17 use llvm::{SequentiallyConsistent, Acquire, Release, AtomicXchg, ValueRef, TypeKind};
  18 use middle::subst;
  19 use middle::subst::FnSpace;
  20 use trans::adt;
  21 use trans::attributes;
  22 use trans::base::*;
  23 use trans::build::*;
  24 use trans::callee;
  25 use trans::cleanup;
  26 use trans::cleanup::CleanupMethods;
  27 use trans::common::*;
  28 use trans::consts;
  29 use trans::datum::*;
  30 use trans::debuginfo::DebugLoc;
  31 use trans::declare;
  32 use trans::expr;
  33 use trans::glue;
  34 use trans::type_of::*;
  35 use trans::type_of;
  36 use trans::machine;
  37 use trans::machine::llsize_of;
  38 use trans::type_::Type;
  39 use middle::ty::{self, Ty, HasTypeFlags};
  40 use middle::subst::Substs;
  41 use rustc_front::hir;
  42 use syntax::abi::{self, RustIntrinsic};
  43 use syntax::ast;
  44 use syntax::ptr::P;
  45 use syntax::parse::token;
  46
  47 use rustc::session::Session;
  48 use syntax::codemap::Span;
  49
  50 use std::cmp::Ordering;
  51
  52 pub fn get_simple_intrinsic(ccx: &CrateContext, item: &hir::ForeignItem) -> Option<ValueRef> {
  53     let name = match &*item.name.as_str() {
  54         "sqrtf32" => "llvm.sqrt.f32",
  55         "sqrtf64" => "llvm.sqrt.f64",
  56         "powif32" => "llvm.powi.f32",
  57         "powif64" => "llvm.powi.f64",
  58         "sinf32" => "llvm.sin.f32",
  59         "sinf64" => "llvm.sin.f64",
  60         "cosf32" => "llvm.cos.f32",
  61         "cosf64" => "llvm.cos.f64",
  62         "powf32" => "llvm.pow.f32",
  63         "powf64" => "llvm.pow.f64",
  64         "expf32" => "llvm.exp.f32",
  65         "expf64" => "llvm.exp.f64",
  66         "exp2f32" => "llvm.exp2.f32",
  67         "exp2f64" => "llvm.exp2.f64",
  68         "logf32" => "llvm.log.f32",
  69         "logf64" => "llvm.log.f64",
  70         "log10f32" => "llvm.log10.f32",
  71         "log10f64" => "llvm.log10.f64",
  72         "log2f32" => "llvm.log2.f32",
  73         "log2f64" => "llvm.log2.f64",
  74         "fmaf32" => "llvm.fma.f32",
  75         "fmaf64" => "llvm.fma.f64",
  76         "fabsf32" => "llvm.fabs.f32",
  77         "fabsf64" => "llvm.fabs.f64",
  78         "copysignf32" => "llvm.copysign.f32",
  79         "copysignf64" => "llvm.copysign.f64",
  80         "floorf32" => "llvm.floor.f32",
  81         "floorf64" => "llvm.floor.f64",
  82         "ceilf32" => "llvm.ceil.f32",
  83         "ceilf64" => "llvm.ceil.f64",
  84         "truncf32" => "llvm.trunc.f32",
  85         "truncf64" => "llvm.trunc.f64",
  86         "rintf32" => "llvm.rint.f32",
  87         "rintf64" => "llvm.rint.f64",
  88         "nearbyintf32" => "llvm.nearbyint.f32",
  89         "nearbyintf64" => "llvm.nearbyint.f64",
  90         "roundf32" => "llvm.round.f32",
  91         "roundf64" => "llvm.round.f64",
  92         "ctpop8" => "llvm.ctpop.i8",
  93         "ctpop16" => "llvm.ctpop.i16",
  94         "ctpop32" => "llvm.ctpop.i32",
  95         "ctpop64" => "llvm.ctpop.i64",
  96         "bswap16" => "llvm.bswap.i16",
  97         "bswap32" => "llvm.bswap.i32",
  98         "bswap64" => "llvm.bswap.i64",
  99         "assume" => "llvm.assume",
 100         _ => return None
 101     };
 102     Some(ccx.get_intrinsic(&name))
 103 }
 104
 105 pub fn span_transmute_size_error(a: &Session, b: Span, msg: &str) {
 106     span_err!(a, b, E0512, "{}", msg);
 107 }
 108
 109 /// Performs late verification that intrinsics are used correctly. At present,
 110 /// the only intrinsic that needs such verification is `transmute`.
 111 pub fn check_intrinsics(ccx: &CrateContext) {
 112     let mut last_failing_id = None;
 113     for transmute_restriction in ccx.tcx().transmute_restrictions.borrow().iter() {
 114         // Sometimes, a single call to transmute will push multiple
 115         // type pairs to test in order to exhaustively test the
 116         // possibility around a type parameter. If one of those fails,
 117         // there is no sense reporting errors on the others.
 118         if last_failing_id == Some(transmute_restriction.id) {
 119             continue;
 120         }
 121
 122         debug!("transmute_restriction: {:?}", transmute_restriction);
 123
 124         assert!(!transmute_restriction.substituted_from.has_param_types());
 125         assert!(!transmute_restriction.substituted_to.has_param_types());
 126
 127         let llfromtype = type_of::sizing_type_of(ccx,
 128                                                  transmute_restriction.substituted_from);
 129         let lltotype = type_of::sizing_type_of(ccx,
 130                                                transmute_restriction.substituted_to);
 131         let from_type_size = machine::llbitsize_of_real(ccx, llfromtype);
 132         let to_type_size = machine::llbitsize_of_real(ccx, lltotype);
 133         if from_type_size != to_type_size {
 134             last_failing_id = Some(transmute_restriction.id);
 135
 136             if transmute_restriction.original_from != transmute_restriction.substituted_from {
 137                 span_transmute_size_error(ccx.sess(), transmute_restriction.span,
 138                     &format!("transmute called on types with potentially different sizes: \
 139                               {} (could be {} bit{}) to {} (could be {} bit{})",
 140                              transmute_restriction.original_from,
 141                              from_type_size as usize,
 142                              if from_type_size == 1 {""} else {"s"},
 143                              transmute_restriction.original_to,
 144                              to_type_size as usize,
 145                              if to_type_size == 1 {""} else {"s"}));
 146             } else {
 147                 span_transmute_size_error(ccx.sess(), transmute_restriction.span,
 148                     &format!("transmute called on types with different sizes: \
 149                               {} ({} bit{}) to {} ({} bit{})",
 150                              transmute_restriction.original_from,
 151                              from_type_size as usize,
 152                              if from_type_size == 1 {""} else {"s"},
 153                              transmute_restriction.original_to,
 154                              to_type_size as usize,
 155                              if to_type_size == 1 {""} else {"s"}));
 156             }
 157         }
 158     }
 159     ccx.sess().abort_if_errors();
 160 }
 161
 162 /// Remember to add all intrinsics here, in librustc_typeck/check/mod.rs,
 163 /// and in libcore/intrinsics.rs; if you need access to any llvm intrinsics,
 164 /// add them to librustc_trans/trans/context.rs
 165 pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>,
 166                                             node: ast::NodeId,
 167                                             callee_ty: Ty<'tcx>,
 168                                             cleanup_scope: cleanup::CustomScopeIndex,
 169                                             args: callee::CallArgs<'a, 'tcx>,
 170                                             dest: expr::Dest,
 171                                             substs: subst::Substs<'tcx>,
 172                                             call_info: NodeIdAndSpan)
 173                                             -> Result<'blk, 'tcx> {
 174     let fcx = bcx.fcx;
 175     let ccx = fcx.ccx;
 176     let tcx = bcx.tcx();
 177
 178     let _icx = push_ctxt("trans_intrinsic_call");
 179
 180     let (arg_tys, ret_ty) = match callee_ty.sty {
 181         ty::TyBareFn(_, ref f) => {
 182             (bcx.tcx().erase_late_bound_regions(&f.sig.inputs()),
 183              bcx.tcx().erase_late_bound_regions(&f.sig.output()))
 184         }
 185         _ => panic!("expected bare_fn in trans_intrinsic_call")
 186     };
 187     let foreign_item = tcx.map.expect_foreign_item(node);
 188     let name = foreign_item.name.as_str();
 189
 190     // For `transmute` we can just trans the input expr directly into dest
 191     if name == "transmute" {
 192         let llret_ty = type_of::type_of(ccx, ret_ty.unwrap());
 193         match args {
 194             callee::ArgExprs(arg_exprs) => {
 195                 assert_eq!(arg_exprs.len(), 1);
 196
 197                 let (in_type, out_type) = (*substs.types.get(FnSpace, 0),
 198                                            *substs.types.get(FnSpace, 1));
 199                 let llintype = type_of::type_of(ccx, in_type);
 200                 let llouttype = type_of::type_of(ccx, out_type);
 201
 202                 let in_type_size = machine::llbitsize_of_real(ccx, llintype);
 203                 let out_type_size = machine::llbitsize_of_real(ccx, llouttype);
 204
 205                 // This should be caught by the intrinsicck pass
 206                 assert_eq!(in_type_size, out_type_size);
 207
 208                 let nonpointer_nonaggregate = |llkind: TypeKind| -> bool {
 209                     use llvm::TypeKind::*;
 210                     match llkind {
 211                         Half | Float | Double | X86_FP80 | FP128 |
 212                             PPC_FP128 | Integer | Vector | X86_MMX => true,
 213                         _ => false
 214                     }
 215                 };
 216
 217                 // An approximation to which types can be directly cast via
 218                 // LLVM's bitcast.  This doesn't cover pointer -> pointer casts,
 219                 // but does, importantly, cover SIMD types.
 220                 let in_kind = llintype.kind();
 221                 let ret_kind = llret_ty.kind();
 222                 let bitcast_compatible =
 223                     (nonpointer_nonaggregate(in_kind) && nonpointer_nonaggregate(ret_kind)) || {
 224                         in_kind == TypeKind::Pointer && ret_kind == TypeKind::Pointer
 225                     };
 226
 227                 let dest = if bitcast_compatible {
 228                     // if we're here, the type is scalar-like (a primitive, a
 229                     // SIMD type or a pointer), and so can be handled as a
 230                     // by-value ValueRef and can also be directly bitcast to the
 231                     // target type.  Doing this special case makes conversions
 232                     // like `u32x4` -> `u64x2` much nicer for LLVM and so more
 233                     // efficient (these are done efficiently implicitly in C
 234                     // with the `__m128i` type and so this means Rust doesn't
 235                     // lose out there).
 236                     let expr = &*arg_exprs[0];
 237                     let datum = unpack_datum!(bcx, expr::trans(bcx, expr));
 238                     let datum = unpack_datum!(bcx, datum.to_rvalue_datum(bcx, "transmute_temp"));
 239                     let val = if datum.kind.is_by_ref() {
 240                         load_ty(bcx, datum.val, datum.ty)
 241                     } else {
 242                         from_arg_ty(bcx, datum.val, datum.ty)
 243                     };
 244
 245                     let cast_val = BitCast(bcx, val, llret_ty);
 246
 247                     match dest {
 248                         expr::SaveIn(d) => {
 249                             // this often occurs in a sequence like `Store(val,
 250                             // d); val2 = Load(d)`, so disappears easily.
 251                             Store(bcx, cast_val, d);
 252                         }
 253                         expr::Ignore => {}
 254                     }
 255                     dest
 256                 } else {
 257                     // The types are too complicated to do with a by-value
 258                     // bitcast, so pointer cast instead. We need to cast the
 259                     // dest so the types work out.
 260                     let dest = match dest {
 261                         expr::SaveIn(d) => expr::SaveIn(PointerCast(bcx, d, llintype.ptr_to())),
 262                         expr::Ignore => expr::Ignore
 263                     };
 264                     bcx = expr::trans_into(bcx, &*arg_exprs[0], dest);
 265                     dest
 266                 };
 267
 268                 fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean();
 269                 fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
 270
 271                 return match dest {
 272                     expr::SaveIn(d) => Result::new(bcx, d),
 273                     expr::Ignore => Result::new(bcx, C_undef(llret_ty.ptr_to()))
 274                 };
 275
 276             }
 277
 278             _ => {
 279                 ccx.sess().bug("expected expr as argument for transmute");
 280             }
 281         }
 282     }
 283
 284     // For `move_val_init` we can evaluate the destination address
 285     // (the first argument) and then trans the source value (the
 286     // second argument) directly into the resulting destination
 287     // address.
 288     if name == "move_val_init" {
 289         if let callee::ArgExprs(ref exprs) = args {
 290             let (dest_expr, source_expr) = if exprs.len() != 2 {
 291                 ccx.sess().bug("expected two exprs as arguments for `move_val_init` intrinsic");
 292             } else {
 293                 (&exprs[0], &exprs[1])
 294             };
 295
 296             // evaluate destination address
 297             let dest_datum = unpack_datum!(bcx, expr::trans(bcx, dest_expr));
 298             let dest_datum = unpack_datum!(
 299                 bcx, dest_datum.to_rvalue_datum(bcx, "arg"));
 300             let dest_datum = unpack_datum!(
 301                 bcx, dest_datum.to_appropriate_datum(bcx));
 302
 303             // `expr::trans_into(bcx, expr, dest)` is equiv to
 304             //
 305             //    `trans(bcx, expr).store_to_dest(dest)`,
 306             //
 307             // which for `dest == expr::SaveIn(addr)`, is equivalent to:
 308             //
 309             //    `trans(bcx, expr).store_to(bcx, addr)`.
 310             let lldest = expr::Dest::SaveIn(dest_datum.val);
 311             bcx = expr::trans_into(bcx, source_expr, lldest);
 312
 313             let llresult = C_nil(ccx);
 314             fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
 315
 316             return Result::new(bcx, llresult);
 317         } else {
 318             ccx.sess().bug("expected two exprs as arguments for `move_val_init` intrinsic");
 319         }
 320     }
 321
 322     let call_debug_location = DebugLoc::At(call_info.id, call_info.span);
 323
 324     // For `try` we need some custom control flow
 325     if &name[..] == "try" {
 326         if let callee::ArgExprs(ref exprs) = args {
 327             let (func, data) = if exprs.len() != 2 {
 328                 ccx.sess().bug("expected two exprs as arguments for \
 329                                 `try` intrinsic");
 330             } else {
 331                 (&exprs[0], &exprs[1])
 332             };
 333
 334             // translate arguments
 335             let func = unpack_datum!(bcx, expr::trans(bcx, func));
 336             let func = unpack_datum!(bcx, func.to_rvalue_datum(bcx, "func"));
 337             let data = unpack_datum!(bcx, expr::trans(bcx, data));
 338             let data = unpack_datum!(bcx, data.to_rvalue_datum(bcx, "data"));
 339
 340             let dest = match dest {
 341                 expr::SaveIn(d) => d,
 342                 expr::Ignore => alloc_ty(bcx, tcx.mk_mut_ptr(tcx.types.i8),
 343                                          "try_result"),
 344             };
 345
 346             // do the invoke
 347             bcx = try_intrinsic(bcx, func.val, data.val, dest,
 348                                 call_debug_location);
 349
 350             fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
 351             return Result::new(bcx, dest);
 352         } else {
 353             ccx.sess().bug("expected two exprs as arguments for \
 354                             `try` intrinsic");
 355         }
 356     }
 357
 358     // save the actual AST arguments for later (some places need to do
 359     // const-evaluation on them)
 360     let expr_arguments = match args {
 361         callee::ArgExprs(args) => Some(args),
 362         _ => None,
 363     };
 364
 365     // Push the arguments.
 366     let mut llargs = Vec::new();
 367     bcx = callee::trans_args(bcx,
 368                              args,
 369                              callee_ty,
 370                              &mut llargs,
 371                              cleanup::CustomScope(cleanup_scope),
 372                              false,
 373                              RustIntrinsic);
 374
 375     fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean();
 376
 377     // These are the only intrinsic functions that diverge.
 378     if name == "abort" {
 379         let llfn = ccx.get_intrinsic(&("llvm.trap"));
 380         Call(bcx, llfn, &[], None, call_debug_location);
 381         fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
 382         Unreachable(bcx);
 383         return Result::new(bcx, C_undef(Type::nil(ccx).ptr_to()));
 384     } else if &name[..] == "unreachable" {
 385         fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
 386         Unreachable(bcx);
 387         return Result::new(bcx, C_nil(ccx));
 388     }
 389
 390     let ret_ty = match ret_ty {
 391         ty::FnConverging(ret_ty) => ret_ty,
 392         ty::FnDiverging => unreachable!()
 393     };
 394
 395     let llret_ty = type_of::type_of(ccx, ret_ty);
 396
 397     // Get location to store the result. If the user does
 398     // not care about the result, just make a stack slot
 399     let llresult = match dest {
 400         expr::SaveIn(d) => d,
 401         expr::Ignore => {
 402             if !type_is_zero_size(ccx, ret_ty) {
 403                 let llresult = alloc_ty(bcx, ret_ty, "intrinsic_result");
 404                 call_lifetime_start(bcx, llresult);
 405                 llresult
 406             } else {
 407                 C_undef(llret_ty.ptr_to())
 408             }
 409         }
 410     };
 411
 412     let simple = get_simple_intrinsic(ccx, &*foreign_item);
 413     let llval = match (simple, &*name) {
 414         (Some(llfn), _) => {
 415             Call(bcx, llfn, &llargs, None, call_debug_location)
 416         }
 417         (_, "breakpoint") => {
 418             let llfn = ccx.get_intrinsic(&("llvm.debugtrap"));
 419             Call(bcx, llfn, &[], None, call_debug_location)
 420         }
 421         (_, "size_of") => {
 422             let tp_ty = *substs.types.get(FnSpace, 0);
 423             let lltp_ty = type_of::type_of(ccx, tp_ty);
 424             C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty))
 425         }
 426         (_, "size_of_val") => {
 427             let tp_ty = *substs.types.get(FnSpace, 0);
 428             if !type_is_sized(tcx, tp_ty) {
 429                 let (llsize, _) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]);
 430                 llsize
 431             } else {
 432                 let lltp_ty = type_of::type_of(ccx, tp_ty);
 433                 C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty))
 434             }
 435         }
 436         (_, "min_align_of") => {
 437             let tp_ty = *substs.types.get(FnSpace, 0);
 438             C_uint(ccx, type_of::align_of(ccx, tp_ty))
 439         }
 440         (_, "min_align_of_val") => {
 441             let tp_ty = *substs.types.get(FnSpace, 0);
 442             if !type_is_sized(tcx, tp_ty) {
 443                 let (_, llalign) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]);
 444                 llalign
 445             } else {
 446                 C_uint(ccx, type_of::align_of(ccx, tp_ty))
 447             }
 448         }
 449         (_, "pref_align_of") => {
 450             let tp_ty = *substs.types.get(FnSpace, 0);
 451             let lltp_ty = type_of::type_of(ccx, tp_ty);
 452             C_uint(ccx, machine::llalign_of_pref(ccx, lltp_ty))
 453         }
 454         (_, "drop_in_place") => {
 455             let tp_ty = *substs.types.get(FnSpace, 0);
 456             let ptr = if type_is_sized(tcx, tp_ty) {
 457                 llargs[0]
 458             } else {
 459                 let scratch = rvalue_scratch_datum(bcx, tp_ty, "tmp");
 460                 Store(bcx, llargs[0], expr::get_dataptr(bcx, scratch.val));
 461                 Store(bcx, llargs[1], expr::get_meta(bcx, scratch.val));
 462                 fcx.schedule_lifetime_end(cleanup::CustomScope(cleanup_scope), scratch.val);
 463                 scratch.val
 464             };
 465             glue::drop_ty(bcx, ptr, tp_ty, call_debug_location);
 466             C_nil(ccx)
 467         }
 468         (_, "type_name") => {
 469             let tp_ty = *substs.types.get(FnSpace, 0);
 470             let ty_name = token::intern_and_get_ident(&tp_ty.to_string());
 471             C_str_slice(ccx, ty_name)
 472         }
 473         (_, "type_id") => {
 474             let hash = ccx.tcx().hash_crate_independent(*substs.types.get(FnSpace, 0),
 475                                                         &ccx.link_meta().crate_hash);
 476             C_u64(ccx, hash)
 477         }
 478         (_, "init_dropped") => {
 479             let tp_ty = *substs.types.get(FnSpace, 0);
 480             if !return_type_is_void(ccx, tp_ty) {
 481                 drop_done_fill_mem(bcx, llresult, tp_ty);
 482             }
 483             C_nil(ccx)
 484         }
 485         (_, "init") => {
 486             let tp_ty = *substs.types.get(FnSpace, 0);
 487             if !return_type_is_void(ccx, tp_ty) {
 488                 // Just zero out the stack slot. (See comment on base::memzero for explanation)
 489                 init_zero_mem(bcx, llresult, tp_ty);
 490             }
 491             C_nil(ccx)
 492         }
 493         // Effectively no-ops
 494         (_, "uninit") | (_, "forget") => {
 495             C_nil(ccx)
 496         }
 497         (_, "needs_drop") => {
 498             let tp_ty = *substs.types.get(FnSpace, 0);
 499
 500             C_bool(ccx, bcx.fcx.type_needs_drop(tp_ty))
 501         }
 502         (_, "offset") => {
 503             let ptr = llargs[0];
 504             let offset = llargs[1];
 505             InBoundsGEP(bcx, ptr, &[offset])
 506         }
 507         (_, "arith_offset") => {
 508             let ptr = llargs[0];
 509             let offset = llargs[1];
 510             GEP(bcx, ptr, &[offset])
 511         }
 512
 513         (_, "copy_nonoverlapping") => {
 514             copy_intrinsic(bcx,
 515                            false,
 516                            false,
 517                            *substs.types.get(FnSpace, 0),
 518                            llargs[1],
 519                            llargs[0],
 520                            llargs[2],
 521                            call_debug_location)
 522         }
 523         (_, "copy") => {
 524             copy_intrinsic(bcx,
 525                            true,
 526                            false,
 527                            *substs.types.get(FnSpace, 0),
 528                            llargs[1],
 529                            llargs[0],
 530                            llargs[2],
 531                            call_debug_location)
 532         }
 533         (_, "write_bytes") => {
 534             memset_intrinsic(bcx,
 535                              false,
 536                              *substs.types.get(FnSpace, 0),
 537                              llargs[0],
 538                              llargs[1],
 539                              llargs[2],
 540                              call_debug_location)
 541         }
 542
 543         (_, "volatile_copy_nonoverlapping_memory") => {
 544             copy_intrinsic(bcx,
 545                            false,
 546                            true,
 547                            *substs.types.get(FnSpace, 0),
 548                            llargs[0],
 549                            llargs[1],
 550                            llargs[2],
 551                            call_debug_location)
 552         }
 553         (_, "volatile_copy_memory") => {
 554             copy_intrinsic(bcx,
 555                            true,
 556                            true,
 557                            *substs.types.get(FnSpace, 0),
 558                            llargs[0],
 559                            llargs[1],
 560                            llargs[2],
 561                            call_debug_location)
 562         }
 563         (_, "volatile_set_memory") => {
 564             memset_intrinsic(bcx,
 565                              true,
 566                              *substs.types.get(FnSpace, 0),
 567                              llargs[0],
 568                              llargs[1],
 569                              llargs[2],
 570                              call_debug_location)
 571         }
 572         (_, "volatile_load") => {
 573             let tp_ty = *substs.types.get(FnSpace, 0);
 574             let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 575             let load = VolatileLoad(bcx, ptr);
 576             unsafe {
 577                 llvm::LLVMSetAlignment(load, type_of::align_of(ccx, tp_ty));
 578             }
 579             to_arg_ty(bcx, load, tp_ty)
 580         },
 581         (_, "volatile_store") => {
 582             let tp_ty = *substs.types.get(FnSpace, 0);
 583             let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 584             let val = from_arg_ty(bcx, llargs[1], tp_ty);
 585             let store = VolatileStore(bcx, val, ptr);
 586             unsafe {
 587                 llvm::LLVMSetAlignment(store, type_of::align_of(ccx, tp_ty));
 588             }
 589             C_nil(ccx)
 590         },
 591
 592         (_, "ctlz8") => count_zeros_intrinsic(bcx,
 593                                               "llvm.ctlz.i8",
 594                                               llargs[0],
 595                                               call_debug_location),
 596         (_, "ctlz16") => count_zeros_intrinsic(bcx,
 597                                                "llvm.ctlz.i16",
 598                                                llargs[0],
 599                                                call_debug_location),
 600         (_, "ctlz32") => count_zeros_intrinsic(bcx,
 601                                                "llvm.ctlz.i32",
 602                                                llargs[0],
 603                                                call_debug_location),
 604         (_, "ctlz64") => count_zeros_intrinsic(bcx,
 605                                                "llvm.ctlz.i64",
 606                                                llargs[0],
 607                                                call_debug_location),
 608         (_, "cttz8") => count_zeros_intrinsic(bcx,
 609                                               "llvm.cttz.i8",
 610                                               llargs[0],
 611                                               call_debug_location),
 612         (_, "cttz16") => count_zeros_intrinsic(bcx,
 613                                                "llvm.cttz.i16",
 614                                                llargs[0],
 615                                                call_debug_location),
 616         (_, "cttz32") => count_zeros_intrinsic(bcx,
 617                                                "llvm.cttz.i32",
 618                                                llargs[0],
 619                                                call_debug_location),
 620         (_, "cttz64") => count_zeros_intrinsic(bcx,
 621                                                "llvm.cttz.i64",
 622                                                llargs[0],
 623                                                call_debug_location),
 624
 625         (_, "i8_add_with_overflow") =>
 626             with_overflow_intrinsic(bcx,
 627                                     "llvm.sadd.with.overflow.i8",
 628                                     llargs[0],
 629                                     llargs[1],
 630                                     llresult,
 631                                     call_debug_location),
 632         (_, "i16_add_with_overflow") =>
 633             with_overflow_intrinsic(bcx,
 634                                     "llvm.sadd.with.overflow.i16",
 635                                     llargs[0],
 636                                     llargs[1],
 637                                     llresult,
 638                                     call_debug_location),
 639         (_, "i32_add_with_overflow") =>
 640             with_overflow_intrinsic(bcx,
 641                                     "llvm.sadd.with.overflow.i32",
 642                                     llargs[0],
 643                                     llargs[1],
 644                                     llresult,
 645                                     call_debug_location),
 646         (_, "i64_add_with_overflow") =>
 647             with_overflow_intrinsic(bcx,
 648                                     "llvm.sadd.with.overflow.i64",
 649                                     llargs[0],
 650                                     llargs[1],
 651                                     llresult,
 652                                     call_debug_location),
 653
 654         (_, "u8_add_with_overflow") =>
 655             with_overflow_intrinsic(bcx,
 656                                     "llvm.uadd.with.overflow.i8",
 657                                     llargs[0],
 658                                     llargs[1],
 659                                     llresult,
 660                                     call_debug_location),
 661         (_, "u16_add_with_overflow") =>
 662             with_overflow_intrinsic(bcx,
 663                                     "llvm.uadd.with.overflow.i16",
 664                                     llargs[0],
 665                                     llargs[1],
 666                                     llresult,
 667                                     call_debug_location),
 668         (_, "u32_add_with_overflow") =>
 669             with_overflow_intrinsic(bcx,
 670                                     "llvm.uadd.with.overflow.i32",
 671                                     llargs[0],
 672                                     llargs[1],
 673                                     llresult,
 674                                     call_debug_location),
 675         (_, "u64_add_with_overflow") =>
 676             with_overflow_intrinsic(bcx,
 677                                     "llvm.uadd.with.overflow.i64",
 678                                     llargs[0],
 679                                     llargs[1],
 680                                     llresult,
 681                                     call_debug_location),
 682         (_, "i8_sub_with_overflow") =>
 683             with_overflow_intrinsic(bcx,
 684                                     "llvm.ssub.with.overflow.i8",
 685                                     llargs[0],
 686                                     llargs[1],
 687                                     llresult,
 688                                     call_debug_location),
 689         (_, "i16_sub_with_overflow") =>
 690             with_overflow_intrinsic(bcx,
 691                                     "llvm.ssub.with.overflow.i16",
 692                                     llargs[0],
 693                                     llargs[1],
 694                                     llresult,
 695                                     call_debug_location),
 696         (_, "i32_sub_with_overflow") =>
 697             with_overflow_intrinsic(bcx,
 698                                     "llvm.ssub.with.overflow.i32",
 699                                     llargs[0],
 700                                     llargs[1],
 701                                     llresult,
 702                                     call_debug_location),
 703         (_, "i64_sub_with_overflow") =>
 704             with_overflow_intrinsic(bcx,
 705                                     "llvm.ssub.with.overflow.i64",
 706                                     llargs[0],
 707                                     llargs[1],
 708                                     llresult,
 709                                     call_debug_location),
 710         (_, "u8_sub_with_overflow") =>
 711             with_overflow_intrinsic(bcx,
 712                                     "llvm.usub.with.overflow.i8",
 713                                     llargs[0],
 714                                     llargs[1],
 715                                     llresult,
 716                                     call_debug_location),
 717         (_, "u16_sub_with_overflow") =>
 718             with_overflow_intrinsic(bcx,
 719                                     "llvm.usub.with.overflow.i16",
 720                                     llargs[0],
 721                                     llargs[1],
 722                                     llresult,
 723                                     call_debug_location),
 724         (_, "u32_sub_with_overflow") =>
 725             with_overflow_intrinsic(bcx,
 726                                     "llvm.usub.with.overflow.i32",
 727                                     llargs[0],
 728                                     llargs[1],
 729                                     llresult,
 730                                     call_debug_location),
 731         (_, "u64_sub_with_overflow") =>
 732             with_overflow_intrinsic(bcx,
 733                                     "llvm.usub.with.overflow.i64",
 734                                     llargs[0],
 735                                     llargs[1],
 736                                     llresult,
 737                                     call_debug_location),
 738         (_, "i8_mul_with_overflow") =>
 739             with_overflow_intrinsic(bcx,
 740                                     "llvm.smul.with.overflow.i8",
 741                                     llargs[0],
 742                                     llargs[1],
 743                                     llresult,
 744                                     call_debug_location),
 745         (_, "i16_mul_with_overflow") =>
 746             with_overflow_intrinsic(bcx,
 747                                     "llvm.smul.with.overflow.i16",
 748                                     llargs[0],
 749                                     llargs[1],
 750                                     llresult,
 751                                     call_debug_location),
 752         (_, "i32_mul_with_overflow") =>
 753             with_overflow_intrinsic(bcx,
 754                                     "llvm.smul.with.overflow.i32",
 755                                     llargs[0],
 756                                     llargs[1],
 757                                     llresult,
 758                                     call_debug_location),
 759         (_, "i64_mul_with_overflow") =>
 760             with_overflow_intrinsic(bcx,
 761                                     "llvm.smul.with.overflow.i64",
 762                                     llargs[0],
 763                                     llargs[1],
 764                                     llresult,
 765                                     call_debug_location),
 766         (_, "u8_mul_with_overflow") =>
 767             with_overflow_intrinsic(bcx,
 768                                     "llvm.umul.with.overflow.i8",
 769                                     llargs[0],
 770                                     llargs[1],
 771                                     llresult,
 772                                     call_debug_location),
 773         (_, "u16_mul_with_overflow") =>
 774             with_overflow_intrinsic(bcx,
 775                                     "llvm.umul.with.overflow.i16",
 776                                     llargs[0],
 777                                     llargs[1],
 778                                     llresult,
 779                                     call_debug_location),
 780         (_, "u32_mul_with_overflow") =>
 781             with_overflow_intrinsic(bcx,
 782                                     "llvm.umul.with.overflow.i32",
 783                                     llargs[0],
 784                                     llargs[1],
 785                                     llresult,
 786                                     call_debug_location),
 787         (_, "u64_mul_with_overflow") =>
 788             with_overflow_intrinsic(bcx,
 789                                     "llvm.umul.with.overflow.i64",
 790                                     llargs[0],
 791                                     llargs[1],
 792                                     llresult,
 793                                     call_debug_location),
 794
 795         (_, "unchecked_udiv") => UDiv(bcx, llargs[0], llargs[1], call_debug_location),
 796         (_, "unchecked_sdiv") => SDiv(bcx, llargs[0], llargs[1], call_debug_location),
 797         (_, "unchecked_urem") => URem(bcx, llargs[0], llargs[1], call_debug_location),
 798         (_, "unchecked_srem") => SRem(bcx, llargs[0], llargs[1], call_debug_location),
 799
 800         (_, "overflowing_add") => Add(bcx, llargs[0], llargs[1], call_debug_location),
 801         (_, "overflowing_sub") => Sub(bcx, llargs[0], llargs[1], call_debug_location),
 802         (_, "overflowing_mul") => Mul(bcx, llargs[0], llargs[1], call_debug_location),
 803
 804         (_, "return_address") => {
 805             if !fcx.caller_expects_out_pointer {
 806                 span_err!(tcx.sess, call_info.span, E0510,
 807                           "invalid use of `return_address` intrinsic: function \
 808                            does not use out pointer");
 809                 C_null(Type::i8p(ccx))
 810             } else {
 811                 PointerCast(bcx, llvm::get_param(fcx.llfn, 0), Type::i8p(ccx))
 812             }
 813         }
 814
 815         (_, "discriminant_value") => {
 816             let val_ty = substs.types.get(FnSpace, 0);
 817             match val_ty.sty {
 818                 ty::TyEnum(..) => {
 819                     let repr = adt::represent_type(ccx, *val_ty);
 820                     adt::trans_get_discr(bcx, &*repr, llargs[0], Some(llret_ty))
 821                 }
 822                 _ => C_null(llret_ty)
 823             }
 824         }
 825         (_, name) if name.starts_with("simd_") => {
 826             generic_simd_intrinsic(bcx, name,
 827                                    substs,
 828                                    callee_ty,
 829                                    expr_arguments,
 830                                    &llargs,
 831                                    ret_ty, llret_ty,
 832                                    call_debug_location,
 833                                    call_info)
 834         }
 835         // This requires that atomic intrinsics follow a specific naming pattern:
 836         // "atomic_<operation>[_<ordering>]", and no ordering means SeqCst
 837         (_, name) if name.starts_with("atomic_") => {
 838             let split: Vec<&str> = name.split('_').collect();
 839             assert!(split.len() >= 2, "Atomic intrinsic not correct format");
 840
 841             let order = if split.len() == 2 {
 842                 llvm::SequentiallyConsistent
 843             } else {
 844                 match split[2] {
 845                     "unordered" => llvm::Unordered,
 846                     "relaxed" => llvm::Monotonic,
 847                     "acq"     => llvm::Acquire,
 848                     "rel"     => llvm::Release,
 849                     "acqrel"  => llvm::AcquireRelease,
 850                     _ => ccx.sess().fatal("unknown ordering in atomic intrinsic")
 851                 }
 852             };
 853
 854             match split[1] {
 855                 "cxchg" => {
 856                     // See include/llvm/IR/Instructions.h for their implementation
 857                     // of this, I assume that it's good enough for us to use for
 858                     // now.
 859                     let strongest_failure_ordering = match order {
 860                         llvm::NotAtomic | llvm::Unordered =>
 861                             ccx.sess().fatal("cmpxchg must be atomic"),
 862
 863                         llvm::Monotonic | llvm::Release =>
 864                             llvm::Monotonic,
 865
 866                         llvm::Acquire | llvm::AcquireRelease =>
 867                             llvm::Acquire,
 868
 869                         llvm::SequentiallyConsistent =>
 870                             llvm::SequentiallyConsistent
 871                     };
 872
 873                     let tp_ty = *substs.types.get(FnSpace, 0);
 874                     let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 875                     let cmp = from_arg_ty(bcx, llargs[1], tp_ty);
 876                     let src = from_arg_ty(bcx, llargs[2], tp_ty);
 877                     let res = AtomicCmpXchg(bcx, ptr, cmp, src, order,
 878                                             strongest_failure_ordering);
 879                     ExtractValue(bcx, res, 0)
 880                 }
 881
 882                 "load" => {
 883                     let tp_ty = *substs.types.get(FnSpace, 0);
 884                     let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 885                     to_arg_ty(bcx, AtomicLoad(bcx, ptr, order), tp_ty)
 886                 }
 887                 "store" => {
 888                     let tp_ty = *substs.types.get(FnSpace, 0);
 889                     let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 890                     let val = from_arg_ty(bcx, llargs[1], tp_ty);
 891                     AtomicStore(bcx, val, ptr, order);
 892                     C_nil(ccx)
 893                 }
 894
 895                 "fence" => {
 896                     AtomicFence(bcx, order, llvm::CrossThread);
 897                     C_nil(ccx)
 898                 }
 899
 900                 "singlethreadfence" => {
 901                     AtomicFence(bcx, order, llvm::SingleThread);
 902                     C_nil(ccx)
 903                 }
 904
 905                 // These are all AtomicRMW ops
 906                 op => {
 907                     let atom_op = match op {
 908                         "xchg"  => llvm::AtomicXchg,
 909                         "xadd"  => llvm::AtomicAdd,
 910                         "xsub"  => llvm::AtomicSub,
 911                         "and"   => llvm::AtomicAnd,
 912                         "nand"  => llvm::AtomicNand,
 913                         "or"    => llvm::AtomicOr,
 914                         "xor"   => llvm::AtomicXor,
 915                         "max"   => llvm::AtomicMax,
 916                         "min"   => llvm::AtomicMin,
 917                         "umax"  => llvm::AtomicUMax,
 918                         "umin"  => llvm::AtomicUMin,
 919                         _ => ccx.sess().fatal("unknown atomic operation")
 920                     };
 921
 922                     let tp_ty = *substs.types.get(FnSpace, 0);
 923                     let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 924                     let val = from_arg_ty(bcx, llargs[1], tp_ty);
 925                     AtomicRMW(bcx, atom_op, ptr, val, order)
 926                 }
 927             }
 928
 929         }
 930
 931         (_, _) => {
 932             let intr = match Intrinsic::find(tcx, &name) {
 933                 Some(intr) => intr,
 934                 None => ccx.sess().span_bug(foreign_item.span,
 935                                             &format!("unknown intrinsic '{}'", name)),
 936             };
 937             fn one<T>(x: Vec<T>) -> T {
 938                 assert_eq!(x.len(), 1);
 939                 x.into_iter().next().unwrap()
 940             }
 941             fn ty_to_type(ccx: &CrateContext, t: &intrinsics::Type,
 942                           any_changes_needed: &mut bool) -> Vec<Type> {
 943                 use intrinsics::Type::*;
 944                 match *t {
 945                     Void => vec![Type::void(ccx)],
 946                     Integer(_signed, width, llvm_width) => {
 947                         *any_changes_needed |= width != llvm_width;
 948                         vec![Type::ix(ccx, llvm_width as u64)]
 949                     }
 950                     Float(x) => {
 951                         match x {
 952                             32 => vec![Type::f32(ccx)],
 953                             64 => vec![Type::f64(ccx)],
 954                             _ => unreachable!()
 955                         }
 956                     }
 957                     Pointer(ref t, ref llvm_elem, _const) => {
 958                         *any_changes_needed |= llvm_elem.is_some();
 959
 960                         let t = llvm_elem.as_ref().unwrap_or(t);
 961                         let elem = one(ty_to_type(ccx, t,
 962                                                   any_changes_needed));
 963                         vec![elem.ptr_to()]
 964                     }
 965                     Vector(ref t, ref llvm_elem, length) => {
 966                         *any_changes_needed |= llvm_elem.is_some();
 967
 968                         let t = llvm_elem.as_ref().unwrap_or(t);
 969                         let elem = one(ty_to_type(ccx, t,
 970                                                   any_changes_needed));
 971                         vec![Type::vector(&elem,
 972                                           length as u64)]
 973                     }
 974                     Aggregate(false, ref contents) => {
 975                         let elems = contents.iter()
 976                                             .map(|t| one(ty_to_type(ccx, t, any_changes_needed)))
 977                                             .collect::<Vec<_>>();
 978                         vec![Type::struct_(ccx, &elems, false)]
 979                     }
 980                     Aggregate(true, ref contents) => {
 981                         *any_changes_needed = true;
 982                         contents.iter()
 983                                 .flat_map(|t| ty_to_type(ccx, t, any_changes_needed))
 984                                 .collect()
 985                     }
 986                 }
 987             }
 988
 989             // This allows an argument list like `foo, (bar, baz),
 990             // qux` to be converted into `foo, bar, baz, qux`, integer
 991             // arguments to be truncated as needed and pointers to be
 992             // cast.
 993             fn modify_as_needed<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
 994                                             t: &intrinsics::Type,
 995                                             arg_type: Ty<'tcx>,
 996                                             llarg: ValueRef)
 997                                             -> Vec<ValueRef>
 998             {
 999                 match *t {
1000                     intrinsics::Type::Aggregate(true, ref contents) => {
1001                         // We found a tuple that needs squishing! So
1002                         // run over the tuple and load each field.
1003                         //
1004                         // This assumes the type is "simple", i.e. no
1005                         // destructors, and the contents are SIMD
1006                         // etc.
1007                         assert!(!bcx.fcx.type_needs_drop(arg_type));
1008
1009                         let repr = adt::represent_type(bcx.ccx(), arg_type);
1010                         let repr_ptr = &*repr;
1011                         (0..contents.len())
1012                             .map(|i| {
1013                                 Load(bcx, adt::trans_field_ptr(bcx, repr_ptr, llarg, 0, i))
1014                             })
1015                             .collect()
1016                     }
1017                     intrinsics::Type::Pointer(_, Some(ref llvm_elem), _) => {
1018                         let llvm_elem = one(ty_to_type(bcx.ccx(), llvm_elem, &mut false));
1019                         vec![PointerCast(bcx, llarg,
1020                                          llvm_elem.ptr_to())]
1021                     }
1022                     intrinsics::Type::Vector(_, Some(ref llvm_elem), length) => {
1023                         let llvm_elem = one(ty_to_type(bcx.ccx(), llvm_elem, &mut false));
1024                         vec![BitCast(bcx, llarg,
1025                                      Type::vector(&llvm_elem, length as u64))]
1026                     }
1027                     intrinsics::Type::Integer(_, width, llvm_width) if width != llvm_width => {
1028                         // the LLVM intrinsic uses a smaller integer
1029                         // size than the C intrinsic's signature, so
1030                         // we have to trim it down here.
1031                         vec![Trunc(bcx, llarg, Type::ix(bcx.ccx(), llvm_width as u64))]
1032                     }
1033                     _ => vec![llarg],
1034                 }
1035             }
1036
1037
1038             let mut any_changes_needed = false;
1039             let inputs = intr.inputs.iter()
1040                                     .flat_map(|t| ty_to_type(ccx, t, &mut any_changes_needed))
1041                                     .collect::<Vec<_>>();
1042
1043             let mut out_changes = false;
1044             let outputs = one(ty_to_type(ccx, &intr.output, &mut out_changes));
1045             // outputting a flattened aggregate is nonsense
1046             assert!(!out_changes);
1047
1048             let llargs = if !any_changes_needed {
1049                 // no aggregates to flatten, so no change needed
1050                 llargs
1051             } else {
1052                 // there are some aggregates that need to be flattened
1053                 // in the LLVM call, so we need to run over the types
1054                 // again to find them and extract the arguments
1055                 intr.inputs.iter()
1056                            .zip(&llargs)
1057                            .zip(&arg_tys)
1058                            .flat_map(|((t, llarg), ty)| modify_as_needed(bcx, t, ty, *llarg))
1059                            .collect()
1060             };
1061             assert_eq!(inputs.len(), llargs.len());
1062
1063             let val = match intr.definition {
1064                 intrinsics::IntrinsicDef::Named(name) => {
1065                     let f = declare::declare_cfn(ccx,
1066                                                  name,
1067                                                  Type::func(&inputs, &outputs),
1068                                                  tcx.mk_nil());
1069                     Call(bcx, f, &llargs, None, call_debug_location)
1070                 }
1071             };
1072
1073             match intr.output {
1074                 intrinsics::Type::Aggregate(flatten, ref elems) => {
1075                     // the output is a tuple so we need to munge it properly
1076                     assert!(!flatten);
1077
1078                     for i in 0..elems.len() {
1079                         let val = ExtractValue(bcx, val, i);
1080                         Store(bcx, val, StructGEP(bcx, llresult, i));
1081                     }
1082                     C_nil(ccx)
1083                 }
1084                 _ => val,
1085             }
1086         }
1087     };
1088
1089     if val_ty(llval) != Type::void(ccx) &&
1090        machine::llsize_of_alloc(ccx, val_ty(llval)) != 0 {
1091         store_ty(bcx, llval, llresult, ret_ty);
1092     }
1093
1094     // If we made a temporary stack slot, let's clean it up
1095     match dest {
1096         expr::Ignore => {
1097             bcx = glue::drop_ty(bcx, llresult, ret_ty, call_debug_location);
1098             call_lifetime_end(bcx, llresult);
1099         }
1100         expr::SaveIn(_) => {}
1101     }
1102
1103     fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
1104
1105     Result::new(bcx, llresult)
1106 }
1107
1108 fn copy_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1109                               allow_overlap: bool,
1110                               volatile: bool,
1111                               tp_ty: Ty<'tcx>,
1112                               dst: ValueRef,
1113                               src: ValueRef,
1114                               count: ValueRef,
1115                               call_debug_location: DebugLoc)
1116                               -> ValueRef {
1117     let ccx = bcx.ccx();
1118     let lltp_ty = type_of::type_of(ccx, tp_ty);
1119     let align = C_i32(ccx, type_of::align_of(ccx, tp_ty) as i32);
1120     let size = machine::llsize_of(ccx, lltp_ty);
1121     let int_size = machine::llbitsize_of_real(ccx, ccx.int_type());
1122
1123     let operation = if allow_overlap {
1124         "memmove"
1125     } else {
1126         "memcpy"
1127     };
1128
1129     let name = format!("llvm.{}.p0i8.p0i8.i{}", operation, int_size);
1130
1131     let dst_ptr = PointerCast(bcx, dst, Type::i8p(ccx));
1132     let src_ptr = PointerCast(bcx, src, Type::i8p(ccx));
1133     let llfn = ccx.get_intrinsic(&name);
1134
1135     Call(bcx,
1136          llfn,
1137          &[dst_ptr,
1138            src_ptr,
1139            Mul(bcx, size, count, DebugLoc::None),
1140            align,
1141            C_bool(ccx, volatile)],
1142          None,
1143          call_debug_location)
1144 }
1145
1146 fn memset_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1147                                 volatile: bool,
1148                                 tp_ty: Ty<'tcx>,
1149                                 dst: ValueRef,
1150                                 val: ValueRef,
1151                                 count: ValueRef,
1152                                 call_debug_location: DebugLoc)
1153                                 -> ValueRef {
1154     let ccx = bcx.ccx();
1155     let lltp_ty = type_of::type_of(ccx, tp_ty);
1156     let align = C_i32(ccx, type_of::align_of(ccx, tp_ty) as i32);
1157     let size = machine::llsize_of(ccx, lltp_ty);
1158     let int_size = machine::llbitsize_of_real(ccx, ccx.int_type());
1159
1160     let name = format!("llvm.memset.p0i8.i{}", int_size);
1161
1162     let dst_ptr = PointerCast(bcx, dst, Type::i8p(ccx));
1163     let llfn = ccx.get_intrinsic(&name);
1164
1165     Call(bcx,
1166          llfn,
1167          &[dst_ptr,
1168            val,
1169            Mul(bcx, size, count, DebugLoc::None),
1170            align,
1171            C_bool(ccx, volatile)],
1172          None,
1173          call_debug_location)
1174 }
1175
1176 fn count_zeros_intrinsic(bcx: Block,
1177                          name: &'static str,
1178                          val: ValueRef,
1179                          call_debug_location: DebugLoc)
1180                          -> ValueRef {
1181     let y = C_bool(bcx.ccx(), false);
1182     let llfn = bcx.ccx().get_intrinsic(&name);
1183     Call(bcx, llfn, &[val, y], None, call_debug_location)
1184 }
1185
1186 fn with_overflow_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1187                                        name: &'static str,
1188                                        a: ValueRef,
1189                                        b: ValueRef,
1190                                        out: ValueRef,
1191                                        call_debug_location: DebugLoc)
1192                                        -> ValueRef {
1193     let llfn = bcx.ccx().get_intrinsic(&name);
1194
1195     // Convert `i1` to a `bool`, and write it to the out parameter
1196     let val = Call(bcx, llfn, &[a, b], None, call_debug_location);
1197     let result = ExtractValue(bcx, val, 0);
1198     let overflow = ZExt(bcx, ExtractValue(bcx, val, 1), Type::bool(bcx.ccx()));
1199     Store(bcx, result, StructGEP(bcx, out, 0));
1200     Store(bcx, overflow, StructGEP(bcx, out, 1));
1201
1202     C_nil(bcx.ccx())
1203 }
1204
1205 fn try_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1206                              func: ValueRef,
1207                              data: ValueRef,
1208                              dest: ValueRef,
1209                              dloc: DebugLoc) -> Block<'blk, 'tcx> {
1210     if bcx.sess().no_landing_pads() {
1211         Call(bcx, func, &[data], None, dloc);
1212         Store(bcx, C_null(Type::i8p(bcx.ccx())), dest);
1213         bcx
1214     } else if wants_msvc_seh(bcx.sess()) {
1215         trans_msvc_try(bcx, func, data, dest, dloc)
1216     } else {
1217         trans_gnu_try(bcx, func, data, dest, dloc)
1218     }
1219 }
1220
1221 // MSVC's definition of the `rust_try` function. The exact implementation here
1222 // is a little different than the GNU (standard) version below, not only because
1223 // of the personality function but also because of the other fiddly bits about
1224 // SEH. LLVM also currently requires us to structure this in a very particular
1225 // way as explained below.
1226 //
1227 // Like with the GNU version we generate a shim wrapper
1228 fn trans_msvc_try<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1229                               func: ValueRef,
1230                               data: ValueRef,
1231                               dest: ValueRef,
1232                               dloc: DebugLoc) -> Block<'blk, 'tcx> {
1233     let llfn = get_rust_try_fn(bcx.fcx, &mut |try_fn_ty, output| {
1234         let ccx = bcx.ccx();
1235         let dloc = DebugLoc::None;
1236         let rust_try = declare::define_internal_rust_fn(ccx, "__rust_try",
1237                                                          try_fn_ty);
1238         let (fcx, block_arena);
1239         block_arena = TypedArena::new();
1240         fcx = new_fn_ctxt(ccx, rust_try, ast::DUMMY_NODE_ID, false,
1241                           output, ccx.tcx().mk_substs(Substs::trans_empty()),
1242                           None, &block_arena);
1243         let bcx = init_function(&fcx, true, output);
1244         let then = fcx.new_temp_block("then");
1245         let catch = fcx.new_temp_block("catch");
1246         let catch_return = fcx.new_temp_block("catch-return");
1247         let catch_resume = fcx.new_temp_block("catch-resume");
1248         let personality = fcx.eh_personality();
1249
1250         let eh_typeid_for = ccx.get_intrinsic(&"llvm.eh.typeid.for");
1251         let rust_try_filter = match bcx.tcx().lang_items.msvc_try_filter() {
1252             Some(did) => callee::trans_fn_ref(ccx, did, ExprId(0),
1253                                               bcx.fcx.param_substs).val,
1254             None => bcx.sess().bug("msvc_try_filter not defined"),
1255         };
1256
1257         // Type indicator for the exception being thrown, not entirely sure
1258         // what's going on here but it's what all the examples in LLVM use.
1259         let lpad_ty = Type::struct_(ccx, &[Type::i8p(ccx), Type::i32(ccx)],
1260                                     false);
1261
1262         llvm::SetFunctionAttribute(rust_try, llvm::Attribute::NoInline);
1263         llvm::SetFunctionAttribute(rust_try, llvm::Attribute::OptimizeNone);
1264         let func = llvm::get_param(rust_try, 0);
1265         let data = llvm::get_param(rust_try, 1);
1266
1267         // Invoke the function, specifying our two temporary landing pads as the
1268         // ext point. After the invoke we've terminated our basic block.
1269         Invoke(bcx, func, &[data], then.llbb, catch.llbb, None, dloc);
1270
1271         // All the magic happens in this landing pad, and this is basically the
1272         // only landing pad in rust tagged with "catch" to indicate that we're
1273         // catching an exception. The other catch handlers in the GNU version
1274         // below just catch *all* exceptions, but that's because most exceptions
1275         // are already filtered out by the gnu personality function.
1276         //
1277         // For MSVC we're just using a standard personality function that we
1278         // can't customize (e.g. _except_handler3 or __C_specific_handler), so
1279         // we need to do the exception filtering ourselves. This is currently
1280         // performed by the `__rust_try_filter` function. This function,
1281         // specified in the landingpad instruction, will be invoked by Windows
1282         // SEH routines and will return whether the exception in question can be
1283         // caught (aka the Rust runtime is the one that threw the exception).
1284         //
1285         // To get this to compile (currently LLVM segfaults if it's not in this
1286         // particular structure), when the landingpad is executing we test to
1287         // make sure that the ID of the exception being thrown is indeed the one
1288         // that we were expecting. If it's not, we resume the exception, and
1289         // otherwise we return the pointer that we got Full disclosure: It's not
1290         // clear to me what this `llvm.eh.typeid` stuff is doing *other* then
1291         // just allowing LLVM to compile this file without segfaulting. I would
1292         // expect the entire landing pad to just be:
1293         //
1294         //     %vals = landingpad ...
1295         //     %ehptr = extractvalue { i8*, i32 } %vals, 0
1296         //     ret i8* %ehptr
1297         //
1298         // but apparently LLVM chokes on this, so we do the more complicated
1299         // thing to placate it.
1300         let vals = LandingPad(catch, lpad_ty, personality, 1);
1301         let rust_try_filter = BitCast(catch, rust_try_filter, Type::i8p(ccx));
1302         AddClause(catch, vals, rust_try_filter);
1303         let ehptr = ExtractValue(catch, vals, 0);
1304         let sel = ExtractValue(catch, vals, 1);
1305         let filter_sel = Call(catch, eh_typeid_for, &[rust_try_filter], None,
1306                               dloc);
1307         let is_filter = ICmp(catch, llvm::IntEQ, sel, filter_sel, dloc);
1308         CondBr(catch, is_filter, catch_return.llbb, catch_resume.llbb, dloc);
1309
1310         // Our "catch-return" basic block is where we've determined that we
1311         // actually need to catch this exception, in which case we just return
1312         // the exception pointer.
1313         Ret(catch_return, ehptr, dloc);
1314
1315         // The "catch-resume" block is where we're running this landing pad but
1316         // we actually need to not catch the exception, so just resume the
1317         // exception to return.
1318         Resume(catch_resume, vals);
1319
1320         // On the successful branch we just return null.
1321         Ret(then, C_null(Type::i8p(ccx)), dloc);
1322
1323         return rust_try
1324     });
1325
1326     // Note that no invoke is used here because by definition this function
1327     // can't panic (that's what it's catching).
1328     let ret = Call(bcx, llfn, &[func, data], None, dloc);
1329     Store(bcx, ret, dest);
1330     return bcx;
1331 }
1332
1333 // Definition of the standard "try" function for Rust using the GNU-like model
1334 // of exceptions (e.g. the normal semantics of LLVM's landingpad and invoke
1335 // instructions).
1336 //
1337 // This translation is a little surprising because
1338 // we always call a shim function instead of inlining the call to `invoke`
1339 // manually here. This is done because in LLVM we're only allowed to have one
1340 // personality per function definition. The call to the `try` intrinsic is
1341 // being inlined into the function calling it, and that function may already
1342 // have other personality functions in play. By calling a shim we're
1343 // guaranteed that our shim will have the right personality function.
1344 //
1345 fn trans_gnu_try<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1346                              func: ValueRef,
1347                              data: ValueRef,
1348                              dest: ValueRef,
1349                              dloc: DebugLoc) -> Block<'blk, 'tcx> {
1350     let llfn = get_rust_try_fn(bcx.fcx, &mut |try_fn_ty, output| {
1351         let ccx = bcx.ccx();
1352         let dloc = DebugLoc::None;
1353
1354         // Translates the shims described above:
1355         //
1356         //   bcx:
1357         //      invoke %func(%args...) normal %normal unwind %catch
1358         //
1359         //   normal:
1360         //      ret null
1361         //
1362         //   catch:
1363         //      (ptr, _) = landingpad
1364         //      ret ptr
1365
1366         let rust_try = declare::define_internal_rust_fn(ccx, "__rust_try", try_fn_ty);
1367         attributes::emit_uwtable(rust_try, true);
1368         let catch_pers = match bcx.tcx().lang_items.eh_personality_catch() {
1369             Some(did) => callee::trans_fn_ref(ccx, did, ExprId(0),
1370                                               bcx.fcx.param_substs).val,
1371             None => bcx.tcx().sess.bug("eh_personality_catch not defined"),
1372         };
1373
1374         let (fcx, block_arena);
1375         block_arena = TypedArena::new();
1376         fcx = new_fn_ctxt(ccx, rust_try, ast::DUMMY_NODE_ID, false,
1377                           output, ccx.tcx().mk_substs(Substs::trans_empty()),
1378                           None, &block_arena);
1379         let bcx = init_function(&fcx, true, output);
1380         let then = bcx.fcx.new_temp_block("then");
1381         let catch = bcx.fcx.new_temp_block("catch");
1382
1383         let func = llvm::get_param(rust_try, 0);
1384         let data = llvm::get_param(rust_try, 1);
1385         Invoke(bcx, func, &[data], then.llbb, catch.llbb, None, dloc);
1386         Ret(then, C_null(Type::i8p(ccx)), dloc);
1387
1388         // Type indicator for the exception being thrown.
1389         // The first value in this tuple is a pointer to the exception object being thrown.
1390         // The second value is a "selector" indicating which of the landing pad clauses
1391         // the exception's type had been matched to.  rust_try ignores the selector.
1392         let lpad_ty = Type::struct_(ccx, &[Type::i8p(ccx), Type::i32(ccx)],
1393                                     false);
1394         let vals = LandingPad(catch, lpad_ty, catch_pers, 1);
1395         AddClause(catch, vals, C_null(Type::i8p(ccx)));
1396         let ptr = ExtractValue(catch, vals, 0);
1397         Ret(catch, ptr, dloc);
1398         fcx.cleanup();
1399
1400         return rust_try
1401     });
1402
1403     // Note that no invoke is used here because by definition this function
1404     // can't panic (that's what it's catching).
1405     let ret = Call(bcx, llfn, &[func, data], None, dloc);
1406     Store(bcx, ret, dest);
1407     return bcx;
1408 }
1409
1410 // Helper to generate the `Ty` associated with `rust_try`
1411 fn get_rust_try_fn<'a, 'tcx>(fcx: &FunctionContext<'a, 'tcx>,
1412                              f: &mut FnMut(Ty<'tcx>,
1413                                            ty::FnOutput<'tcx>) -> ValueRef)
1414                              -> ValueRef {
1415     let ccx = fcx.ccx;
1416     if let Some(llfn) = *ccx.rust_try_fn().borrow() {
1417         return llfn
1418     }
1419
1420     // Define the type up front for the signature of the rust_try function.
1421     let tcx = ccx.tcx();
1422     let i8p = tcx.mk_mut_ptr(tcx.types.i8);
1423     let fn_ty = tcx.mk_bare_fn(ty::BareFnTy {
1424         unsafety: hir::Unsafety::Unsafe,
1425         abi: abi::Rust,
1426         sig: ty::Binder(ty::FnSig {
1427             inputs: vec![i8p],
1428             output: ty::FnOutput::FnConverging(tcx.mk_nil()),
1429             variadic: false,
1430         }),
1431     });
1432     let fn_ty = tcx.mk_fn(None, fn_ty);
1433     let output = ty::FnOutput::FnConverging(i8p);
1434     let try_fn_ty  = tcx.mk_bare_fn(ty::BareFnTy {
1435         unsafety: hir::Unsafety::Unsafe,
1436         abi: abi::Rust,
1437         sig: ty::Binder(ty::FnSig {
1438             inputs: vec![fn_ty, i8p],
1439             output: output,
1440             variadic: false,
1441         }),
1442     });
1443     let rust_try = f(tcx.mk_fn(None, try_fn_ty), output);
1444     *ccx.rust_try_fn().borrow_mut() = Some(rust_try);
1445     return rust_try
1446 }
1447
1448 fn span_invalid_monomorphization_error(a: &Session, b: Span, c: &str) {
1449     span_err!(a, b, E0511, "{}", c);
1450 }
1451
1452 fn generic_simd_intrinsic<'blk, 'tcx, 'a>
1453     (bcx: Block<'blk, 'tcx>,
1454      name: &str,
1455      substs: subst::Substs<'tcx>,
1456      callee_ty: Ty<'tcx>,
1457      args: Option<&[P<hir::Expr>]>,
1458      llargs: &[ValueRef],
1459      ret_ty: Ty<'tcx>,
1460      llret_ty: Type,
1461      call_debug_location: DebugLoc,
1462      call_info: NodeIdAndSpan) -> ValueRef
1463 {
1464     // macros for error handling:
1465     macro_rules! emit_error {
1466         ($msg: tt) => {
1467             emit_error!($msg, )
1468         };
1469         ($msg: tt, $($fmt: tt)*) => {
1470             span_invalid_monomorphization_error(
1471                 bcx.sess(), call_info.span,
1472                 &format!(concat!("invalid monomorphization of `{}` intrinsic: ",
1473                                  $msg),
1474                          name, $($fmt)*));
1475         }
1476     }
1477     macro_rules! require {
1478         ($cond: expr, $($fmt: tt)*) => {
1479             if !$cond {
1480                 emit_error!($($fmt)*);
1481                 return C_null(llret_ty)
1482             }
1483         }
1484     }
1485     macro_rules! require_simd {
1486         ($ty: expr, $position: expr) => {
1487             require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty)
1488         }
1489     }
1490
1491
1492
1493     let tcx = bcx.tcx();
1494     let arg_tys = match callee_ty.sty {
1495         ty::TyBareFn(_, ref f) => {
1496             bcx.tcx().erase_late_bound_regions(&f.sig.inputs())
1497         }
1498         _ => unreachable!()
1499     };
1500
1501     // every intrinsic takes a SIMD vector as its first argument
1502     require_simd!(arg_tys[0], "input");
1503     let in_ty = arg_tys[0];
1504     let in_elem = arg_tys[0].simd_type(tcx);
1505     let in_len = arg_tys[0].simd_size(tcx);
1506
1507     let comparison = match name {
1508         "simd_eq" => Some(hir::BiEq),
1509         "simd_ne" => Some(hir::BiNe),
1510         "simd_lt" => Some(hir::BiLt),
1511         "simd_le" => Some(hir::BiLe),
1512         "simd_gt" => Some(hir::BiGt),
1513         "simd_ge" => Some(hir::BiGe),
1514         _ => None
1515     };
1516
1517     if let Some(cmp_op) = comparison {
1518         require_simd!(ret_ty, "return");
1519
1520         let out_len = ret_ty.simd_size(tcx);
1521         require!(in_len == out_len,
1522                  "expected return type with length {} (same as input type `{}`), \
1523                   found `{}` with length {}",
1524                  in_len, in_ty,
1525                  ret_ty, out_len);
1526         require!(llret_ty.element_type().kind() == llvm::Integer,
1527                  "expected return type with integer elements, found `{}` with non-integer `{}`",
1528                  ret_ty,
1529                  ret_ty.simd_type(tcx));
1530
1531         return compare_simd_types(bcx,
1532                                   llargs[0],
1533                                   llargs[1],
1534                                   in_elem,
1535                                   llret_ty,
1536                                   cmp_op,
1537                                   call_debug_location)
1538     }
1539
1540     if name.starts_with("simd_shuffle") {
1541         let n: usize = match name["simd_shuffle".len()..].parse() {
1542             Ok(n) => n,
1543             Err(_) => tcx.sess.span_bug(call_info.span,
1544                                         "bad `simd_shuffle` instruction only caught in trans?")
1545         };
1546
1547         require_simd!(ret_ty, "return");
1548
1549         let out_len = ret_ty.simd_size(tcx);
1550         require!(out_len == n,
1551                  "expected return type of length {}, found `{}` with length {}",
1552                  n, ret_ty, out_len);
1553         require!(in_elem == ret_ty.simd_type(tcx),
1554                  "expected return element type `{}` (element of input `{}`), \
1555                   found `{}` with element type `{}`",
1556                  in_elem, in_ty,
1557                  ret_ty, ret_ty.simd_type(tcx));
1558
1559         let total_len = in_len as u64 * 2;
1560
1561         let vector = match args {
1562             Some(args) => &args[2],
1563             None => bcx.sess().span_bug(call_info.span,
1564                                         "intrinsic call with unexpected argument shape"),
1565         };
1566         let vector = consts::const_expr(bcx.ccx(), vector, tcx.mk_substs(substs), None).0;
1567
1568         let indices: Option<Vec<_>> = (0..n)
1569             .map(|i| {
1570                 let arg_idx = i;
1571                 let val = const_get_elt(bcx.ccx(), vector, &[i as libc::c_uint]);
1572                 let c = const_to_opt_uint(val);
1573                 match c {
1574                     None => {
1575                         emit_error!("shuffle index #{} is not a constant", arg_idx);
1576                         None
1577                     }
1578                     Some(idx) if idx >= total_len => {
1579                         emit_error!("shuffle index #{} is out of bounds (limit {})",
1580                                     arg_idx, total_len);
1581                         None
1582                     }
1583                     Some(idx) => Some(C_i32(bcx.ccx(), idx as i32)),
1584                 }
1585             })
1586             .collect();
1587         let indices = match indices {
1588             Some(i) => i,
1589             None => return C_null(llret_ty)
1590         };
1591
1592         return ShuffleVector(bcx, llargs[0], llargs[1], C_vector(&indices))
1593     }
1594
1595     if name == "simd_insert" {
1596         require!(in_elem == arg_tys[2],
1597                  "expected inserted type `{}` (element of input `{}`), found `{}`",
1598                  in_elem, in_ty, arg_tys[2]);
1599         return InsertElement(bcx, llargs[0], llargs[2], llargs[1])
1600     }
1601     if name == "simd_extract" {
1602         require!(ret_ty == in_elem,
1603                  "expected return type `{}` (element of input `{}`), found `{}`",
1604                  in_elem, in_ty, ret_ty);
1605         return ExtractElement(bcx, llargs[0], llargs[1])
1606     }
1607
1608     if name == "simd_cast" {
1609         require_simd!(ret_ty, "return");
1610         let out_len = ret_ty.simd_size(tcx);
1611         require!(in_len == out_len,
1612                  "expected return type with length {} (same as input type `{}`), \
1613                   found `{}` with length {}",
1614                  in_len, in_ty,
1615                  ret_ty, out_len);
1616         // casting cares about nominal type, not just structural type
1617         let out_elem = ret_ty.simd_type(tcx);
1618
1619         if in_elem == out_elem { return llargs[0]; }
1620
1621         enum Style { Float, Int(/* is signed? */ bool), Unsupported }
1622
1623         let (in_style, in_width) = match in_elem.sty {
1624             // vectors of pointer-sized integers should've been
1625             // disallowed before here, so this unwrap is safe.
1626             ty::TyInt(i) => (Style::Int(true), i.bit_width().unwrap()),
1627             ty::TyUint(u) => (Style::Int(false), u.bit_width().unwrap()),
1628             ty::TyFloat(f) => (Style::Float, f.bit_width()),
1629             _ => (Style::Unsupported, 0)
1630         };
1631         let (out_style, out_width) = match out_elem.sty {
1632             ty::TyInt(i) => (Style::Int(true), i.bit_width().unwrap()),
1633             ty::TyUint(u) => (Style::Int(false), u.bit_width().unwrap()),
1634             ty::TyFloat(f) => (Style::Float, f.bit_width()),
1635             _ => (Style::Unsupported, 0)
1636         };
1637
1638         match (in_style, out_style) {
1639             (Style::Int(in_is_signed), Style::Int(_)) => {
1640                 return match in_width.cmp(&out_width) {
1641                     Ordering::Greater => Trunc(bcx, llargs[0], llret_ty),
1642                     Ordering::Equal => llargs[0],
1643                     Ordering::Less => if in_is_signed {
1644                         SExt(bcx, llargs[0], llret_ty)
1645                     } else {
1646                         ZExt(bcx, llargs[0], llret_ty)
1647                     }
1648                 }
1649             }
1650             (Style::Int(in_is_signed), Style::Float) => {
1651                 return if in_is_signed {
1652                     SIToFP(bcx, llargs[0], llret_ty)
1653                 } else {
1654                     UIToFP(bcx, llargs[0], llret_ty)
1655                 }
1656             }
1657             (Style::Float, Style::Int(out_is_signed)) => {
1658                 return if out_is_signed {
1659                     FPToSI(bcx, llargs[0], llret_ty)
1660                 } else {
1661                     FPToUI(bcx, llargs[0], llret_ty)
1662                 }
1663             }
1664             (Style::Float, Style::Float) => {
1665                 return match in_width.cmp(&out_width) {
1666                     Ordering::Greater => FPTrunc(bcx, llargs[0], llret_ty),
1667                     Ordering::Equal => llargs[0],
1668                     Ordering::Less => FPExt(bcx, llargs[0], llret_ty)
1669                 }
1670             }
1671             _ => {/* Unsupported. Fallthrough. */}
1672         }
1673         require!(false,
1674                  "unsupported cast from `{}` with element `{}` to `{}` with element `{}`",
1675                  in_ty, in_elem,
1676                  ret_ty, out_elem);
1677     }
1678     macro_rules! arith {
1679         ($($name: ident: $($($p: ident),* => $call: expr),*;)*) => {
1680             $(
1681                 if name == stringify!($name) {
1682                     match in_elem.sty {
1683                         $(
1684                             $(ty::$p(_))|* => {
1685                                 return $call(bcx, llargs[0], llargs[1], call_debug_location)
1686                             }
1687                             )*
1688                         _ => {},
1689                     }
1690                     require!(false,
1691                              "unsupported operation on `{}` with element `{}`",
1692                              in_ty,
1693                              in_elem)
1694                 })*
1695         }
1696     }
1697     arith! {
1698         simd_add: TyUint, TyInt => Add, TyFloat => FAdd;
1699         simd_sub: TyUint, TyInt => Sub, TyFloat => FSub;
1700         simd_mul: TyUint, TyInt => Mul, TyFloat => FMul;
1701         simd_div: TyFloat => FDiv;
1702         simd_shl: TyUint, TyInt => Shl;
1703         simd_shr: TyUint => LShr, TyInt => AShr;
1704         simd_and: TyUint, TyInt => And;
1705         simd_or: TyUint, TyInt => Or;
1706         simd_xor: TyUint, TyInt => Xor;
1707     }
1708     bcx.sess().span_bug(call_info.span, "unknown SIMD intrinsic");
1709 }