src/intrinsics/mod.rs

   1 //! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"`
   2 //! and LLVM intrinsics that have symbol names starting with `llvm.`.
   3
   4 macro_rules! intrinsic_pat {
   5     (_) => {
   6         _
   7     };
   8     ($name:ident) => {
   9         sym::$name
  10     };
  11     (kw.$name:ident) => {
  12         kw::$name
  13     };
  14     ($name:literal) => {
  15         $name
  16     };
  17 }
  18
  19 macro_rules! intrinsic_arg {
  20     (o $fx:expr, $arg:ident) => {};
  21     (c $fx:expr, $arg:ident) => {
  22         let $arg = codegen_operand($fx, $arg);
  23     };
  24     (v $fx:expr, $arg:ident) => {
  25         let $arg = codegen_operand($fx, $arg).load_scalar($fx);
  26     };
  27 }
  28
  29 macro_rules! intrinsic_match {
  30     ($fx:expr, $intrinsic:expr, $args:expr,
  31     _ => $unknown:block;
  32     $(
  33         $($($name:tt).*)|+ $(if $cond:expr)?, ($($a:ident $arg:ident),*) $content:block;
  34     )*) => {
  35         match $intrinsic {
  36             $(
  37                 $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
  38                     if let [$($arg),*] = $args {
  39                         $(intrinsic_arg!($a $fx, $arg);)*
  40                         $content
  41                     } else {
  42                         bug!("wrong number of args for intrinsic {:?}", $intrinsic);
  43                     }
  44                 }
  45             )*
  46             _ => $unknown,
  47         }
  48     }
  49 }
  50
  51 mod cpuid;
  52 mod llvm;
  53 mod simd;
  54
  55 pub(crate) use cpuid::codegen_cpuid_call;
  56 pub(crate) use llvm::codegen_llvm_intrinsic_call;
  57
  58 use rustc_middle::ty::print::with_no_trimmed_paths;
  59 use rustc_middle::ty::subst::SubstsRef;
  60 use rustc_span::symbol::{kw, sym, Symbol};
  61
  62 use crate::prelude::*;
  63 use cranelift_codegen::ir::AtomicRmwOp;
  64
  65 fn report_atomic_type_validation_error<'tcx>(
  66     fx: &mut FunctionCx<'_, '_, 'tcx>,
  67     intrinsic: Symbol,
  68     span: Span,
  69     ty: Ty<'tcx>,
  70 ) {
  71     fx.tcx.sess.span_err(
  72         span,
  73         &format!(
  74             "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`",
  75             intrinsic, ty
  76         ),
  77     );
  78     // Prevent verifier error
  79     crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
  80 }
  81
  82 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
  83     let (element, count) = match layout.abi {
  84         Abi::Vector { element, count } => (element, count),
  85         _ => unreachable!(),
  86     };
  87
  88     match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
  89         // Cranelift currently only implements icmp for 128bit vectors.
  90         Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
  91         _ => None,
  92     }
  93 }
  94
  95 fn simd_for_each_lane<'tcx>(
  96     fx: &mut FunctionCx<'_, '_, 'tcx>,
  97     val: CValue<'tcx>,
  98     ret: CPlace<'tcx>,
  99     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value) -> Value,
 100 ) {
 101     let layout = val.layout();
 102
 103     let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
 104     let lane_layout = fx.layout_of(lane_ty);
 105     let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
 106     let ret_lane_layout = fx.layout_of(ret_lane_ty);
 107     assert_eq!(lane_count, ret_lane_count);
 108
 109     for lane_idx in 0..lane_count {
 110         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
 111
 112         let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, lane);
 113         let res_lane = CValue::by_val(res_lane, ret_lane_layout);
 114
 115         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
 116     }
 117 }
 118
 119 fn simd_pair_for_each_lane<'tcx>(
 120     fx: &mut FunctionCx<'_, '_, 'tcx>,
 121     x: CValue<'tcx>,
 122     y: CValue<'tcx>,
 123     ret: CPlace<'tcx>,
 124     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value) -> Value,
 125 ) {
 126     assert_eq!(x.layout(), y.layout());
 127     let layout = x.layout();
 128
 129     let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
 130     let lane_layout = fx.layout_of(lane_ty);
 131     let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
 132     let ret_lane_layout = fx.layout_of(ret_lane_ty);
 133     assert_eq!(lane_count, ret_lane_count);
 134
 135     for lane_idx in 0..lane_count {
 136         let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx);
 137         let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx);
 138
 139         let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, x_lane, y_lane);
 140         let res_lane = CValue::by_val(res_lane, ret_lane_layout);
 141
 142         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
 143     }
 144 }
 145
 146 fn simd_reduce<'tcx>(
 147     fx: &mut FunctionCx<'_, '_, 'tcx>,
 148     val: CValue<'tcx>,
 149     acc: Option<Value>,
 150     ret: CPlace<'tcx>,
 151     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Value, Value) -> Value,
 152 ) {
 153     let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
 154     let lane_layout = fx.layout_of(lane_ty);
 155     assert_eq!(lane_layout, ret.layout());
 156
 157     let (mut res_val, start_lane) =
 158         if let Some(acc) = acc { (acc, 0) } else { (val.value_lane(fx, 0).load_scalar(fx), 1) };
 159     for lane_idx in start_lane..lane_count {
 160         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
 161         res_val = f(fx, lane_layout.ty, res_val, lane);
 162     }
 163     let res = CValue::by_val(res_val, lane_layout);
 164     ret.write_cvalue(fx, res);
 165 }
 166
 167 // FIXME move all uses to `simd_reduce`
 168 fn simd_reduce_bool<'tcx>(
 169     fx: &mut FunctionCx<'_, '_, 'tcx>,
 170     val: CValue<'tcx>,
 171     ret: CPlace<'tcx>,
 172     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
 173 ) {
 174     let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
 175     assert!(ret.layout().ty.is_bool());
 176
 177     let res_val = val.value_lane(fx, 0).load_scalar(fx);
 178     let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
 179     for lane_idx in 1..lane_count {
 180         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
 181         let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
 182         res_val = f(fx, res_val, lane);
 183     }
 184     let res_val = if fx.bcx.func.dfg.value_type(res_val) != types::I8 {
 185         fx.bcx.ins().ireduce(types::I8, res_val)
 186     } else {
 187         res_val
 188     };
 189     let res = CValue::by_val(res_val, ret.layout());
 190     ret.write_cvalue(fx, res);
 191 }
 192
 193 fn bool_to_zero_or_max_uint<'tcx>(
 194     fx: &mut FunctionCx<'_, '_, 'tcx>,
 195     ty: Ty<'tcx>,
 196     val: Value,
 197 ) -> Value {
 198     let ty = fx.clif_type(ty).unwrap();
 199
 200     let int_ty = match ty {
 201         types::F32 => types::I32,
 202         types::F64 => types::I64,
 203         ty => ty,
 204     };
 205
 206     let val = fx.bcx.ins().bint(int_ty, val);
 207     let mut res = fx.bcx.ins().ineg(val);
 208
 209     if ty.is_float() {
 210         res = fx.bcx.ins().bitcast(ty, res);
 211     }
 212
 213     res
 214 }
 215
 216 pub(crate) fn codegen_intrinsic_call<'tcx>(
 217     fx: &mut FunctionCx<'_, '_, 'tcx>,
 218     instance: Instance<'tcx>,
 219     args: &[mir::Operand<'tcx>],
 220     destination: CPlace<'tcx>,
 221     target: Option<BasicBlock>,
 222     source_info: mir::SourceInfo,
 223 ) {
 224     let intrinsic = fx.tcx.item_name(instance.def_id());
 225     let substs = instance.substs;
 226
 227     let target = if let Some(target) = target {
 228         target
 229     } else {
 230         // Insert non returning intrinsics here
 231         match intrinsic {
 232             sym::abort => {
 233                 fx.bcx.ins().trap(TrapCode::User(0));
 234             }
 235             sym::transmute => {
 236                 crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
 237             }
 238             _ => unimplemented!("unsupported instrinsic {}", intrinsic),
 239         }
 240         return;
 241     };
 242
 243     if intrinsic.as_str().starts_with("simd_") {
 244         self::simd::codegen_simd_intrinsic_call(
 245             fx,
 246             intrinsic,
 247             substs,
 248             args,
 249             destination,
 250             source_info.span,
 251         );
 252         let ret_block = fx.get_block(target);
 253         fx.bcx.ins().jump(ret_block, &[]);
 254     } else if codegen_float_intrinsic_call(fx, intrinsic, args, destination) {
 255         let ret_block = fx.get_block(target);
 256         fx.bcx.ins().jump(ret_block, &[]);
 257     } else {
 258         codegen_regular_intrinsic_call(
 259             fx,
 260             instance,
 261             intrinsic,
 262             substs,
 263             args,
 264             destination,
 265             Some(target),
 266             source_info,
 267         );
 268     }
 269 }
 270
 271 fn codegen_float_intrinsic_call<'tcx>(
 272     fx: &mut FunctionCx<'_, '_, 'tcx>,
 273     intrinsic: Symbol,
 274     args: &[mir::Operand<'tcx>],
 275     ret: CPlace<'tcx>,
 276 ) -> bool {
 277     let (name, arg_count, ty) = match intrinsic {
 278         sym::expf32 => ("expf", 1, fx.tcx.types.f32),
 279         sym::expf64 => ("exp", 1, fx.tcx.types.f64),
 280         sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32),
 281         sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64),
 282         sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32),
 283         sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64),
 284         sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32), // compiler-builtins
 285         sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64), // compiler-builtins
 286         sym::powf32 => ("powf", 2, fx.tcx.types.f32),
 287         sym::powf64 => ("pow", 2, fx.tcx.types.f64),
 288         sym::logf32 => ("logf", 1, fx.tcx.types.f32),
 289         sym::logf64 => ("log", 1, fx.tcx.types.f64),
 290         sym::log2f32 => ("log2f", 1, fx.tcx.types.f32),
 291         sym::log2f64 => ("log2", 1, fx.tcx.types.f64),
 292         sym::log10f32 => ("log10f", 1, fx.tcx.types.f32),
 293         sym::log10f64 => ("log10", 1, fx.tcx.types.f64),
 294         sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32),
 295         sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64),
 296         sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32),
 297         sym::fmaf64 => ("fma", 3, fx.tcx.types.f64),
 298         sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32),
 299         sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64),
 300         sym::floorf32 => ("floorf", 1, fx.tcx.types.f32),
 301         sym::floorf64 => ("floor", 1, fx.tcx.types.f64),
 302         sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32),
 303         sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64),
 304         sym::truncf32 => ("truncf", 1, fx.tcx.types.f32),
 305         sym::truncf64 => ("trunc", 1, fx.tcx.types.f64),
 306         sym::roundf32 => ("roundf", 1, fx.tcx.types.f32),
 307         sym::roundf64 => ("round", 1, fx.tcx.types.f64),
 308         sym::sinf32 => ("sinf", 1, fx.tcx.types.f32),
 309         sym::sinf64 => ("sin", 1, fx.tcx.types.f64),
 310         sym::cosf32 => ("cosf", 1, fx.tcx.types.f32),
 311         sym::cosf64 => ("cos", 1, fx.tcx.types.f64),
 312         _ => return false,
 313     };
 314
 315     if args.len() != arg_count {
 316         bug!("wrong number of args for intrinsic {:?}", intrinsic);
 317     }
 318
 319     let (a, b, c);
 320     let args = match args {
 321         [x] => {
 322             a = [codegen_operand(fx, x)];
 323             &a as &[_]
 324         }
 325         [x, y] => {
 326             b = [codegen_operand(fx, x), codegen_operand(fx, y)];
 327             &b
 328         }
 329         [x, y, z] => {
 330             c = [codegen_operand(fx, x), codegen_operand(fx, y), codegen_operand(fx, z)];
 331             &c
 332         }
 333         _ => unreachable!(),
 334     };
 335
 336     let res = fx.easy_call(name, &args, ty);
 337     ret.write_cvalue(fx, res);
 338
 339     true
 340 }
 341
 342 fn codegen_regular_intrinsic_call<'tcx>(
 343     fx: &mut FunctionCx<'_, '_, 'tcx>,
 344     instance: Instance<'tcx>,
 345     intrinsic: Symbol,
 346     substs: SubstsRef<'tcx>,
 347     args: &[mir::Operand<'tcx>],
 348     ret: CPlace<'tcx>,
 349     destination: Option<BasicBlock>,
 350     source_info: mir::SourceInfo,
 351 ) {
 352     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 353
 354     intrinsic_match! {
 355         fx, intrinsic, args,
 356         _ => {
 357             fx.tcx.sess.span_fatal(source_info.span, &format!("unsupported intrinsic {}", intrinsic));
 358         };
 359
 360         assume, (c _a) {};
 361         likely | unlikely, (c a) {
 362             ret.write_cvalue(fx, a);
 363         };
 364         breakpoint, () {
 365             fx.bcx.ins().debugtrap();
 366         };
 367         copy | copy_nonoverlapping, (v src, v dst, v count) {
 368             let elem_ty = substs.type_at(0);
 369             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 370             assert_eq!(args.len(), 3);
 371             let byte_amount = if elem_size != 1 {
 372                 fx.bcx.ins().imul_imm(count, elem_size as i64)
 373             } else {
 374                 count
 375             };
 376
 377             if intrinsic == sym::copy_nonoverlapping {
 378                 // FIXME emit_small_memcpy
 379                 fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
 380             } else {
 381                 // FIXME emit_small_memmove
 382                 fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
 383             }
 384         };
 385         // NOTE: the volatile variants have src and dst swapped
 386         volatile_copy_memory | volatile_copy_nonoverlapping_memory, (v dst, v src, v count) {
 387             let elem_ty = substs.type_at(0);
 388             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 389             assert_eq!(args.len(), 3);
 390             let byte_amount = if elem_size != 1 {
 391                 fx.bcx.ins().imul_imm(count, elem_size as i64)
 392             } else {
 393                 count
 394             };
 395
 396             // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
 397             if intrinsic == sym::volatile_copy_nonoverlapping_memory {
 398                 // FIXME emit_small_memcpy
 399                 fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
 400             } else {
 401                 // FIXME emit_small_memmove
 402                 fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
 403             }
 404         };
 405         size_of_val, (c ptr) {
 406             let layout = fx.layout_of(substs.type_at(0));
 407             // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
 408             // branch
 409             let size = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
 410                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 411                 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 412                 size
 413             } else {
 414                 fx
 415                     .bcx
 416                     .ins()
 417                     .iconst(fx.pointer_type, layout.size.bytes() as i64)
 418             };
 419             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 420         };
 421         min_align_of_val, (c ptr) {
 422             let layout = fx.layout_of(substs.type_at(0));
 423             // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
 424             // branch
 425             let align = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
 426                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 427                 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 428                 align
 429             } else {
 430                 fx
 431                     .bcx
 432                     .ins()
 433                     .iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
 434             };
 435             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 436         };
 437
 438         vtable_size, (v vtable) {
 439             let size = crate::vtable::size_of_obj(fx, vtable);
 440             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 441         };
 442
 443         vtable_align, (v vtable) {
 444             let align = crate::vtable::min_align_of_obj(fx, vtable);
 445             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 446         };
 447
 448         unchecked_add | unchecked_sub | unchecked_mul | unchecked_div | exact_div | unchecked_rem
 449         | unchecked_shl | unchecked_shr, (c x, c y) {
 450             // FIXME trap on overflow
 451             let bin_op = match intrinsic {
 452                 sym::unchecked_add => BinOp::Add,
 453                 sym::unchecked_sub => BinOp::Sub,
 454                 sym::unchecked_mul => BinOp::Mul,
 455                 sym::unchecked_div | sym::exact_div => BinOp::Div,
 456                 sym::unchecked_rem => BinOp::Rem,
 457                 sym::unchecked_shl => BinOp::Shl,
 458                 sym::unchecked_shr => BinOp::Shr,
 459                 _ => unreachable!(),
 460             };
 461             let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
 462             ret.write_cvalue(fx, res);
 463         };
 464         add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) {
 465             assert_eq!(x.layout().ty, y.layout().ty);
 466             let bin_op = match intrinsic {
 467                 sym::add_with_overflow => BinOp::Add,
 468                 sym::sub_with_overflow => BinOp::Sub,
 469                 sym::mul_with_overflow => BinOp::Mul,
 470                 _ => unreachable!(),
 471             };
 472
 473             let res = crate::num::codegen_checked_int_binop(
 474                 fx,
 475                 bin_op,
 476                 x,
 477                 y,
 478             );
 479             ret.write_cvalue(fx, res);
 480         };
 481         saturating_add | saturating_sub, (c lhs, c rhs) {
 482             assert_eq!(lhs.layout().ty, rhs.layout().ty);
 483             let bin_op = match intrinsic {
 484                 sym::saturating_add => BinOp::Add,
 485                 sym::saturating_sub => BinOp::Sub,
 486                 _ => unreachable!(),
 487             };
 488
 489             let signed = type_sign(lhs.layout().ty);
 490
 491             let checked_res = crate::num::codegen_checked_int_binop(
 492                 fx,
 493                 bin_op,
 494                 lhs,
 495                 rhs,
 496             );
 497
 498             let (val, has_overflow) = checked_res.load_scalar_pair(fx);
 499             let clif_ty = fx.clif_type(lhs.layout().ty).unwrap();
 500
 501             let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
 502
 503             let val = match (intrinsic, signed) {
 504                 (sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val),
 505                 (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val),
 506                 (sym::saturating_add, true) => {
 507                     let rhs = rhs.load_scalar(fx);
 508                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 509                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
 510                     fx.bcx.ins().select(has_overflow, sat_val, val)
 511                 }
 512                 (sym::saturating_sub, true) => {
 513                     let rhs = rhs.load_scalar(fx);
 514                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 515                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
 516                     fx.bcx.ins().select(has_overflow, sat_val, val)
 517                 }
 518                 _ => unreachable!(),
 519             };
 520
 521             let res = CValue::by_val(val, lhs.layout());
 522
 523             ret.write_cvalue(fx, res);
 524         };
 525         rotate_left, (c x, v y) {
 526             let layout = x.layout();
 527             let x = x.load_scalar(fx);
 528             let res = fx.bcx.ins().rotl(x, y);
 529             ret.write_cvalue(fx, CValue::by_val(res, layout));
 530         };
 531         rotate_right, (c x, v y) {
 532             let layout = x.layout();
 533             let x = x.load_scalar(fx);
 534             let res = fx.bcx.ins().rotr(x, y);
 535             ret.write_cvalue(fx, CValue::by_val(res, layout));
 536         };
 537
 538         // The only difference between offset and arith_offset is regarding UB. Because Cranelift
 539         // doesn't have UB both are codegen'ed the same way
 540         offset | arith_offset, (c base, v offset) {
 541             let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
 542             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 543             let ptr_diff = if pointee_size != 1 {
 544                 fx.bcx.ins().imul_imm(offset, pointee_size as i64)
 545             } else {
 546                 offset
 547             };
 548             let base_val = base.load_scalar(fx);
 549             let res = fx.bcx.ins().iadd(base_val, ptr_diff);
 550             ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
 551         };
 552
 553         transmute, (c from) {
 554             ret.write_cvalue_transmute(fx, from);
 555         };
 556         write_bytes | volatile_set_memory, (c dst, v val, v count) {
 557             let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
 558             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 559             let count = if pointee_size != 1 {
 560                 fx.bcx.ins().imul_imm(count, pointee_size as i64)
 561             } else {
 562                 count
 563             };
 564             let dst_ptr = dst.load_scalar(fx);
 565             // FIXME make the memset actually volatile when switching to emit_small_memset
 566             // FIXME use emit_small_memset
 567             fx.bcx.call_memset(fx.target_config, dst_ptr, val, count);
 568         };
 569         ctlz | ctlz_nonzero, (c arg) {
 570             let val = arg.load_scalar(fx);
 571             // FIXME trap on `ctlz_nonzero` with zero arg.
 572             let res = fx.bcx.ins().clz(val);
 573             let res = CValue::by_val(res, arg.layout());
 574             ret.write_cvalue(fx, res);
 575         };
 576         cttz | cttz_nonzero, (c arg) {
 577             let val = arg.load_scalar(fx);
 578             // FIXME trap on `cttz_nonzero` with zero arg.
 579             let res = fx.bcx.ins().ctz(val);
 580             let res = CValue::by_val(res, arg.layout());
 581             ret.write_cvalue(fx, res);
 582         };
 583         ctpop, (c arg) {
 584             let val = arg.load_scalar(fx);
 585             let res = fx.bcx.ins().popcnt(val);
 586             let res = CValue::by_val(res, arg.layout());
 587             ret.write_cvalue(fx, res);
 588         };
 589         bitreverse, (c arg) {
 590             let val = arg.load_scalar(fx);
 591             let res = fx.bcx.ins().bitrev(val);
 592             let res = CValue::by_val(res, arg.layout());
 593             ret.write_cvalue(fx, res);
 594         };
 595         bswap, (c arg) {
 596             // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
 597             fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value {
 598                 match bcx.func.dfg.value_type(v) {
 599                     types::I8 => v,
 600
 601                     // https://code.woboq.org/gcc/include/bits/byteswap.h.html
 602                     types::I16 => {
 603                         let tmp1 = bcx.ins().ishl_imm(v, 8);
 604                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00);
 605
 606                         let tmp2 = bcx.ins().ushr_imm(v, 8);
 607                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF);
 608
 609                         bcx.ins().bor(n1, n2)
 610                     }
 611                     types::I32 => {
 612                         let tmp1 = bcx.ins().ishl_imm(v, 24);
 613                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000);
 614
 615                         let tmp2 = bcx.ins().ishl_imm(v, 8);
 616                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000);
 617
 618                         let tmp3 = bcx.ins().ushr_imm(v, 8);
 619                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00);
 620
 621                         let tmp4 = bcx.ins().ushr_imm(v, 24);
 622                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF);
 623
 624                         let or_tmp1 = bcx.ins().bor(n1, n2);
 625                         let or_tmp2 = bcx.ins().bor(n3, n4);
 626                         bcx.ins().bor(or_tmp1, or_tmp2)
 627                     }
 628                     types::I64 => {
 629                         let tmp1 = bcx.ins().ishl_imm(v, 56);
 630                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64);
 631
 632                         let tmp2 = bcx.ins().ishl_imm(v, 40);
 633                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64);
 634
 635                         let tmp3 = bcx.ins().ishl_imm(v, 24);
 636                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64);
 637
 638                         let tmp4 = bcx.ins().ishl_imm(v, 8);
 639                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64);
 640
 641                         let tmp5 = bcx.ins().ushr_imm(v, 8);
 642                         let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64);
 643
 644                         let tmp6 = bcx.ins().ushr_imm(v, 24);
 645                         let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64);
 646
 647                         let tmp7 = bcx.ins().ushr_imm(v, 40);
 648                         let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64);
 649
 650                         let tmp8 = bcx.ins().ushr_imm(v, 56);
 651                         let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64);
 652
 653                         let or_tmp1 = bcx.ins().bor(n1, n2);
 654                         let or_tmp2 = bcx.ins().bor(n3, n4);
 655                         let or_tmp3 = bcx.ins().bor(n5, n6);
 656                         let or_tmp4 = bcx.ins().bor(n7, n8);
 657
 658                         let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2);
 659                         let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4);
 660                         bcx.ins().bor(or_tmp5, or_tmp6)
 661                     }
 662                     types::I128 => {
 663                         let (lo, hi) = bcx.ins().isplit(v);
 664                         let lo = swap(bcx, lo);
 665                         let hi = swap(bcx, hi);
 666                         bcx.ins().iconcat(hi, lo)
 667                     }
 668                     ty => unreachable!("bswap {}", ty),
 669                 }
 670             }
 671             let val = arg.load_scalar(fx);
 672             let res = CValue::by_val(swap(&mut fx.bcx, val), arg.layout());
 673             ret.write_cvalue(fx, res);
 674         };
 675         assert_inhabited | assert_zero_valid | assert_uninit_valid, () {
 676             let layout = fx.layout_of(substs.type_at(0));
 677             if layout.abi.is_uninhabited() {
 678                 with_no_trimmed_paths!({
 679                     crate::base::codegen_panic(
 680                         fx,
 681                         &format!("attempted to instantiate uninhabited type `{}`", layout.ty),
 682                         source_info,
 683                     )
 684                 });
 685                 return;
 686             }
 687
 688             if intrinsic == sym::assert_zero_valid && !fx.tcx.permits_zero_init(layout) {
 689                 with_no_trimmed_paths!({
 690                     crate::base::codegen_panic(
 691                         fx,
 692                         &format!("attempted to zero-initialize type `{}`, which is invalid", layout.ty),
 693                         source_info,
 694                     );
 695                 });
 696                 return;
 697             }
 698
 699             if intrinsic == sym::assert_uninit_valid && !fx.tcx.permits_uninit_init(layout) {
 700                 with_no_trimmed_paths!({
 701                     crate::base::codegen_panic(
 702                         fx,
 703                         &format!("attempted to leave type `{}` uninitialized, which is invalid", layout.ty),
 704                         source_info,
 705                     )
 706                 });
 707                 return;
 708             }
 709         };
 710
 711         volatile_load | unaligned_volatile_load, (c ptr) {
 712             // Cranelift treats loads as volatile by default
 713             // FIXME correctly handle unaligned_volatile_load
 714             let inner_layout =
 715                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 716             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 717             ret.write_cvalue(fx, val);
 718         };
 719         volatile_store | unaligned_volatile_store, (v ptr, c val) {
 720             // Cranelift treats stores as volatile by default
 721             // FIXME correctly handle unaligned_volatile_store
 722             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 723             dest.write_cvalue(fx, val);
 724         };
 725
 726         pref_align_of | needs_drop | type_id | type_name | variant_count, () {
 727             let const_val =
 728                 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
 729             let val = crate::constant::codegen_const_value(
 730                 fx,
 731                 const_val,
 732                 ret.layout().ty,
 733             );
 734             ret.write_cvalue(fx, val);
 735         };
 736
 737         ptr_offset_from | ptr_offset_from_unsigned, (v ptr, v base) {
 738             let ty = substs.type_at(0);
 739
 740             let pointee_size: u64 = fx.layout_of(ty).size.bytes();
 741             let diff_bytes = fx.bcx.ins().isub(ptr, base);
 742             // FIXME this can be an exact division.
 743             let val = if intrinsic == sym::ptr_offset_from_unsigned {
 744                 let usize_layout = fx.layout_of(fx.tcx.types.usize);
 745                 // Because diff_bytes ULE isize::MAX, this would be fine as signed,
 746                 // but unsigned is slightly easier to codegen, so might as well.
 747                 CValue::by_val(fx.bcx.ins().udiv_imm(diff_bytes, pointee_size as i64), usize_layout)
 748             } else {
 749                 let isize_layout = fx.layout_of(fx.tcx.types.isize);
 750                 CValue::by_val(fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64), isize_layout)
 751             };
 752             ret.write_cvalue(fx, val);
 753         };
 754
 755         ptr_guaranteed_eq, (c a, c b) {
 756             let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b);
 757             ret.write_cvalue(fx, val);
 758         };
 759
 760         ptr_guaranteed_ne, (c a, c b) {
 761             let val = crate::num::codegen_ptr_binop(fx, BinOp::Ne, a, b);
 762             ret.write_cvalue(fx, val);
 763         };
 764
 765         caller_location, () {
 766             let caller_location = fx.get_caller_location(source_info);
 767             ret.write_cvalue(fx, caller_location);
 768         };
 769
 770         _ if intrinsic.as_str().starts_with("atomic_fence"), () {
 771             fx.bcx.ins().fence();
 772         };
 773         _ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () {
 774             // FIXME use a compiler fence once Cranelift supports it
 775             fx.bcx.ins().fence();
 776         };
 777         _ if intrinsic.as_str().starts_with("atomic_load"), (v ptr) {
 778             let ty = substs.type_at(0);
 779             match ty.kind() {
 780                 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
 781                     // FIXME implement 128bit atomics
 782                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
 783                         // special case for compiler-builtins to avoid having to patch it
 784                         crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
 785                         let ret_block = fx.get_block(destination.unwrap());
 786                         fx.bcx.ins().jump(ret_block, &[]);
 787                         return;
 788                     } else {
 789                         fx.tcx.sess.span_fatal(source_info.span, "128bit atomics not yet supported");
 790                     }
 791                 }
 792                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 793                 _ => {
 794                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
 795                     return;
 796                 }
 797             }
 798             let clif_ty = fx.clif_type(ty).unwrap();
 799
 800             let val = fx.bcx.ins().atomic_load(clif_ty, MemFlags::trusted(), ptr);
 801
 802             let val = CValue::by_val(val, fx.layout_of(ty));
 803             ret.write_cvalue(fx, val);
 804         };
 805         _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) {
 806             let ty = substs.type_at(0);
 807             match ty.kind() {
 808                 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
 809                     // FIXME implement 128bit atomics
 810                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
 811                         // special case for compiler-builtins to avoid having to patch it
 812                         crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
 813                         let ret_block = fx.get_block(destination.unwrap());
 814                         fx.bcx.ins().jump(ret_block, &[]);
 815                         return;
 816                     } else {
 817                         fx.tcx.sess.span_fatal(source_info.span, "128bit atomics not yet supported");
 818                     }
 819                 }
 820                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 821                 _ => {
 822                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
 823                     return;
 824                 }
 825             }
 826
 827             let val = val.load_scalar(fx);
 828
 829             fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
 830         };
 831         _ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) {
 832             let layout = new.layout();
 833             match layout.ty.kind() {
 834                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 835                 _ => {
 836                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 837                     return;
 838                 }
 839             }
 840             let ty = fx.clif_type(layout.ty).unwrap();
 841
 842             let new = new.load_scalar(fx);
 843
 844             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new);
 845
 846             let old = CValue::by_val(old, layout);
 847             ret.write_cvalue(fx, old);
 848         };
 849         _ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
 850             let layout = new.layout();
 851             match layout.ty.kind() {
 852                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 853                 _ => {
 854                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 855                     return;
 856                 }
 857             }
 858
 859             let test_old = test_old.load_scalar(fx);
 860             let new = new.load_scalar(fx);
 861
 862             let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new);
 863             let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
 864
 865             let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
 866             ret.write_cvalue(fx, ret_val)
 867         };
 868
 869         _ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) {
 870             let layout = amount.layout();
 871             match layout.ty.kind() {
 872                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 873                 _ => {
 874                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 875                     return;
 876                 }
 877             }
 878             let ty = fx.clif_type(layout.ty).unwrap();
 879
 880             let amount = amount.load_scalar(fx);
 881
 882             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
 883
 884             let old = CValue::by_val(old, layout);
 885             ret.write_cvalue(fx, old);
 886         };
 887         _ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) {
 888             let layout = amount.layout();
 889             match layout.ty.kind() {
 890                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 891                 _ => {
 892                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 893                     return;
 894                 }
 895             }
 896             let ty = fx.clif_type(layout.ty).unwrap();
 897
 898             let amount = amount.load_scalar(fx);
 899
 900             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
 901
 902             let old = CValue::by_val(old, layout);
 903             ret.write_cvalue(fx, old);
 904         };
 905         _ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) {
 906             let layout = src.layout();
 907             match layout.ty.kind() {
 908                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 909                 _ => {
 910                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 911                     return;
 912                 }
 913             }
 914             let ty = fx.clif_type(layout.ty).unwrap();
 915
 916             let src = src.load_scalar(fx);
 917
 918             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src);
 919
 920             let old = CValue::by_val(old, layout);
 921             ret.write_cvalue(fx, old);
 922         };
 923         _ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) {
 924             let layout = src.layout();
 925             match layout.ty.kind() {
 926                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 927                 _ => {
 928                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 929                     return;
 930                 }
 931             }
 932             let ty = fx.clif_type(layout.ty).unwrap();
 933
 934             let src = src.load_scalar(fx);
 935
 936             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src);
 937
 938             let old = CValue::by_val(old, layout);
 939             ret.write_cvalue(fx, old);
 940         };
 941         _ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) {
 942             let layout = src.layout();
 943             match layout.ty.kind() {
 944                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 945                 _ => {
 946                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 947                     return;
 948                 }
 949             }
 950             let ty = fx.clif_type(layout.ty).unwrap();
 951
 952             let src = src.load_scalar(fx);
 953
 954             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src);
 955
 956             let old = CValue::by_val(old, layout);
 957             ret.write_cvalue(fx, old);
 958         };
 959         _ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) {
 960             let layout = src.layout();
 961             match layout.ty.kind() {
 962                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 963                 _ => {
 964                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 965                     return;
 966                 }
 967             }
 968             let ty = fx.clif_type(layout.ty).unwrap();
 969
 970             let src = src.load_scalar(fx);
 971
 972             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src);
 973
 974             let old = CValue::by_val(old, layout);
 975             ret.write_cvalue(fx, old);
 976         };
 977         _ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) {
 978             let layout = src.layout();
 979             match layout.ty.kind() {
 980                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 981                 _ => {
 982                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 983                     return;
 984                 }
 985             }
 986             let ty = fx.clif_type(layout.ty).unwrap();
 987
 988             let src = src.load_scalar(fx);
 989
 990             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src);
 991
 992             let old = CValue::by_val(old, layout);
 993             ret.write_cvalue(fx, old);
 994         };
 995         _ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) {
 996             let layout = src.layout();
 997             match layout.ty.kind() {
 998                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 999                 _ => {
1000                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1001                     return;
1002                 }
1003             }
1004             let ty = fx.clif_type(layout.ty).unwrap();
1005
1006             let src = src.load_scalar(fx);
1007
1008             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src);
1009
1010             let old = CValue::by_val(old, layout);
1011             ret.write_cvalue(fx, old);
1012         };
1013         _ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) {
1014             let layout = src.layout();
1015             match layout.ty.kind() {
1016                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1017                 _ => {
1018                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1019                     return;
1020                 }
1021             }
1022             let ty = fx.clif_type(layout.ty).unwrap();
1023
1024             let src = src.load_scalar(fx);
1025
1026             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src);
1027
1028             let old = CValue::by_val(old, layout);
1029             ret.write_cvalue(fx, old);
1030         };
1031         _ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) {
1032             let layout = src.layout();
1033             match layout.ty.kind() {
1034                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1035                 _ => {
1036                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1037                     return;
1038                 }
1039             }
1040             let ty = fx.clif_type(layout.ty).unwrap();
1041
1042             let src = src.load_scalar(fx);
1043
1044             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src);
1045
1046             let old = CValue::by_val(old, layout);
1047             ret.write_cvalue(fx, old);
1048         };
1049
1050         minnumf32, (v a, v b) {
1051             let val = crate::num::codegen_float_min(fx, a, b);
1052             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1053             ret.write_cvalue(fx, val);
1054         };
1055         minnumf64, (v a, v b) {
1056             let val = crate::num::codegen_float_min(fx, a, b);
1057             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1058             ret.write_cvalue(fx, val);
1059         };
1060         maxnumf32, (v a, v b) {
1061             let val = crate::num::codegen_float_max(fx, a, b);
1062             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1063             ret.write_cvalue(fx, val);
1064         };
1065         maxnumf64, (v a, v b) {
1066             let val = crate::num::codegen_float_max(fx, a, b);
1067             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1068             ret.write_cvalue(fx, val);
1069         };
1070
1071         kw.Try, (v f, v data, v _catch_fn) {
1072             // FIXME once unwinding is supported, change this to actually catch panics
1073             let f_sig = fx.bcx.func.import_signature(Signature {
1074                 call_conv: fx.target_config.default_call_conv,
1075                 params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
1076                 returns: vec![],
1077             });
1078
1079             fx.bcx.ins().call_indirect(f_sig, f, &[data]);
1080
1081             let layout = ret.layout();
1082             let ret_val = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size));
1083             ret.write_cvalue(fx, ret_val);
1084         };
1085
1086         fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) {
1087             let res = crate::num::codegen_float_binop(fx, match intrinsic {
1088                 sym::fadd_fast => BinOp::Add,
1089                 sym::fsub_fast => BinOp::Sub,
1090                 sym::fmul_fast => BinOp::Mul,
1091                 sym::fdiv_fast => BinOp::Div,
1092                 sym::frem_fast => BinOp::Rem,
1093                 _ => unreachable!(),
1094             }, x, y);
1095             ret.write_cvalue(fx, res);
1096         };
1097         float_to_int_unchecked, (v f) {
1098             let res = crate::cast::clif_int_or_float_cast(
1099                 fx,
1100                 f,
1101                 false,
1102                 fx.clif_type(ret.layout().ty).unwrap(),
1103                 type_sign(ret.layout().ty),
1104             );
1105             ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
1106         };
1107
1108         raw_eq, (v lhs_ref, v rhs_ref) {
1109             let size = fx.layout_of(substs.type_at(0)).layout.size();
1110             // FIXME add and use emit_small_memcmp
1111             let is_eq_value =
1112                 if size == Size::ZERO {
1113                     // No bytes means they're trivially equal
1114                     fx.bcx.ins().iconst(types::I8, 1)
1115                 } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) {
1116                     // Can't use `trusted` for these loads; they could be unaligned.
1117                     let mut flags = MemFlags::new();
1118                     flags.set_notrap();
1119                     let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0);
1120                     let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0);
1121                     let eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val);
1122                     fx.bcx.ins().bint(types::I8, eq)
1123                 } else {
1124                     // Just call `memcmp` (like slices do in core) when the
1125                     // size is too large or it's not a power-of-two.
1126                     let signed_bytes = i64::try_from(size.bytes()).unwrap();
1127                     let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes);
1128                     let params = vec![AbiParam::new(fx.pointer_type); 3];
1129                     let returns = vec![AbiParam::new(types::I32)];
1130                     let args = &[lhs_ref, rhs_ref, bytes_val];
1131                     let cmp = fx.lib_call("memcmp", params, returns, args)[0];
1132                     let eq = fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0);
1133                     fx.bcx.ins().bint(types::I8, eq)
1134                 };
1135             ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout()));
1136         };
1137
1138         const_allocate, (c _size, c _align) {
1139             // returns a null pointer at runtime.
1140             let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
1141             ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
1142         };
1143
1144         const_deallocate, (c _ptr, c _size, c _align) {
1145             // nop at runtime.
1146         };
1147
1148         black_box, (c a) {
1149             // FIXME implement black_box semantics
1150             ret.write_cvalue(fx, a);
1151         };
1152
1153         // FIXME implement variadics in cranelift
1154         va_copy, (o _dest, o _src) {
1155             fx.tcx.sess.span_fatal(
1156                 source_info.span,
1157                 "Defining variadic functions is not yet supported by Cranelift",
1158             );
1159         };
1160         va_arg | va_end, (o _valist) {
1161             fx.tcx.sess.span_fatal(
1162                 source_info.span,
1163                 "Defining variadic functions is not yet supported by Cranelift",
1164             );
1165         };
1166     }
1167
1168     let ret_block = fx.get_block(destination.unwrap());
1169     fx.bcx.ins().jump(ret_block, &[]);
1170 }