src/intrinsics/mod.rs

   1 //! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"`
   2 //! and LLVM intrinsics that have symbol names starting with `llvm.`.
   3
   4 mod cpuid;
   5 mod llvm;
   6 mod simd;
   7
   8 pub(crate) use cpuid::codegen_cpuid_call;
   9 pub(crate) use llvm::codegen_llvm_intrinsic_call;
  10
  11 use crate::prelude::*;
  12
  13 macro intrinsic_pat {
  14     (_) => {
  15         _
  16     },
  17     ($name:ident) => {
  18         stringify!($name)
  19     },
  20     ($name:literal) => {
  21         stringify!($name)
  22     },
  23     ($x:ident . $($xs:tt).*) => {
  24         concat!(stringify!($x), ".", intrinsic_pat!($($xs).*))
  25     }
  26 }
  27
  28 macro intrinsic_arg {
  29     (o $fx:expr, $arg:ident) => {
  30         $arg
  31     },
  32     (c $fx:expr, $arg:ident) => {
  33         trans_operand($fx, $arg)
  34     },
  35     (v $fx:expr, $arg:ident) => {
  36         trans_operand($fx, $arg).load_scalar($fx)
  37     }
  38 }
  39
  40 macro intrinsic_substs {
  41     ($substs:expr, $index:expr,) => {},
  42     ($substs:expr, $index:expr, $first:ident $(,$rest:ident)*) => {
  43         let $first = $substs.type_at($index);
  44         intrinsic_substs!($substs, $index+1, $($rest),*);
  45     }
  46 }
  47
  48 macro intrinsic_match {
  49     ($fx:expr, $intrinsic:expr, $substs:expr, $args:expr,
  50     _ => $unknown:block;
  51     $(
  52         $($($name:tt).*)|+ $(if $cond:expr)?, $(<$($subst:ident),*>)? ($($a:ident $arg:ident),*) $content:block;
  53     )*) => {
  54         let _ = $substs; // Silence warning when substs is unused.
  55         match $intrinsic {
  56             $(
  57                 $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
  58                     #[allow(unused_parens, non_snake_case)]
  59                     {
  60                         $(
  61                             intrinsic_substs!($substs, 0, $($subst),*);
  62                         )?
  63                         if let [$($arg),*] = $args {
  64                             let ($($arg,)*) = (
  65                                 $(intrinsic_arg!($a $fx, $arg),)*
  66                             );
  67                             #[warn(unused_parens, non_snake_case)]
  68                             {
  69                                 $content
  70                             }
  71                         } else {
  72                             bug!("wrong number of args for intrinsic {:?}", $intrinsic);
  73                         }
  74                     }
  75                 }
  76             )*
  77             _ => $unknown,
  78         }
  79     }
  80 }
  81
  82 macro call_intrinsic_match {
  83     ($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $destination:expr, $args:expr, $(
  84         $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
  85     )*) => {
  86         match $intrinsic {
  87             $(
  88                 stringify!($name) => {
  89                     assert!($substs.is_noop());
  90                     if let [$(ref $arg),*] = *$args {
  91                         let ($($arg,)*) = (
  92                             $(trans_operand($fx, $arg),)*
  93                         );
  94                         let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty);
  95                         $ret.write_cvalue($fx, res);
  96
  97                         if let Some((_, dest)) = $destination {
  98                             let ret_block = $fx.get_block(dest);
  99                             $fx.bcx.ins().jump(ret_block, &[]);
 100                             return;
 101                         } else {
 102                             unreachable!();
 103                         }
 104                     } else {
 105                         bug!("wrong number of args for intrinsic {:?}", $intrinsic);
 106                     }
 107                 }
 108             )*
 109             _ => {}
 110         }
 111     }
 112 }
 113
 114 macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) {
 115     crate::atomic_shim::lock_global_lock($fx);
 116
 117     let clif_ty = $fx.clif_type($T).unwrap();
 118     let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 119     let new = $fx.bcx.ins().$op(old, $src);
 120     $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 121     $ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
 122
 123     crate::atomic_shim::unlock_global_lock($fx);
 124 }
 125
 126 macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) {
 127     crate::atomic_shim::lock_global_lock($fx);
 128
 129     // Read old
 130     let clif_ty = $fx.clif_type($T).unwrap();
 131     let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 132
 133     // Compare
 134     let is_eq = $fx.bcx.ins().icmp(IntCC::SignedGreaterThan, old, $src);
 135     let new = $fx.bcx.ins().select(is_eq, old, $src);
 136
 137     // Write new
 138     $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 139
 140     let ret_val = CValue::by_val(old, $ret.layout());
 141     $ret.write_cvalue($fx, ret_val);
 142
 143     crate::atomic_shim::unlock_global_lock($fx);
 144 }
 145
 146 macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
 147     match $ty.kind() {
 148         ty::Uint(_) | ty::Int(_) => {}
 149         _ => {
 150             $fx.tcx.sess.span_err(
 151                 $span,
 152                 &format!(
 153                     "`{}` intrinsic: expected basic integer type, found `{:?}`",
 154                     $intrinsic, $ty
 155                 ),
 156             );
 157             // Prevent verifier error
 158             crate::trap::trap_unreachable($fx, "compilation should not have succeeded");
 159             return;
 160         }
 161     }
 162 }
 163
 164 macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
 165     if !$ty.is_simd() {
 166         $fx.tcx.sess.span_err($span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", $intrinsic, $ty));
 167         // Prevent verifier error
 168         crate::trap::trap_unreachable($fx, "compilation should not have succeeded");
 169         return;
 170     }
 171 }
 172
 173 fn lane_type_and_count<'tcx>(
 174     tcx: TyCtxt<'tcx>,
 175     layout: TyAndLayout<'tcx>,
 176 ) -> (TyAndLayout<'tcx>, u16) {
 177     assert!(layout.ty.is_simd());
 178     let lane_count = match layout.fields {
 179         rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(),
 180         _ => unreachable!("lane_type_and_count({:?})", layout),
 181     };
 182     let lane_layout = layout
 183         .field(
 184             &ty::layout::LayoutCx {
 185                 tcx,
 186                 param_env: ParamEnv::reveal_all(),
 187             },
 188             0,
 189         )
 190         .unwrap();
 191     (lane_layout, lane_count)
 192 }
 193
 194 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
 195     let (element, count) = match &layout.abi {
 196         Abi::Vector { element, count } => (element.clone(), *count),
 197         _ => unreachable!(),
 198     };
 199
 200     match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
 201         // Cranelift currently only implements icmp for 128bit vectors.
 202         Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
 203         _ => None,
 204     }
 205 }
 206
 207 fn simd_for_each_lane<'tcx, M: Module>(
 208     fx: &mut FunctionCx<'_, 'tcx, M>,
 209     val: CValue<'tcx>,
 210     ret: CPlace<'tcx>,
 211     f: impl Fn(
 212         &mut FunctionCx<'_, 'tcx, M>,
 213         TyAndLayout<'tcx>,
 214         TyAndLayout<'tcx>,
 215         Value,
 216     ) -> CValue<'tcx>,
 217 ) {
 218     let layout = val.layout();
 219
 220     let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
 221     let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
 222     assert_eq!(lane_count, ret_lane_count);
 223
 224     for lane_idx in 0..lane_count {
 225         let lane_idx = mir::Field::new(lane_idx.try_into().unwrap());
 226         let lane = val.value_field(fx, lane_idx).load_scalar(fx);
 227
 228         let res_lane = f(fx, lane_layout, ret_lane_layout, lane);
 229
 230         ret.place_field(fx, lane_idx).write_cvalue(fx, res_lane);
 231     }
 232 }
 233
 234 fn simd_pair_for_each_lane<'tcx, M: Module>(
 235     fx: &mut FunctionCx<'_, 'tcx, M>,
 236     x: CValue<'tcx>,
 237     y: CValue<'tcx>,
 238     ret: CPlace<'tcx>,
 239     f: impl Fn(
 240         &mut FunctionCx<'_, 'tcx, M>,
 241         TyAndLayout<'tcx>,
 242         TyAndLayout<'tcx>,
 243         Value,
 244         Value,
 245     ) -> CValue<'tcx>,
 246 ) {
 247     assert_eq!(x.layout(), y.layout());
 248     let layout = x.layout();
 249
 250     let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
 251     let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
 252     assert_eq!(lane_count, ret_lane_count);
 253
 254     for lane in 0..lane_count {
 255         let lane = mir::Field::new(lane.try_into().unwrap());
 256         let x_lane = x.value_field(fx, lane).load_scalar(fx);
 257         let y_lane = y.value_field(fx, lane).load_scalar(fx);
 258
 259         let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane);
 260
 261         ret.place_field(fx, lane).write_cvalue(fx, res_lane);
 262     }
 263 }
 264
 265 fn bool_to_zero_or_max_uint<'tcx>(
 266     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 267     layout: TyAndLayout<'tcx>,
 268     val: Value,
 269 ) -> CValue<'tcx> {
 270     let ty = fx.clif_type(layout.ty).unwrap();
 271
 272     let int_ty = match ty {
 273         types::F32 => types::I32,
 274         types::F64 => types::I64,
 275         ty => ty,
 276     };
 277
 278     let val = fx.bcx.ins().bint(int_ty, val);
 279     let mut res = fx.bcx.ins().ineg(val);
 280
 281     if ty.is_float() {
 282         res = fx.bcx.ins().bitcast(ty, res);
 283     }
 284
 285     CValue::by_val(res, layout)
 286 }
 287
 288 macro simd_cmp {
 289     ($fx:expr, $cc:ident($x:ident, $y:ident) -> $ret:ident) => {
 290         let vector_ty = clif_vector_type($fx.tcx, $x.layout());
 291
 292         if let Some(vector_ty) = vector_ty {
 293             let x = $x.load_scalar($fx);
 294             let y = $y.load_scalar($fx);
 295             let val = $fx.bcx.ins().icmp(IntCC::$cc, x, y);
 296
 297             // HACK This depends on the fact that icmp for vectors represents bools as 0 and !0, not 0 and 1.
 298             let val = $fx.bcx.ins().raw_bitcast(vector_ty, val);
 299
 300             $ret.write_cvalue($fx, CValue::by_val(val, $ret.layout()));
 301         } else {
 302             simd_pair_for_each_lane(
 303                 $fx,
 304                 $x,
 305                 $y,
 306                 $ret,
 307                 |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 308                     let res_lane = match lane_layout.ty.kind() {
 309                         ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane),
 310                         _ => unreachable!("{:?}", lane_layout.ty),
 311                     };
 312                     bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 313                 },
 314             );
 315         }
 316     },
 317     ($fx:expr, $cc_u:ident|$cc_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 318         // FIXME use vector icmp when possible
 319         simd_pair_for_each_lane(
 320             $fx,
 321             $x,
 322             $y,
 323             $ret,
 324             |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 325                 let res_lane = match lane_layout.ty.kind() {
 326                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
 327                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
 328                     _ => unreachable!("{:?}", lane_layout.ty),
 329                 };
 330                 bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 331             },
 332         );
 333     },
 334 }
 335
 336 macro simd_int_binop {
 337     ($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
 338         simd_int_binop!($fx, $op|$op($x, $y) -> $ret);
 339     },
 340     ($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 341         simd_pair_for_each_lane(
 342             $fx,
 343             $x,
 344             $y,
 345             $ret,
 346             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 347                 let res_lane = match lane_layout.ty.kind() {
 348                     ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 349                     ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 350                     _ => unreachable!("{:?}", lane_layout.ty),
 351                 };
 352                 CValue::by_val(res_lane, ret_lane_layout)
 353             },
 354         );
 355     },
 356 }
 357
 358 macro simd_int_flt_binop {
 359     ($fx:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 360         simd_int_flt_binop!($fx, $op|$op|$op_f($x, $y) -> $ret);
 361     },
 362     ($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 363         simd_pair_for_each_lane(
 364             $fx,
 365             $x,
 366             $y,
 367             $ret,
 368             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 369                 let res_lane = match lane_layout.ty.kind() {
 370                     ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 371                     ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 372                     ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
 373                     _ => unreachable!("{:?}", lane_layout.ty),
 374                 };
 375                 CValue::by_val(res_lane, ret_lane_layout)
 376             },
 377         );
 378     },
 379 }
 380
 381 macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
 382     simd_pair_for_each_lane(
 383         $fx,
 384         $x,
 385         $y,
 386         $ret,
 387         |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 388             let res_lane = match lane_layout.ty.kind() {
 389                 ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
 390                 _ => unreachable!("{:?}", lane_layout.ty),
 391             };
 392             CValue::by_val(res_lane, ret_lane_layout)
 393         },
 394     );
 395 }
 396
 397 pub(crate) fn codegen_intrinsic_call<'tcx>(
 398     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 399     instance: Instance<'tcx>,
 400     args: &[mir::Operand<'tcx>],
 401     destination: Option<(CPlace<'tcx>, BasicBlock)>,
 402     span: Span,
 403 ) {
 404     let def_id = instance.def_id();
 405     let substs = instance.substs;
 406
 407     let intrinsic = fx.tcx.item_name(def_id).as_str();
 408     let intrinsic = &intrinsic[..];
 409
 410     let ret = match destination {
 411         Some((place, _)) => place,
 412         None => {
 413             // Insert non returning intrinsics here
 414             match intrinsic {
 415                 "abort" => {
 416                     trap_abort(fx, "Called intrinsic::abort.");
 417                 }
 418                 "unreachable" => {
 419                     trap_unreachable(fx, "[corruption] Called intrinsic::unreachable.");
 420                 }
 421                 "transmute" => {
 422                     crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span);
 423                 }
 424                 _ => unimplemented!("unsupported instrinsic {}", intrinsic),
 425             }
 426             return;
 427         }
 428     };
 429
 430     if intrinsic.starts_with("simd_") {
 431         self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span);
 432         let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
 433         fx.bcx.ins().jump(ret_block, &[]);
 434         return;
 435     }
 436
 437     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 438
 439     call_intrinsic_match! {
 440         fx, intrinsic, substs, ret, destination, args,
 441         expf32(flt) -> f32 => expf,
 442         expf64(flt) -> f64 => exp,
 443         exp2f32(flt) -> f32 => exp2f,
 444         exp2f64(flt) -> f64 => exp2,
 445         sqrtf32(flt) -> f32 => sqrtf,
 446         sqrtf64(flt) -> f64 => sqrt,
 447         powif32(a, x) -> f32 => __powisf2, // compiler-builtins
 448         powif64(a, x) -> f64 => __powidf2, // compiler-builtins
 449         powf32(a, x) -> f32 => powf,
 450         powf64(a, x) -> f64 => pow,
 451         logf32(flt) -> f32 => logf,
 452         logf64(flt) -> f64 => log,
 453         log2f32(flt) -> f32 => log2f,
 454         log2f64(flt) -> f64 => log2,
 455         log10f32(flt) -> f32 => log10f,
 456         log10f64(flt) -> f64 => log10,
 457         fabsf32(flt) -> f32 => fabsf,
 458         fabsf64(flt) -> f64 => fabs,
 459         fmaf32(x, y, z) -> f32 => fmaf,
 460         fmaf64(x, y, z) -> f64 => fma,
 461         copysignf32(x, y) -> f32 => copysignf,
 462         copysignf64(x, y) -> f64 => copysign,
 463
 464         // rounding variants
 465         // FIXME use clif insts
 466         floorf32(flt) -> f32 => floorf,
 467         floorf64(flt) -> f64 => floor,
 468         ceilf32(flt) -> f32 => ceilf,
 469         ceilf64(flt) -> f64 => ceil,
 470         truncf32(flt) -> f32 => truncf,
 471         truncf64(flt) -> f64 => trunc,
 472         roundf32(flt) -> f32 => roundf,
 473         roundf64(flt) -> f64 => round,
 474
 475         // trigonometry
 476         sinf32(flt) -> f32 => sinf,
 477         sinf64(flt) -> f64 => sin,
 478         cosf32(flt) -> f32 => cosf,
 479         cosf64(flt) -> f64 => cos,
 480         tanf32(flt) -> f32 => tanf,
 481         tanf64(flt) -> f64 => tan,
 482     }
 483
 484     intrinsic_match! {
 485         fx, intrinsic, substs, args,
 486         _ => {
 487             fx.tcx.sess.span_fatal(span, &format!("unsupported intrinsic {}", intrinsic));
 488         };
 489
 490         assume, (c _a) {};
 491         likely | unlikely, (c a) {
 492             ret.write_cvalue(fx, a);
 493         };
 494         breakpoint, () {
 495             fx.bcx.ins().debugtrap();
 496         };
 497         copy | copy_nonoverlapping, <elem_ty> (v src, v dst, v count) {
 498             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 499             let elem_size = fx
 500                 .bcx
 501                 .ins()
 502                 .iconst(fx.pointer_type, elem_size as i64);
 503             assert_eq!(args.len(), 3);
 504             let byte_amount = fx.bcx.ins().imul(count, elem_size);
 505
 506             if intrinsic.contains("nonoverlapping") {
 507                 // FIXME emit_small_memcpy
 508                 fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount);
 509             } else {
 510                 // FIXME emit_small_memmove
 511                 fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
 512             }
 513         };
 514         // NOTE: the volatile variants have src and dst swapped
 515         volatile_copy_memory | volatile_copy_nonoverlapping_memory, <elem_ty> (v dst, v src, v count) {
 516             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 517             let elem_size = fx
 518                 .bcx
 519                 .ins()
 520                 .iconst(fx.pointer_type, elem_size as i64);
 521             assert_eq!(args.len(), 3);
 522             let byte_amount = fx.bcx.ins().imul(count, elem_size);
 523
 524             // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
 525             if intrinsic.contains("nonoverlapping") {
 526                 // FIXME emit_small_memcpy
 527                 fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount);
 528             } else {
 529                 // FIXME emit_small_memmove
 530                 fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
 531             }
 532         };
 533         discriminant_value, (c ptr) {
 534             let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 535             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout);
 536             let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout());
 537             ret.write_cvalue(fx, discr);
 538         };
 539         size_of_val, <T> (c ptr) {
 540             let layout = fx.layout_of(T);
 541             let size = if layout.is_unsized() {
 542                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 543                 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 544                 size
 545             } else {
 546                 fx
 547                     .bcx
 548                     .ins()
 549                     .iconst(fx.pointer_type, layout.size.bytes() as i64)
 550             };
 551             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 552         };
 553         min_align_of_val, <T> (c ptr) {
 554             let layout = fx.layout_of(T);
 555             let align = if layout.is_unsized() {
 556                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 557                 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 558                 align
 559             } else {
 560                 fx
 561                     .bcx
 562                     .ins()
 563                     .iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
 564             };
 565             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 566         };
 567
 568         _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) {
 569             // FIXME trap on overflow
 570             let bin_op = match intrinsic {
 571                 "unchecked_add" => BinOp::Add,
 572                 "unchecked_sub" => BinOp::Sub,
 573                 "unchecked_div" | "exact_div" => BinOp::Div,
 574                 "unchecked_rem" => BinOp::Rem,
 575                 "unchecked_shl" => BinOp::Shl,
 576                 "unchecked_shr" => BinOp::Shr,
 577                 _ => unreachable!("intrinsic {}", intrinsic),
 578             };
 579             let res = crate::num::trans_int_binop(fx, bin_op, x, y);
 580             ret.write_cvalue(fx, res);
 581         };
 582         _ if intrinsic.ends_with("_with_overflow"), (c x, c y) {
 583             assert_eq!(x.layout().ty, y.layout().ty);
 584             let bin_op = match intrinsic {
 585                 "add_with_overflow" => BinOp::Add,
 586                 "sub_with_overflow" => BinOp::Sub,
 587                 "mul_with_overflow" => BinOp::Mul,
 588                 _ => unreachable!("intrinsic {}", intrinsic),
 589             };
 590
 591             let res = crate::num::trans_checked_int_binop(
 592                 fx,
 593                 bin_op,
 594                 x,
 595                 y,
 596             );
 597             ret.write_cvalue(fx, res);
 598         };
 599         _ if intrinsic.starts_with("wrapping_"), (c x, c y) {
 600             assert_eq!(x.layout().ty, y.layout().ty);
 601             let bin_op = match intrinsic {
 602                 "wrapping_add" => BinOp::Add,
 603                 "wrapping_sub" => BinOp::Sub,
 604                 "wrapping_mul" => BinOp::Mul,
 605                 _ => unreachable!("intrinsic {}", intrinsic),
 606             };
 607             let res = crate::num::trans_int_binop(
 608                 fx,
 609                 bin_op,
 610                 x,
 611                 y,
 612             );
 613             ret.write_cvalue(fx, res);
 614         };
 615         _ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) {
 616             assert_eq!(lhs.layout().ty, rhs.layout().ty);
 617             let bin_op = match intrinsic {
 618                 "saturating_add" => BinOp::Add,
 619                 "saturating_sub" => BinOp::Sub,
 620                 _ => unreachable!("intrinsic {}", intrinsic),
 621             };
 622
 623             let signed = type_sign(T);
 624
 625             let checked_res = crate::num::trans_checked_int_binop(
 626                 fx,
 627                 bin_op,
 628                 lhs,
 629                 rhs,
 630             );
 631
 632             let (val, has_overflow) = checked_res.load_scalar_pair(fx);
 633             let clif_ty = fx.clif_type(T).unwrap();
 634
 635             // `select.i8` is not implemented by Cranelift.
 636             let has_overflow = fx.bcx.ins().uextend(types::I32, has_overflow);
 637
 638             let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
 639
 640             let val = match (intrinsic, signed) {
 641                 ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val),
 642                 ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val),
 643                 ("saturating_add", true) => {
 644                     let rhs = rhs.load_scalar(fx);
 645                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 646                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
 647                     fx.bcx.ins().select(has_overflow, sat_val, val)
 648                 }
 649                 ("saturating_sub", true) => {
 650                     let rhs = rhs.load_scalar(fx);
 651                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 652                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
 653                     fx.bcx.ins().select(has_overflow, sat_val, val)
 654                 }
 655                 _ => unreachable!(),
 656             };
 657
 658             let res = CValue::by_val(val, fx.layout_of(T));
 659
 660             ret.write_cvalue(fx, res);
 661         };
 662         rotate_left, <T>(v x, v y) {
 663             let layout = fx.layout_of(T);
 664             let res = fx.bcx.ins().rotl(x, y);
 665             ret.write_cvalue(fx, CValue::by_val(res, layout));
 666         };
 667         rotate_right, <T>(v x, v y) {
 668             let layout = fx.layout_of(T);
 669             let res = fx.bcx.ins().rotr(x, y);
 670             ret.write_cvalue(fx, CValue::by_val(res, layout));
 671         };
 672
 673         // The only difference between offset and arith_offset is regarding UB. Because Cranelift
 674         // doesn't have UB both are codegen'ed the same way
 675         offset | arith_offset, (c base, v offset) {
 676             let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
 677             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 678             let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
 679             let base_val = base.load_scalar(fx);
 680             let res = fx.bcx.ins().iadd(base_val, ptr_diff);
 681             ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
 682         };
 683
 684         transmute, (c from) {
 685             ret.write_cvalue_transmute(fx, from);
 686         };
 687         write_bytes | volatile_set_memory, (c dst, v val, v count) {
 688             let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
 689             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 690             let count = fx.bcx.ins().imul_imm(count, pointee_size as i64);
 691             let dst_ptr = dst.load_scalar(fx);
 692             // FIXME make the memset actually volatile when switching to emit_small_memset
 693             // FIXME use emit_small_memset
 694             fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count);
 695         };
 696         ctlz | ctlz_nonzero, <T> (v arg) {
 697             // FIXME trap on `ctlz_nonzero` with zero arg.
 698             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 699                 // FIXME verify this algorithm is correct
 700                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 701                 let lsb_lz = fx.bcx.ins().clz(lsb);
 702                 let msb_lz = fx.bcx.ins().clz(msb);
 703                 let msb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, msb, 0);
 704                 let lsb_lz_plus_64 = fx.bcx.ins().iadd_imm(lsb_lz, 64);
 705                 let res = fx.bcx.ins().select(msb_is_zero, lsb_lz_plus_64, msb_lz);
 706                 fx.bcx.ins().uextend(types::I128, res)
 707             } else {
 708                 fx.bcx.ins().clz(arg)
 709             };
 710             let res = CValue::by_val(res, fx.layout_of(T));
 711             ret.write_cvalue(fx, res);
 712         };
 713         cttz | cttz_nonzero, <T> (v arg) {
 714             // FIXME trap on `cttz_nonzero` with zero arg.
 715             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 716                 // FIXME verify this algorithm is correct
 717                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 718                 let lsb_tz = fx.bcx.ins().ctz(lsb);
 719                 let msb_tz = fx.bcx.ins().ctz(msb);
 720                 let lsb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, lsb, 0);
 721                 let msb_tz_plus_64 = fx.bcx.ins().iadd_imm(msb_tz, 64);
 722                 let res = fx.bcx.ins().select(lsb_is_zero, msb_tz_plus_64, lsb_tz);
 723                 fx.bcx.ins().uextend(types::I128, res)
 724             } else {
 725                 fx.bcx.ins().ctz(arg)
 726             };
 727             let res = CValue::by_val(res, fx.layout_of(T));
 728             ret.write_cvalue(fx, res);
 729         };
 730         ctpop, <T> (v arg) {
 731             let res = fx.bcx.ins().popcnt(arg);
 732             let res = CValue::by_val(res, fx.layout_of(T));
 733             ret.write_cvalue(fx, res);
 734         };
 735         bitreverse, <T> (v arg) {
 736             let res = fx.bcx.ins().bitrev(arg);
 737             let res = CValue::by_val(res, fx.layout_of(T));
 738             ret.write_cvalue(fx, res);
 739         };
 740         bswap, <T> (v arg) {
 741             // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
 742             fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value {
 743                 match bcx.func.dfg.value_type(v) {
 744                     types::I8 => v,
 745
 746                     // https://code.woboq.org/gcc/include/bits/byteswap.h.html
 747                     types::I16 => {
 748                         let tmp1 = bcx.ins().ishl_imm(v, 8);
 749                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00);
 750
 751                         let tmp2 = bcx.ins().ushr_imm(v, 8);
 752                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF);
 753
 754                         bcx.ins().bor(n1, n2)
 755                     }
 756                     types::I32 => {
 757                         let tmp1 = bcx.ins().ishl_imm(v, 24);
 758                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000);
 759
 760                         let tmp2 = bcx.ins().ishl_imm(v, 8);
 761                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000);
 762
 763                         let tmp3 = bcx.ins().ushr_imm(v, 8);
 764                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00);
 765
 766                         let tmp4 = bcx.ins().ushr_imm(v, 24);
 767                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF);
 768
 769                         let or_tmp1 = bcx.ins().bor(n1, n2);
 770                         let or_tmp2 = bcx.ins().bor(n3, n4);
 771                         bcx.ins().bor(or_tmp1, or_tmp2)
 772                     }
 773                     types::I64 => {
 774                         let tmp1 = bcx.ins().ishl_imm(v, 56);
 775                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64);
 776
 777                         let tmp2 = bcx.ins().ishl_imm(v, 40);
 778                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64);
 779
 780                         let tmp3 = bcx.ins().ishl_imm(v, 24);
 781                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64);
 782
 783                         let tmp4 = bcx.ins().ishl_imm(v, 8);
 784                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64);
 785
 786                         let tmp5 = bcx.ins().ushr_imm(v, 8);
 787                         let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64);
 788
 789                         let tmp6 = bcx.ins().ushr_imm(v, 24);
 790                         let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64);
 791
 792                         let tmp7 = bcx.ins().ushr_imm(v, 40);
 793                         let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64);
 794
 795                         let tmp8 = bcx.ins().ushr_imm(v, 56);
 796                         let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64);
 797
 798                         let or_tmp1 = bcx.ins().bor(n1, n2);
 799                         let or_tmp2 = bcx.ins().bor(n3, n4);
 800                         let or_tmp3 = bcx.ins().bor(n5, n6);
 801                         let or_tmp4 = bcx.ins().bor(n7, n8);
 802
 803                         let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2);
 804                         let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4);
 805                         bcx.ins().bor(or_tmp5, or_tmp6)
 806                     }
 807                     types::I128 => {
 808                         let (lo, hi) = bcx.ins().isplit(v);
 809                         let lo = swap(bcx, lo);
 810                         let hi = swap(bcx, hi);
 811                         bcx.ins().iconcat(hi, lo)
 812                     }
 813                     ty => unreachable!("bswap {}", ty),
 814                 }
 815             };
 816             let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T));
 817             ret.write_cvalue(fx, res);
 818         };
 819         assert_inhabited | assert_zero_valid | assert_uninit_valid, <T> () {
 820             let layout = fx.layout_of(T);
 821             if layout.abi.is_uninhabited() {
 822                 crate::base::codegen_panic(
 823                     fx,
 824                     &format!("attempted to instantiate uninhabited type `{}`", T),
 825                     span,
 826                 );
 827                 return;
 828             }
 829
 830             if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() {
 831                 crate::base::codegen_panic(
 832                     fx,
 833                     &format!("attempted to zero-initialize type `{}`, which is invalid", T),
 834                     span,
 835                 );
 836                 return;
 837             }
 838
 839             if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() {
 840                 crate::base::codegen_panic(
 841                     fx,
 842                     &format!("attempted to leave type `{}` uninitialized, which is invalid", T),
 843                     span,
 844                 );
 845                 return;
 846             }
 847         };
 848
 849         volatile_load | unaligned_volatile_load, (c ptr) {
 850             // Cranelift treats loads as volatile by default
 851             // FIXME ignore during stack2reg optimization
 852             // FIXME correctly handle unaligned_volatile_load
 853             let inner_layout =
 854                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 855             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 856             ret.write_cvalue(fx, val);
 857         };
 858         volatile_store | unaligned_volatile_store, (v ptr, c val) {
 859             // Cranelift treats stores as volatile by default
 860             // FIXME ignore during stack2reg optimization
 861             // FIXME correctly handle unaligned_volatile_store
 862             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 863             dest.write_cvalue(fx, val);
 864         };
 865
 866         size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () {
 867             let const_val =
 868                 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
 869             let val = crate::constant::trans_const_value(
 870                 fx,
 871                 const_val,
 872                 ret.layout().ty,
 873             );
 874             ret.write_cvalue(fx, val);
 875         };
 876
 877         ptr_offset_from, <T> (v ptr, v base) {
 878             let isize_layout = fx.layout_of(fx.tcx.types.isize);
 879
 880             let pointee_size: u64 = fx.layout_of(T).size.bytes();
 881             let diff = fx.bcx.ins().isub(ptr, base);
 882             // FIXME this can be an exact division.
 883             let val = CValue::by_val(fx.bcx.ins().sdiv_imm(diff, pointee_size as i64), isize_layout);
 884             ret.write_cvalue(fx, val);
 885         };
 886
 887         ptr_guaranteed_eq, (c a, c b) {
 888             let val = crate::num::trans_ptr_binop(fx, BinOp::Eq, a, b);
 889             ret.write_cvalue(fx, val);
 890         };
 891
 892         ptr_guaranteed_ne, (c a, c b) {
 893             let val = crate::num::trans_ptr_binop(fx, BinOp::Ne, a, b);
 894             ret.write_cvalue(fx, val);
 895         };
 896
 897         caller_location, () {
 898             let caller_location = fx.get_caller_location(span);
 899             ret.write_cvalue(fx, caller_location);
 900         };
 901
 902         _ if intrinsic.starts_with("atomic_fence"), () {
 903             crate::atomic_shim::lock_global_lock(fx);
 904             crate::atomic_shim::unlock_global_lock(fx);
 905         };
 906         _ if intrinsic.starts_with("atomic_singlethreadfence"), () {
 907             crate::atomic_shim::lock_global_lock(fx);
 908             crate::atomic_shim::unlock_global_lock(fx);
 909         };
 910         _ if intrinsic.starts_with("atomic_load"), (c ptr) {
 911             crate::atomic_shim::lock_global_lock(fx);
 912
 913             let inner_layout =
 914                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 915             validate_atomic_type!(fx, intrinsic, span, inner_layout.ty);
 916             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 917             ret.write_cvalue(fx, val);
 918
 919             crate::atomic_shim::unlock_global_lock(fx);
 920         };
 921         _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
 922             validate_atomic_type!(fx, intrinsic, span, val.layout().ty);
 923
 924             crate::atomic_shim::lock_global_lock(fx);
 925
 926             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 927             dest.write_cvalue(fx, val);
 928
 929             crate::atomic_shim::unlock_global_lock(fx);
 930         };
 931         _ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
 932             validate_atomic_type!(fx, intrinsic, span, T);
 933
 934             crate::atomic_shim::lock_global_lock(fx);
 935
 936             // Read old
 937             let clif_ty = fx.clif_type(T).unwrap();
 938             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 939             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
 940
 941             // Write new
 942             let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout());
 943             dest.write_cvalue(fx, src);
 944
 945             crate::atomic_shim::unlock_global_lock(fx);
 946         };
 947         _ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
 948             validate_atomic_type!(fx, intrinsic, span, T);
 949
 950             let test_old = test_old.load_scalar(fx);
 951             let new = new.load_scalar(fx);
 952
 953             crate::atomic_shim::lock_global_lock(fx);
 954
 955             // Read old
 956             let clif_ty = fx.clif_type(T).unwrap();
 957             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 958
 959             // Compare
 960             let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
 961             let new = fx.bcx.ins().select(is_eq, new, old); // Keep old if not equal to test_old
 962
 963             // Write new
 964             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 965
 966             let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
 967             ret.write_cvalue(fx, ret_val);
 968
 969             crate::atomic_shim::unlock_global_lock(fx);
 970         };
 971
 972         _ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, c amount) {
 973             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 974             let amount = amount.load_scalar(fx);
 975             atomic_binop_return_old! (fx, iadd<T>(ptr, amount) -> ret);
 976         };
 977         _ if intrinsic.starts_with("atomic_xsub"), <T> (v ptr, c amount) {
 978             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 979             let amount = amount.load_scalar(fx);
 980             atomic_binop_return_old! (fx, isub<T>(ptr, amount) -> ret);
 981         };
 982         _ if intrinsic.starts_with("atomic_and"), <T> (v ptr, c src) {
 983             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 984             let src = src.load_scalar(fx);
 985             atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
 986         };
 987         _ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, c src) {
 988             validate_atomic_type!(fx, intrinsic, span, T);
 989
 990             let src = src.load_scalar(fx);
 991
 992             crate::atomic_shim::lock_global_lock(fx);
 993
 994             let clif_ty = fx.clif_type(T).unwrap();
 995             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 996             let and = fx.bcx.ins().band(old, src);
 997             let new = fx.bcx.ins().bnot(and);
 998             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 999             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
1000
1001             crate::atomic_shim::unlock_global_lock(fx);
1002         };
1003         _ if intrinsic.starts_with("atomic_or"), <T> (v ptr, c src) {
1004             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1005             let src = src.load_scalar(fx);
1006             atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);
1007         };
1008         _ if intrinsic.starts_with("atomic_xor"), <T> (v ptr, c src) {
1009             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1010             let src = src.load_scalar(fx);
1011             atomic_binop_return_old! (fx, bxor<T>(ptr, src) -> ret);
1012         };
1013
1014         _ if intrinsic.starts_with("atomic_max"), <T> (v ptr, c src) {
1015             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1016             let src = src.load_scalar(fx);
1017             atomic_minmax!(fx, IntCC::SignedGreaterThan, <T> (ptr, src) -> ret);
1018         };
1019         _ if intrinsic.starts_with("atomic_umax"), <T> (v ptr, c src) {
1020             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1021             let src = src.load_scalar(fx);
1022             atomic_minmax!(fx, IntCC::UnsignedGreaterThan, <T> (ptr, src) -> ret);
1023         };
1024         _ if intrinsic.starts_with("atomic_min"), <T> (v ptr, c src) {
1025             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1026             let src = src.load_scalar(fx);
1027             atomic_minmax!(fx, IntCC::SignedLessThan, <T> (ptr, src) -> ret);
1028         };
1029         _ if intrinsic.starts_with("atomic_umin"), <T> (v ptr, c src) {
1030             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1031             let src = src.load_scalar(fx);
1032             atomic_minmax!(fx, IntCC::UnsignedLessThan, <T> (ptr, src) -> ret);
1033         };
1034
1035         minnumf32, (v a, v b) {
1036             let val = fx.bcx.ins().fmin(a, b);
1037             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1038             ret.write_cvalue(fx, val);
1039         };
1040         minnumf64, (v a, v b) {
1041             let val = fx.bcx.ins().fmin(a, b);
1042             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1043             ret.write_cvalue(fx, val);
1044         };
1045         maxnumf32, (v a, v b) {
1046             let val = fx.bcx.ins().fmax(a, b);
1047             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1048             ret.write_cvalue(fx, val);
1049         };
1050         maxnumf64, (v a, v b) {
1051             let val = fx.bcx.ins().fmax(a, b);
1052             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1053             ret.write_cvalue(fx, val);
1054         };
1055
1056         try, (v f, v data, v _catch_fn) {
1057             // FIXME once unwinding is supported, change this to actually catch panics
1058             let f_sig = fx.bcx.func.import_signature(Signature {
1059                 call_conv: CallConv::triple_default(fx.triple()),
1060                 params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
1061                 returns: vec![],
1062             });
1063
1064             fx.bcx.ins().call_indirect(f_sig, f, &[data]);
1065
1066             let ret_val = CValue::const_val(fx, ret.layout(), 0);
1067             ret.write_cvalue(fx, ret_val);
1068         };
1069
1070         fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) {
1071             let res = crate::num::trans_float_binop(fx, match intrinsic {
1072                 "fadd_fast" => BinOp::Add,
1073                 "fsub_fast" => BinOp::Sub,
1074                 "fmul_fast" => BinOp::Mul,
1075                 "fdiv_fast" => BinOp::Div,
1076                 "frem_fast" => BinOp::Rem,
1077                 _ => unreachable!(),
1078             }, x, y);
1079             ret.write_cvalue(fx, res);
1080         };
1081         float_to_int_unchecked, (v f) {
1082             let res = crate::cast::clif_int_or_float_cast(
1083                 fx,
1084                 f,
1085                 false,
1086                 fx.clif_type(ret.layout().ty).unwrap(),
1087                 type_sign(ret.layout().ty),
1088             );
1089             ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
1090         };
1091     }
1092
1093     if let Some((_, dest)) = destination {
1094         let ret_block = fx.get_block(dest);
1095         fx.bcx.ins().jump(ret_block, &[]);
1096     } else {
1097         trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
1098     }
1099 }