src/intrinsics/mod.rs

   1 mod llvm;
   2 mod simd;
   3
   4 pub(crate) use llvm::codegen_llvm_intrinsic_call;
   5
   6 use crate::prelude::*;
   7
   8 macro intrinsic_pat {
   9     (_) => {
  10         _
  11     },
  12     ($name:ident) => {
  13         stringify!($name)
  14     },
  15     ($name:literal) => {
  16         stringify!($name)
  17     },
  18     ($x:ident . $($xs:tt).*) => {
  19         concat!(stringify!($x), ".", intrinsic_pat!($($xs).*))
  20     }
  21 }
  22
  23 macro intrinsic_arg {
  24     (o $fx:expr, $arg:ident) => {
  25         $arg
  26     },
  27     (c $fx:expr, $arg:ident) => {
  28         trans_operand($fx, $arg)
  29     },
  30     (v $fx:expr, $arg:ident) => {
  31         trans_operand($fx, $arg).load_scalar($fx)
  32     }
  33 }
  34
  35 macro intrinsic_substs {
  36     ($substs:expr, $index:expr,) => {},
  37     ($substs:expr, $index:expr, $first:ident $(,$rest:ident)*) => {
  38         let $first = $substs.type_at($index);
  39         intrinsic_substs!($substs, $index+1, $($rest),*);
  40     }
  41 }
  42
  43 macro intrinsic_match {
  44     ($fx:expr, $intrinsic:expr, $substs:expr, $args:expr,
  45     _ => $unknown:block;
  46     $(
  47         $($($name:tt).*)|+ $(if $cond:expr)?, $(<$($subst:ident),*>)? ($($a:ident $arg:ident),*) $content:block;
  48     )*) => {
  49         let _ = $substs; // Silence warning when substs is unused.
  50         match $intrinsic {
  51             $(
  52                 $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
  53                     #[allow(unused_parens, non_snake_case)]
  54                     {
  55                         $(
  56                             intrinsic_substs!($substs, 0, $($subst),*);
  57                         )?
  58                         if let [$($arg),*] = $args {
  59                             let ($($arg,)*) = (
  60                                 $(intrinsic_arg!($a $fx, $arg),)*
  61                             );
  62                             #[warn(unused_parens, non_snake_case)]
  63                             {
  64                                 $content
  65                             }
  66                         } else {
  67                             bug!("wrong number of args for intrinsic {:?}", $intrinsic);
  68                         }
  69                     }
  70                 }
  71             )*
  72             _ => $unknown,
  73         }
  74     }
  75 }
  76
  77 macro call_intrinsic_match {
  78     ($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $destination:expr, $args:expr, $(
  79         $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
  80     )*) => {
  81         match $intrinsic {
  82             $(
  83                 stringify!($name) => {
  84                     assert!($substs.is_noop());
  85                     if let [$(ref $arg),*] = *$args {
  86                         let ($($arg,)*) = (
  87                             $(trans_operand($fx, $arg),)*
  88                         );
  89                         let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty);
  90                         $ret.write_cvalue($fx, res);
  91
  92                         if let Some((_, dest)) = $destination {
  93                             let ret_block = $fx.get_block(dest);
  94                             $fx.bcx.ins().jump(ret_block, &[]);
  95                             return;
  96                         } else {
  97                             unreachable!();
  98                         }
  99                     } else {
 100                         bug!("wrong number of args for intrinsic {:?}", $intrinsic);
 101                     }
 102                 }
 103             )*
 104             _ => {}
 105         }
 106     }
 107 }
 108
 109 macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident)  {
 110     crate::atomic_shim::lock_global_lock($fx);
 111
 112     let clif_ty = $fx.clif_type($T).unwrap();
 113     let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 114     let new = $fx.bcx.ins().$op(old, $src);
 115     $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 116     $ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
 117
 118     crate::atomic_shim::unlock_global_lock($fx);
 119 }
 120
 121 macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) {
 122     crate::atomic_shim::lock_global_lock($fx);
 123
 124     // Read old
 125     let clif_ty = $fx.clif_type($T).unwrap();
 126     let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 127
 128     // Compare
 129     let is_eq = $fx.bcx.ins().icmp(IntCC::SignedGreaterThan, old, $src);
 130     let new = $fx.bcx.ins().select(is_eq, old, $src);
 131
 132     // Write new
 133     $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 134
 135     let ret_val = CValue::by_val(old, $ret.layout());
 136     $ret.write_cvalue($fx, ret_val);
 137
 138     crate::atomic_shim::unlock_global_lock($fx);
 139 }
 140
 141 macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
 142     match $ty.kind {
 143         ty::Uint(_) | ty::Int(_) => {}
 144         _ => {
 145             $fx.tcx.sess.span_err($span, &format!("`{}` intrinsic: expected basic integer type, found `{:?}`", $intrinsic, $ty));
 146             // Prevent verifier error
 147             crate::trap::trap_unreachable($fx, "compilation should not have succeeded");
 148             return;
 149         }
 150     }
 151 }
 152
 153 macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
 154     if !$ty.is_simd() {
 155         $fx.tcx.sess.span_err($span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", $intrinsic, $ty));
 156         // Prevent verifier error
 157         crate::trap::trap_unreachable($fx, "compilation should not have succeeded");
 158         return;
 159     }
 160 }
 161
 162 fn lane_type_and_count<'tcx>(
 163     tcx: TyCtxt<'tcx>,
 164     layout: TyAndLayout<'tcx>,
 165 ) -> (TyAndLayout<'tcx>, u16) {
 166     assert!(layout.ty.is_simd());
 167     let lane_count = match layout.fields {
 168         rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(),
 169         _ => unreachable!("lane_type_and_count({:?})", layout),
 170     };
 171     let lane_layout = layout.field(&ty::layout::LayoutCx {
 172         tcx,
 173         param_env: ParamEnv::reveal_all(),
 174     }, 0).unwrap();
 175     (lane_layout, lane_count)
 176 }
 177
 178 fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
 179     let (element, count) = match &layout.abi {
 180         Abi::Vector { element, count } => (element.clone(), *count),
 181         _ => unreachable!(),
 182     };
 183
 184     match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
 185         // Cranelift currently only implements icmp for 128bit vectors. While 64bit lanes are
 186         // supported, this needs either the `use_sse41_simd` or `use_sse42_simd` target flag
 187         // to be enabled.
 188         Some(vector_ty) if vector_ty.bits() == 128 && vector_ty.lane_type() != types::I64 => Some(vector_ty),
 189         _ => None,
 190     }
 191 }
 192
 193 fn simd_for_each_lane<'tcx, B: Backend>(
 194     fx: &mut FunctionCx<'_, 'tcx, B>,
 195     val: CValue<'tcx>,
 196     ret: CPlace<'tcx>,
 197     f: impl Fn(
 198         &mut FunctionCx<'_, 'tcx, B>,
 199         TyAndLayout<'tcx>,
 200         TyAndLayout<'tcx>,
 201         Value,
 202     ) -> CValue<'tcx>,
 203 ) {
 204     let layout = val.layout();
 205
 206     let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
 207     let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
 208     assert_eq!(lane_count, ret_lane_count);
 209
 210     for lane_idx in 0..lane_count {
 211         let lane_idx = mir::Field::new(lane_idx.try_into().unwrap());
 212         let lane = val.value_field(fx, lane_idx).load_scalar(fx);
 213
 214         let res_lane = f(fx, lane_layout, ret_lane_layout, lane);
 215
 216         ret.place_field(fx, lane_idx).write_cvalue(fx, res_lane);
 217     }
 218 }
 219
 220 fn simd_pair_for_each_lane<'tcx, B: Backend>(
 221     fx: &mut FunctionCx<'_, 'tcx, B>,
 222     x: CValue<'tcx>,
 223     y: CValue<'tcx>,
 224     ret: CPlace<'tcx>,
 225     f: impl Fn(
 226         &mut FunctionCx<'_, 'tcx, B>,
 227         TyAndLayout<'tcx>,
 228         TyAndLayout<'tcx>,
 229         Value,
 230         Value,
 231     ) -> CValue<'tcx>,
 232 ) {
 233     assert_eq!(x.layout(), y.layout());
 234     let layout = x.layout();
 235
 236     let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
 237     let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
 238     assert_eq!(lane_count, ret_lane_count);
 239
 240     for lane in 0..lane_count {
 241         let lane = mir::Field::new(lane.try_into().unwrap());
 242         let x_lane = x.value_field(fx, lane).load_scalar(fx);
 243         let y_lane = y.value_field(fx, lane).load_scalar(fx);
 244
 245         let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane);
 246
 247         ret.place_field(fx, lane).write_cvalue(fx, res_lane);
 248     }
 249 }
 250
 251 fn bool_to_zero_or_max_uint<'tcx>(
 252     fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
 253     layout: TyAndLayout<'tcx>,
 254     val: Value,
 255 ) -> CValue<'tcx> {
 256     let ty = fx.clif_type(layout.ty).unwrap();
 257
 258     let int_ty = match ty {
 259         types::F32 => types::I32,
 260         types::F64 => types::I64,
 261         ty => ty,
 262     };
 263
 264     let val = fx.bcx.ins().bint(int_ty, val);
 265     let mut res = fx.bcx.ins().ineg(val);
 266
 267     if ty.is_float() {
 268         res = fx.bcx.ins().bitcast(ty, res);
 269     }
 270
 271     CValue::by_val(res, layout)
 272 }
 273
 274 macro simd_cmp {
 275     ($fx:expr, $cc:ident($x:ident, $y:ident) -> $ret:ident) => {
 276         let vector_ty = clif_vector_type($fx.tcx, $x.layout());
 277
 278         if let Some(vector_ty) = vector_ty {
 279             let x = $x.load_scalar($fx);
 280             let y = $y.load_scalar($fx);
 281             let val = $fx.bcx.ins().icmp(IntCC::$cc, x, y);
 282
 283             // HACK This depends on the fact that icmp for vectors represents bools as 0 and !0, not 0 and 1.
 284             let val = $fx.bcx.ins().raw_bitcast(vector_ty, val);
 285
 286             $ret.write_cvalue($fx, CValue::by_val(val, $ret.layout()));
 287         } else {
 288             simd_pair_for_each_lane(
 289                 $fx,
 290                 $x,
 291                 $y,
 292                 $ret,
 293                 |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 294                     let res_lane = match lane_layout.ty.kind {
 295                         ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane),
 296                         _ => unreachable!("{:?}", lane_layout.ty),
 297                     };
 298                     bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 299                 },
 300             );
 301         }
 302     },
 303     ($fx:expr, $cc_u:ident|$cc_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 304         // FIXME use vector icmp when possible
 305         simd_pair_for_each_lane(
 306             $fx,
 307             $x,
 308             $y,
 309             $ret,
 310             |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 311                 let res_lane = match lane_layout.ty.kind {
 312                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
 313                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
 314                     _ => unreachable!("{:?}", lane_layout.ty),
 315                 };
 316                 bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 317             },
 318         );
 319     },
 320 }
 321
 322 macro simd_int_binop {
 323     ($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
 324         simd_int_binop!($fx, $op|$op($x, $y) -> $ret);
 325     },
 326     ($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 327         simd_pair_for_each_lane(
 328             $fx,
 329             $x,
 330             $y,
 331             $ret,
 332             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 333                 let res_lane = match lane_layout.ty.kind {
 334                     ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 335                     ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 336                     _ => unreachable!("{:?}", lane_layout.ty),
 337                 };
 338                 CValue::by_val(res_lane, ret_lane_layout)
 339             },
 340         );
 341     },
 342 }
 343
 344 macro simd_int_flt_binop {
 345     ($fx:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 346         simd_int_flt_binop!($fx, $op|$op|$op_f($x, $y) -> $ret);
 347     },
 348     ($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 349         simd_pair_for_each_lane(
 350             $fx,
 351             $x,
 352             $y,
 353             $ret,
 354             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 355                 let res_lane = match lane_layout.ty.kind {
 356                     ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 357                     ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 358                     ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
 359                     _ => unreachable!("{:?}", lane_layout.ty),
 360                 };
 361                 CValue::by_val(res_lane, ret_lane_layout)
 362             },
 363         );
 364     },
 365 }
 366
 367 macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
 368     simd_pair_for_each_lane(
 369         $fx,
 370         $x,
 371         $y,
 372         $ret,
 373         |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 374             let res_lane = match lane_layout.ty.kind {
 375                 ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
 376                 _ => unreachable!("{:?}", lane_layout.ty),
 377             };
 378             CValue::by_val(res_lane, ret_lane_layout)
 379         },
 380     );
 381 }
 382
 383 pub(crate) fn codegen_intrinsic_call<'tcx>(
 384     fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
 385     instance: Instance<'tcx>,
 386     args: &[mir::Operand<'tcx>],
 387     destination: Option<(CPlace<'tcx>, BasicBlock)>,
 388     span: Span,
 389 ) {
 390     let def_id = instance.def_id();
 391     let substs = instance.substs;
 392
 393     let intrinsic = fx.tcx.item_name(def_id).as_str();
 394     let intrinsic = &intrinsic[..];
 395
 396     let ret = match destination {
 397         Some((place, _)) => place,
 398         None => {
 399             // Insert non returning intrinsics here
 400             match intrinsic {
 401                 "abort" => {
 402                     trap_panic(fx, "Called intrinsic::abort.");
 403                 }
 404                 "unreachable" => {
 405                     trap_unreachable(fx, "[corruption] Called intrinsic::unreachable.");
 406                 }
 407                 "transmute" => {
 408                     trap_unreachable(
 409                         fx,
 410                         "[corruption] Transmuting to uninhabited type.",
 411                     );
 412                 }
 413                 _ => unimplemented!("unsupported instrinsic {}", intrinsic),
 414             }
 415             return;
 416         }
 417     };
 418
 419     if intrinsic.starts_with("simd_") {
 420         self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span);
 421         let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
 422         fx.bcx.ins().jump(ret_block, &[]);
 423         return;
 424     }
 425
 426     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 427
 428     call_intrinsic_match! {
 429         fx, intrinsic, substs, ret, destination, args,
 430         expf32(flt) -> f32 => expf,
 431         expf64(flt) -> f64 => exp,
 432         exp2f32(flt) -> f32 => exp2f,
 433         exp2f64(flt) -> f64 => exp2,
 434         sqrtf32(flt) -> f32 => sqrtf,
 435         sqrtf64(flt) -> f64 => sqrt,
 436         powif32(a, x) -> f32 => __powisf2, // compiler-builtins
 437         powif64(a, x) -> f64 => __powidf2, // compiler-builtins
 438         powf32(a, x) -> f32 => powf,
 439         powf64(a, x) -> f64 => pow,
 440         logf32(flt) -> f32 => logf,
 441         logf64(flt) -> f64 => log,
 442         log2f32(flt) -> f32 => log2f,
 443         log2f64(flt) -> f64 => log2,
 444         log10f32(flt) -> f32 => log10f,
 445         log10f64(flt) -> f64 => log10,
 446         fabsf32(flt) -> f32 => fabsf,
 447         fabsf64(flt) -> f64 => fabs,
 448         fmaf32(x, y, z) -> f32 => fmaf,
 449         fmaf64(x, y, z) -> f64 => fma,
 450         copysignf32(x, y) -> f32 => copysignf,
 451         copysignf64(x, y) -> f64 => copysign,
 452
 453         // rounding variants
 454         // FIXME use clif insts
 455         floorf32(flt) -> f32 => floorf,
 456         floorf64(flt) -> f64 => floor,
 457         ceilf32(flt) -> f32 => ceilf,
 458         ceilf64(flt) -> f64 => ceil,
 459         truncf32(flt) -> f32 => truncf,
 460         truncf64(flt) -> f64 => trunc,
 461         roundf32(flt) -> f32 => roundf,
 462         roundf64(flt) -> f64 => round,
 463
 464         // trigonometry
 465         sinf32(flt) -> f32 => sinf,
 466         sinf64(flt) -> f64 => sin,
 467         cosf32(flt) -> f32 => cosf,
 468         cosf64(flt) -> f64 => cos,
 469         tanf32(flt) -> f32 => tanf,
 470         tanf64(flt) -> f64 => tan,
 471     }
 472
 473     intrinsic_match! {
 474         fx, intrinsic, substs, args,
 475         _ => {
 476             fx.tcx.sess.span_fatal(span, &format!("unsupported intrinsic {}", intrinsic));
 477         };
 478
 479         assume, (c _a) {};
 480         likely | unlikely, (c a) {
 481             ret.write_cvalue(fx, a);
 482         };
 483         breakpoint, () {
 484             fx.bcx.ins().debugtrap();
 485         };
 486         copy | copy_nonoverlapping, <elem_ty> (v src, v dst, v count) {
 487             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 488             let elem_size = fx
 489                 .bcx
 490                 .ins()
 491                 .iconst(fx.pointer_type, elem_size as i64);
 492             assert_eq!(args.len(), 3);
 493             let byte_amount = fx.bcx.ins().imul(count, elem_size);
 494
 495             if intrinsic.contains("nonoverlapping") {
 496                 // FIXME emit_small_memcpy
 497                 fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
 498             } else {
 499                 // FIXME emit_small_memmove
 500                 fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
 501             }
 502         };
 503         // NOTE: the volatile variants have src and dst swapped
 504         volatile_copy_memory | volatile_copy_nonoverlapping_memory, <elem_ty> (v dst, v src, v count) {
 505             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 506             let elem_size = fx
 507                 .bcx
 508                 .ins()
 509                 .iconst(fx.pointer_type, elem_size as i64);
 510             assert_eq!(args.len(), 3);
 511             let byte_amount = fx.bcx.ins().imul(count, elem_size);
 512
 513             // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
 514             if intrinsic.contains("nonoverlapping") {
 515                 // FIXME emit_small_memcpy
 516                 fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
 517             } else {
 518                 // FIXME emit_small_memmove
 519                 fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
 520             }
 521         };
 522         discriminant_value, (c ptr) {
 523             let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 524             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout);
 525             let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout());
 526             ret.write_cvalue(fx, discr);
 527         };
 528         size_of_val, <T> (c ptr) {
 529             let layout = fx.layout_of(T);
 530             let size = if layout.is_unsized() {
 531                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 532                 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 533                 size
 534             } else {
 535                 fx
 536                     .bcx
 537                     .ins()
 538                     .iconst(fx.pointer_type, layout.size.bytes() as i64)
 539             };
 540             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 541         };
 542         min_align_of_val, <T> (c ptr) {
 543             let layout = fx.layout_of(T);
 544             let align = if layout.is_unsized() {
 545                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 546                 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 547                 align
 548             } else {
 549                 fx
 550                     .bcx
 551                     .ins()
 552                     .iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
 553             };
 554             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 555         };
 556
 557         _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) {
 558             // FIXME trap on overflow
 559             let bin_op = match intrinsic {
 560                 "unchecked_add" => BinOp::Add,
 561                 "unchecked_sub" => BinOp::Sub,
 562                 "unchecked_div" | "exact_div" => BinOp::Div,
 563                 "unchecked_rem" => BinOp::Rem,
 564                 "unchecked_shl" => BinOp::Shl,
 565                 "unchecked_shr" => BinOp::Shr,
 566                 _ => unreachable!("intrinsic {}", intrinsic),
 567             };
 568             let res = crate::num::trans_int_binop(fx, bin_op, x, y);
 569             ret.write_cvalue(fx, res);
 570         };
 571         _ if intrinsic.ends_with("_with_overflow"), (c x, c y) {
 572             assert_eq!(x.layout().ty, y.layout().ty);
 573             let bin_op = match intrinsic {
 574                 "add_with_overflow" => BinOp::Add,
 575                 "sub_with_overflow" => BinOp::Sub,
 576                 "mul_with_overflow" => BinOp::Mul,
 577                 _ => unreachable!("intrinsic {}", intrinsic),
 578             };
 579
 580             let res = crate::num::trans_checked_int_binop(
 581                 fx,
 582                 bin_op,
 583                 x,
 584                 y,
 585             );
 586             ret.write_cvalue(fx, res);
 587         };
 588         _ if intrinsic.starts_with("wrapping_"), (c x, c y) {
 589             assert_eq!(x.layout().ty, y.layout().ty);
 590             let bin_op = match intrinsic {
 591                 "wrapping_add" => BinOp::Add,
 592                 "wrapping_sub" => BinOp::Sub,
 593                 "wrapping_mul" => BinOp::Mul,
 594                 _ => unreachable!("intrinsic {}", intrinsic),
 595             };
 596             let res = crate::num::trans_int_binop(
 597                 fx,
 598                 bin_op,
 599                 x,
 600                 y,
 601             );
 602             ret.write_cvalue(fx, res);
 603         };
 604         _ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) {
 605             assert_eq!(lhs.layout().ty, rhs.layout().ty);
 606             let bin_op = match intrinsic {
 607                 "saturating_add" => BinOp::Add,
 608                 "saturating_sub" => BinOp::Sub,
 609                 _ => unreachable!("intrinsic {}", intrinsic),
 610             };
 611
 612             let signed = type_sign(T);
 613
 614             let checked_res = crate::num::trans_checked_int_binop(
 615                 fx,
 616                 bin_op,
 617                 lhs,
 618                 rhs,
 619             );
 620
 621             let (val, has_overflow) = checked_res.load_scalar_pair(fx);
 622             let clif_ty = fx.clif_type(T).unwrap();
 623
 624             // `select.i8` is not implemented by Cranelift.
 625             let has_overflow = fx.bcx.ins().uextend(types::I32, has_overflow);
 626
 627             let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
 628
 629             let val = match (intrinsic, signed) {
 630                 ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val),
 631                 ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val),
 632                 ("saturating_add", true) => {
 633                     let rhs = rhs.load_scalar(fx);
 634                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 635                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
 636                     fx.bcx.ins().select(has_overflow, sat_val, val)
 637                 }
 638                 ("saturating_sub", true) => {
 639                     let rhs = rhs.load_scalar(fx);
 640                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 641                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
 642                     fx.bcx.ins().select(has_overflow, sat_val, val)
 643                 }
 644                 _ => unreachable!(),
 645             };
 646
 647             let res = CValue::by_val(val, fx.layout_of(T));
 648
 649             ret.write_cvalue(fx, res);
 650         };
 651         rotate_left, <T>(v x, v y) {
 652             let layout = fx.layout_of(T);
 653             let res = fx.bcx.ins().rotl(x, y);
 654             ret.write_cvalue(fx, CValue::by_val(res, layout));
 655         };
 656         rotate_right, <T>(v x, v y) {
 657             let layout = fx.layout_of(T);
 658             let res = fx.bcx.ins().rotr(x, y);
 659             ret.write_cvalue(fx, CValue::by_val(res, layout));
 660         };
 661
 662         // The only difference between offset and arith_offset is regarding UB. Because Cranelift
 663         // doesn't have UB both are codegen'ed the same way
 664         offset | arith_offset, (c base, v offset) {
 665             let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
 666             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 667             let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
 668             let base_val = base.load_scalar(fx);
 669             let res = fx.bcx.ins().iadd(base_val, ptr_diff);
 670             ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
 671         };
 672
 673         transmute, (c from) {
 674             ret.write_cvalue_transmute(fx, from);
 675         };
 676         write_bytes | volatile_set_memory, (c dst, v val, v count) {
 677             let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
 678             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 679             let count = fx.bcx.ins().imul_imm(count, pointee_size as i64);
 680             let dst_ptr = dst.load_scalar(fx);
 681             // FIXME make the memset actually volatile when switching to emit_small_memset
 682             // FIXME use emit_small_memset
 683             fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count);
 684         };
 685         ctlz | ctlz_nonzero, <T> (v arg) {
 686             // FIXME trap on `ctlz_nonzero` with zero arg.
 687             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 688                 // FIXME verify this algorithm is correct
 689                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 690                 let lsb_lz = fx.bcx.ins().clz(lsb);
 691                 let msb_lz = fx.bcx.ins().clz(msb);
 692                 let msb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, msb, 0);
 693                 let lsb_lz_plus_64 = fx.bcx.ins().iadd_imm(lsb_lz, 64);
 694                 let res = fx.bcx.ins().select(msb_is_zero, lsb_lz_plus_64, msb_lz);
 695                 fx.bcx.ins().uextend(types::I128, res)
 696             } else {
 697                 fx.bcx.ins().clz(arg)
 698             };
 699             let res = CValue::by_val(res, fx.layout_of(T));
 700             ret.write_cvalue(fx, res);
 701         };
 702         cttz | cttz_nonzero, <T> (v arg) {
 703             // FIXME trap on `cttz_nonzero` with zero arg.
 704             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 705                 // FIXME verify this algorithm is correct
 706                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 707                 let lsb_tz = fx.bcx.ins().ctz(lsb);
 708                 let msb_tz = fx.bcx.ins().ctz(msb);
 709                 let lsb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, lsb, 0);
 710                 let msb_tz_plus_64 = fx.bcx.ins().iadd_imm(msb_tz, 64);
 711                 let res = fx.bcx.ins().select(lsb_is_zero, msb_tz_plus_64, lsb_tz);
 712                 fx.bcx.ins().uextend(types::I128, res)
 713             } else {
 714                 fx.bcx.ins().ctz(arg)
 715             };
 716             let res = CValue::by_val(res, fx.layout_of(T));
 717             ret.write_cvalue(fx, res);
 718         };
 719         ctpop, <T> (v arg) {
 720             let res = fx.bcx.ins().popcnt(arg);
 721             let res = CValue::by_val(res, fx.layout_of(T));
 722             ret.write_cvalue(fx, res);
 723         };
 724         bitreverse, <T> (v arg) {
 725             let res = fx.bcx.ins().bitrev(arg);
 726             let res = CValue::by_val(res, fx.layout_of(T));
 727             ret.write_cvalue(fx, res);
 728         };
 729         bswap, <T> (v arg) {
 730             // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
 731             fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value {
 732                 match bcx.func.dfg.value_type(v) {
 733                     types::I8 => v,
 734
 735                     // https://code.woboq.org/gcc/include/bits/byteswap.h.html
 736                     types::I16 => {
 737                         let tmp1 = bcx.ins().ishl_imm(v, 8);
 738                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00);
 739
 740                         let tmp2 = bcx.ins().ushr_imm(v, 8);
 741                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF);
 742
 743                         bcx.ins().bor(n1, n2)
 744                     }
 745                     types::I32 => {
 746                         let tmp1 = bcx.ins().ishl_imm(v, 24);
 747                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000);
 748
 749                         let tmp2 = bcx.ins().ishl_imm(v, 8);
 750                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000);
 751
 752                         let tmp3 = bcx.ins().ushr_imm(v, 8);
 753                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00);
 754
 755                         let tmp4 = bcx.ins().ushr_imm(v, 24);
 756                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF);
 757
 758                         let or_tmp1 = bcx.ins().bor(n1, n2);
 759                         let or_tmp2 = bcx.ins().bor(n3, n4);
 760                         bcx.ins().bor(or_tmp1, or_tmp2)
 761                     }
 762                     types::I64 => {
 763                         let tmp1 = bcx.ins().ishl_imm(v, 56);
 764                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64);
 765
 766                         let tmp2 = bcx.ins().ishl_imm(v, 40);
 767                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64);
 768
 769                         let tmp3 = bcx.ins().ishl_imm(v, 24);
 770                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64);
 771
 772                         let tmp4 = bcx.ins().ishl_imm(v, 8);
 773                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64);
 774
 775                         let tmp5 = bcx.ins().ushr_imm(v, 8);
 776                         let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64);
 777
 778                         let tmp6 = bcx.ins().ushr_imm(v, 24);
 779                         let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64);
 780
 781                         let tmp7 = bcx.ins().ushr_imm(v, 40);
 782                         let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64);
 783
 784                         let tmp8 = bcx.ins().ushr_imm(v, 56);
 785                         let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64);
 786
 787                         let or_tmp1 = bcx.ins().bor(n1, n2);
 788                         let or_tmp2 = bcx.ins().bor(n3, n4);
 789                         let or_tmp3 = bcx.ins().bor(n5, n6);
 790                         let or_tmp4 = bcx.ins().bor(n7, n8);
 791
 792                         let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2);
 793                         let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4);
 794                         bcx.ins().bor(or_tmp5, or_tmp6)
 795                     }
 796                     types::I128 => {
 797                         let (lo, hi) = bcx.ins().isplit(v);
 798                         let lo = swap(bcx, lo);
 799                         let hi = swap(bcx, hi);
 800                         bcx.ins().iconcat(hi, lo)
 801                     }
 802                     ty => unreachable!("bswap {}", ty),
 803                 }
 804             };
 805             let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T));
 806             ret.write_cvalue(fx, res);
 807         };
 808         assert_inhabited | assert_zero_valid | assert_uninit_valid, <T> () {
 809             let layout = fx.layout_of(T);
 810             if layout.abi.is_uninhabited() {
 811                 crate::trap::trap_panic(fx, &format!("attempted to instantiate uninhabited type `{}`", T));
 812                 return;
 813             }
 814
 815             if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() {
 816                 crate::trap::trap_panic(fx, &format!("attempted to zero-initialize type `{}`, which is invalid", T));
 817                 return;
 818             }
 819
 820             if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() {
 821                 crate::trap::trap_panic(fx, &format!("attempted to leave type `{}` uninitialized, which is invalid", T));
 822                 return;
 823             }
 824         };
 825
 826         volatile_load, (c ptr) {
 827             // Cranelift treats loads as volatile by default
 828             // FIXME ignore during stack2reg optimization
 829             let inner_layout =
 830                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 831             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 832             ret.write_cvalue(fx, val);
 833         };
 834         volatile_store, (v ptr, c val) {
 835             // Cranelift treats stores as volatile by default
 836             // FIXME ignore during stack2reg optimization
 837             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 838             dest.write_cvalue(fx, val);
 839         };
 840
 841         size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () {
 842             let const_val =
 843                 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
 844             let val = crate::constant::trans_const_value(
 845                 fx,
 846                 const_val,
 847                 ret.layout().ty,
 848             );
 849             ret.write_cvalue(fx, val);
 850         };
 851
 852         ptr_offset_from, <T> (v ptr, v base) {
 853             let isize_layout = fx.layout_of(fx.tcx.types.isize);
 854
 855             let pointee_size: u64 = fx.layout_of(T).size.bytes();
 856             let diff = fx.bcx.ins().isub(ptr, base);
 857             // FIXME this can be an exact division.
 858             let val = CValue::by_val(fx.bcx.ins().sdiv_imm(diff, pointee_size as i64), isize_layout);
 859             ret.write_cvalue(fx, val);
 860         };
 861
 862         ptr_guaranteed_eq, (c a, c b) {
 863             let val = crate::num::trans_ptr_binop(fx, BinOp::Eq, a, b);
 864             ret.write_cvalue(fx, val);
 865         };
 866
 867         ptr_guaranteed_ne, (c a, c b) {
 868             let val = crate::num::trans_ptr_binop(fx, BinOp::Ne, a, b);
 869             ret.write_cvalue(fx, val);
 870         };
 871
 872         caller_location, () {
 873             let caller_location = fx.get_caller_location(span);
 874             ret.write_cvalue(fx, caller_location);
 875         };
 876
 877         _ if intrinsic.starts_with("atomic_fence"), () {
 878             crate::atomic_shim::lock_global_lock(fx);
 879             crate::atomic_shim::unlock_global_lock(fx);
 880         };
 881         _ if intrinsic.starts_with("atomic_singlethreadfence"), () {
 882             crate::atomic_shim::lock_global_lock(fx);
 883             crate::atomic_shim::unlock_global_lock(fx);
 884         };
 885         _ if intrinsic.starts_with("atomic_load"), (c ptr) {
 886             crate::atomic_shim::lock_global_lock(fx);
 887
 888             let inner_layout =
 889                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 890             validate_atomic_type!(fx, intrinsic, span, inner_layout.ty);
 891             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 892             ret.write_cvalue(fx, val);
 893
 894             crate::atomic_shim::unlock_global_lock(fx);
 895         };
 896         _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
 897             validate_atomic_type!(fx, intrinsic, span, val.layout().ty);
 898
 899             crate::atomic_shim::lock_global_lock(fx);
 900
 901             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 902             dest.write_cvalue(fx, val);
 903
 904             crate::atomic_shim::unlock_global_lock(fx);
 905         };
 906         _ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
 907             validate_atomic_type!(fx, intrinsic, span, T);
 908
 909             crate::atomic_shim::lock_global_lock(fx);
 910
 911             // Read old
 912             let clif_ty = fx.clif_type(T).unwrap();
 913             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 914             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
 915
 916             // Write new
 917             let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout());
 918             dest.write_cvalue(fx, src);
 919
 920             crate::atomic_shim::unlock_global_lock(fx);
 921         };
 922         _ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
 923             validate_atomic_type!(fx, intrinsic, span, T);
 924
 925             let test_old = test_old.load_scalar(fx);
 926             let new = new.load_scalar(fx);
 927
 928             crate::atomic_shim::lock_global_lock(fx);
 929
 930             // Read old
 931             let clif_ty = fx.clif_type(T).unwrap();
 932             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 933
 934             // Compare
 935             let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
 936             let new = fx.bcx.ins().select(is_eq, new, old); // Keep old if not equal to test_old
 937
 938             // Write new
 939             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 940
 941             let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
 942             ret.write_cvalue(fx, ret_val);
 943
 944             crate::atomic_shim::unlock_global_lock(fx);
 945         };
 946
 947         _ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, c amount) {
 948             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 949             let amount = amount.load_scalar(fx);
 950             atomic_binop_return_old! (fx, iadd<T>(ptr, amount) -> ret);
 951         };
 952         _ if intrinsic.starts_with("atomic_xsub"), <T> (v ptr, c amount) {
 953             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 954             let amount = amount.load_scalar(fx);
 955             atomic_binop_return_old! (fx, isub<T>(ptr, amount) -> ret);
 956         };
 957         _ if intrinsic.starts_with("atomic_and"), <T> (v ptr, c src) {
 958             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 959             let src = src.load_scalar(fx);
 960             atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
 961         };
 962         _ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, c src) {
 963             validate_atomic_type!(fx, intrinsic, span, T);
 964
 965             let src = src.load_scalar(fx);
 966
 967             crate::atomic_shim::lock_global_lock(fx);
 968
 969             let clif_ty = fx.clif_type(T).unwrap();
 970             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 971             let and = fx.bcx.ins().band(old, src);
 972             let new = fx.bcx.ins().bnot(and);
 973             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 974             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
 975
 976             crate::atomic_shim::unlock_global_lock(fx);
 977         };
 978         _ if intrinsic.starts_with("atomic_or"), <T> (v ptr, c src) {
 979             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 980             let src = src.load_scalar(fx);
 981             atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);
 982         };
 983         _ if intrinsic.starts_with("atomic_xor"), <T> (v ptr, c src) {
 984             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 985             let src = src.load_scalar(fx);
 986             atomic_binop_return_old! (fx, bxor<T>(ptr, src) -> ret);
 987         };
 988
 989         _ if intrinsic.starts_with("atomic_max"), <T> (v ptr, c src) {
 990             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 991             let src = src.load_scalar(fx);
 992             atomic_minmax!(fx, IntCC::SignedGreaterThan, <T> (ptr, src) -> ret);
 993         };
 994         _ if intrinsic.starts_with("atomic_umax"), <T> (v ptr, c src) {
 995             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 996             let src = src.load_scalar(fx);
 997             atomic_minmax!(fx, IntCC::UnsignedGreaterThan, <T> (ptr, src) -> ret);
 998         };
 999         _ if intrinsic.starts_with("atomic_min"), <T> (v ptr, c src) {
1000             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1001             let src = src.load_scalar(fx);
1002             atomic_minmax!(fx, IntCC::SignedLessThan, <T> (ptr, src) -> ret);
1003         };
1004         _ if intrinsic.starts_with("atomic_umin"), <T> (v ptr, c src) {
1005             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1006             let src = src.load_scalar(fx);
1007             atomic_minmax!(fx, IntCC::UnsignedLessThan, <T> (ptr, src) -> ret);
1008         };
1009
1010         minnumf32, (v a, v b) {
1011             let val = fx.bcx.ins().fmin(a, b);
1012             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1013             ret.write_cvalue(fx, val);
1014         };
1015         minnumf64, (v a, v b) {
1016             let val = fx.bcx.ins().fmin(a, b);
1017             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1018             ret.write_cvalue(fx, val);
1019         };
1020         maxnumf32, (v a, v b) {
1021             let val = fx.bcx.ins().fmax(a, b);
1022             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1023             ret.write_cvalue(fx, val);
1024         };
1025         maxnumf64, (v a, v b) {
1026             let val = fx.bcx.ins().fmax(a, b);
1027             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1028             ret.write_cvalue(fx, val);
1029         };
1030
1031         try, (v f, v data, v _catch_fn) {
1032             // FIXME once unwinding is supported, change this to actually catch panics
1033             let f_sig = fx.bcx.func.import_signature(Signature {
1034                 call_conv: CallConv::triple_default(fx.triple()),
1035                 params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
1036                 returns: vec![],
1037             });
1038
1039             fx.bcx.ins().call_indirect(f_sig, f, &[data]);
1040
1041             let ret_val = CValue::const_val(fx, ret.layout(), 0);
1042             ret.write_cvalue(fx, ret_val);
1043         };
1044
1045         fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) {
1046             let res = crate::num::trans_float_binop(fx, match intrinsic {
1047                 "fadd_fast" => BinOp::Add,
1048                 "fsub_fast" => BinOp::Sub,
1049                 "fmul_fast" => BinOp::Mul,
1050                 "fdiv_fast" => BinOp::Div,
1051                 "frem_fast" => BinOp::Rem,
1052                 _ => unreachable!(),
1053             }, x, y);
1054             ret.write_cvalue(fx, res);
1055         };
1056         float_to_int_unchecked, (v f) {
1057             let res = crate::cast::clif_int_or_float_cast(
1058                 fx,
1059                 f,
1060                 false,
1061                 fx.clif_type(ret.layout().ty).unwrap(),
1062                 type_sign(ret.layout().ty),
1063             );
1064             ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
1065         };
1066     }
1067
1068     if let Some((_, dest)) = destination {
1069         let ret_block = fx.get_block(dest);
1070         fx.bcx.ins().jump(ret_block, &[]);
1071     } else {
1072         trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
1073     }
1074 }