src/intrinsics/mod.rs

   1 //! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"`
   2 //! and LLVM intrinsics that have symbol names starting with `llvm.`.
   3
   4 mod cpuid;
   5 mod llvm;
   6 mod simd;
   7
   8 pub(crate) use cpuid::codegen_cpuid_call;
   9 pub(crate) use llvm::codegen_llvm_intrinsic_call;
  10
  11 use crate::prelude::*;
  12 use rustc_middle::ty::print::with_no_trimmed_paths;
  13
  14 macro intrinsic_pat {
  15     (_) => {
  16         _
  17     },
  18     ($name:ident) => {
  19         stringify!($name)
  20     },
  21     ($name:literal) => {
  22         stringify!($name)
  23     },
  24     ($x:ident . $($xs:tt).*) => {
  25         concat!(stringify!($x), ".", intrinsic_pat!($($xs).*))
  26     }
  27 }
  28
  29 macro intrinsic_arg {
  30     (o $fx:expr, $arg:ident) => {
  31         $arg
  32     },
  33     (c $fx:expr, $arg:ident) => {
  34         codegen_operand($fx, $arg)
  35     },
  36     (v $fx:expr, $arg:ident) => {
  37         codegen_operand($fx, $arg).load_scalar($fx)
  38     }
  39 }
  40
  41 macro intrinsic_substs {
  42     ($substs:expr, $index:expr,) => {},
  43     ($substs:expr, $index:expr, $first:ident $(,$rest:ident)*) => {
  44         let $first = $substs.type_at($index);
  45         intrinsic_substs!($substs, $index+1, $($rest),*);
  46     }
  47 }
  48
  49 macro intrinsic_match {
  50     ($fx:expr, $intrinsic:expr, $substs:expr, $args:expr,
  51     _ => $unknown:block;
  52     $(
  53         $($($name:tt).*)|+ $(if $cond:expr)?, $(<$($subst:ident),*>)? ($($a:ident $arg:ident),*) $content:block;
  54     )*) => {
  55         let _ = $substs; // Silence warning when substs is unused.
  56         match $intrinsic {
  57             $(
  58                 $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
  59                     #[allow(unused_parens, non_snake_case)]
  60                     {
  61                         $(
  62                             intrinsic_substs!($substs, 0, $($subst),*);
  63                         )?
  64                         if let [$($arg),*] = $args {
  65                             let ($($arg,)*) = (
  66                                 $(intrinsic_arg!($a $fx, $arg),)*
  67                             );
  68                             #[warn(unused_parens, non_snake_case)]
  69                             {
  70                                 $content
  71                             }
  72                         } else {
  73                             bug!("wrong number of args for intrinsic {:?}", $intrinsic);
  74                         }
  75                     }
  76                 }
  77             )*
  78             _ => $unknown,
  79         }
  80     }
  81 }
  82
  83 macro call_intrinsic_match {
  84     ($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $destination:expr, $args:expr, $(
  85         $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
  86     )*) => {
  87         match $intrinsic {
  88             $(
  89                 stringify!($name) => {
  90                     assert!($substs.is_noop());
  91                     if let [$(ref $arg),*] = *$args {
  92                         let ($($arg,)*) = (
  93                             $(codegen_operand($fx, $arg),)*
  94                         );
  95                         let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty);
  96                         $ret.write_cvalue($fx, res);
  97
  98                         if let Some((_, dest)) = $destination {
  99                             let ret_block = $fx.get_block(dest);
 100                             $fx.bcx.ins().jump(ret_block, &[]);
 101                             return;
 102                         } else {
 103                             unreachable!();
 104                         }
 105                     } else {
 106                         bug!("wrong number of args for intrinsic {:?}", $intrinsic);
 107                     }
 108                 }
 109             )*
 110             _ => {}
 111         }
 112     }
 113 }
 114
 115 macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) {
 116     crate::atomic_shim::lock_global_lock($fx);
 117
 118     let clif_ty = $fx.clif_type($T).unwrap();
 119     let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 120     let new = $fx.bcx.ins().$op(old, $src);
 121     $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 122     $ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
 123
 124     crate::atomic_shim::unlock_global_lock($fx);
 125 }
 126
 127 macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) {
 128     crate::atomic_shim::lock_global_lock($fx);
 129
 130     // Read old
 131     let clif_ty = $fx.clif_type($T).unwrap();
 132     let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 133
 134     // Compare
 135     let is_eq = $fx.bcx.ins().icmp(IntCC::SignedGreaterThan, old, $src);
 136     let new = $fx.bcx.ins().select(is_eq, old, $src);
 137
 138     // Write new
 139     $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 140
 141     let ret_val = CValue::by_val(old, $ret.layout());
 142     $ret.write_cvalue($fx, ret_val);
 143
 144     crate::atomic_shim::unlock_global_lock($fx);
 145 }
 146
 147 macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
 148     match $ty.kind() {
 149         ty::Uint(_) | ty::Int(_) => {}
 150         _ => {
 151             $fx.tcx.sess.span_err(
 152                 $span,
 153                 &format!(
 154                     "`{}` intrinsic: expected basic integer type, found `{:?}`",
 155                     $intrinsic, $ty
 156                 ),
 157             );
 158             // Prevent verifier error
 159             crate::trap::trap_unreachable($fx, "compilation should not have succeeded");
 160             return;
 161         }
 162     }
 163 }
 164
 165 macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
 166     if !$ty.is_simd() {
 167         $fx.tcx.sess.span_err($span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", $intrinsic, $ty));
 168         // Prevent verifier error
 169         crate::trap::trap_unreachable($fx, "compilation should not have succeeded");
 170         return;
 171     }
 172 }
 173
 174 fn lane_type_and_count<'tcx>(
 175     tcx: TyCtxt<'tcx>,
 176     layout: TyAndLayout<'tcx>,
 177 ) -> (TyAndLayout<'tcx>, u16) {
 178     assert!(layout.ty.is_simd());
 179     let lane_count = match layout.fields {
 180         rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(),
 181         _ => unreachable!("lane_type_and_count({:?})", layout),
 182     };
 183     let lane_layout = layout
 184         .field(
 185             &ty::layout::LayoutCx {
 186                 tcx,
 187                 param_env: ParamEnv::reveal_all(),
 188             },
 189             0,
 190         )
 191         .unwrap();
 192     (lane_layout, lane_count)
 193 }
 194
 195 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
 196     let (element, count) = match &layout.abi {
 197         Abi::Vector { element, count } => (element.clone(), *count),
 198         _ => unreachable!(),
 199     };
 200
 201     match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
 202         // Cranelift currently only implements icmp for 128bit vectors.
 203         Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
 204         _ => None,
 205     }
 206 }
 207
 208 fn simd_for_each_lane<'tcx, M: Module>(
 209     fx: &mut FunctionCx<'_, 'tcx, M>,
 210     val: CValue<'tcx>,
 211     ret: CPlace<'tcx>,
 212     f: impl Fn(
 213         &mut FunctionCx<'_, 'tcx, M>,
 214         TyAndLayout<'tcx>,
 215         TyAndLayout<'tcx>,
 216         Value,
 217     ) -> CValue<'tcx>,
 218 ) {
 219     let layout = val.layout();
 220
 221     let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
 222     let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
 223     assert_eq!(lane_count, ret_lane_count);
 224
 225     for lane_idx in 0..lane_count {
 226         let lane_idx = mir::Field::new(lane_idx.try_into().unwrap());
 227         let lane = val.value_field(fx, lane_idx).load_scalar(fx);
 228
 229         let res_lane = f(fx, lane_layout, ret_lane_layout, lane);
 230
 231         ret.place_field(fx, lane_idx).write_cvalue(fx, res_lane);
 232     }
 233 }
 234
 235 fn simd_pair_for_each_lane<'tcx, M: Module>(
 236     fx: &mut FunctionCx<'_, 'tcx, M>,
 237     x: CValue<'tcx>,
 238     y: CValue<'tcx>,
 239     ret: CPlace<'tcx>,
 240     f: impl Fn(
 241         &mut FunctionCx<'_, 'tcx, M>,
 242         TyAndLayout<'tcx>,
 243         TyAndLayout<'tcx>,
 244         Value,
 245         Value,
 246     ) -> CValue<'tcx>,
 247 ) {
 248     assert_eq!(x.layout(), y.layout());
 249     let layout = x.layout();
 250
 251     let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
 252     let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
 253     assert_eq!(lane_count, ret_lane_count);
 254
 255     for lane in 0..lane_count {
 256         let lane = mir::Field::new(lane.try_into().unwrap());
 257         let x_lane = x.value_field(fx, lane).load_scalar(fx);
 258         let y_lane = y.value_field(fx, lane).load_scalar(fx);
 259
 260         let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane);
 261
 262         ret.place_field(fx, lane).write_cvalue(fx, res_lane);
 263     }
 264 }
 265
 266 fn simd_reduce<'tcx, M: Module>(
 267     fx: &mut FunctionCx<'_, 'tcx, M>,
 268     val: CValue<'tcx>,
 269     ret: CPlace<'tcx>,
 270     f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value,
 271 ) {
 272     let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout());
 273     assert_eq!(lane_layout, ret.layout());
 274
 275     let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
 276     for lane_idx in 1..lane_count {
 277         let lane = val
 278             .value_field(fx, mir::Field::new(lane_idx.into()))
 279             .load_scalar(fx);
 280         res_val = f(fx, lane_layout, res_val, lane);
 281     }
 282     let res = CValue::by_val(res_val, lane_layout);
 283     ret.write_cvalue(fx, res);
 284 }
 285
 286 fn simd_reduce_bool<'tcx, M: Module>(
 287     fx: &mut FunctionCx<'_, 'tcx, M>,
 288     val: CValue<'tcx>,
 289     ret: CPlace<'tcx>,
 290     f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value,
 291 ) {
 292     let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout());
 293     assert!(ret.layout().ty.is_bool());
 294
 295     let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
 296     let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
 297     for lane_idx in 1..lane_count {
 298         let lane = val
 299             .value_field(fx, mir::Field::new(lane_idx.into()))
 300             .load_scalar(fx);
 301         let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
 302         res_val = f(fx, res_val, lane);
 303     }
 304     let res = CValue::by_val(res_val, ret.layout());
 305     ret.write_cvalue(fx, res);
 306 }
 307
 308 fn bool_to_zero_or_max_uint<'tcx>(
 309     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 310     layout: TyAndLayout<'tcx>,
 311     val: Value,
 312 ) -> CValue<'tcx> {
 313     let ty = fx.clif_type(layout.ty).unwrap();
 314
 315     let int_ty = match ty {
 316         types::F32 => types::I32,
 317         types::F64 => types::I64,
 318         ty => ty,
 319     };
 320
 321     let val = fx.bcx.ins().bint(int_ty, val);
 322     let mut res = fx.bcx.ins().ineg(val);
 323
 324     if ty.is_float() {
 325         res = fx.bcx.ins().bitcast(ty, res);
 326     }
 327
 328     CValue::by_val(res, layout)
 329 }
 330
 331 macro simd_cmp {
 332     ($fx:expr, $cc:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 333         let vector_ty = clif_vector_type($fx.tcx, $x.layout());
 334
 335         if let Some(vector_ty) = vector_ty {
 336             let x = $x.load_scalar($fx);
 337             let y = $y.load_scalar($fx);
 338             let val = $fx.bcx.ins().icmp(IntCC::$cc, x, y);
 339
 340             // HACK This depends on the fact that icmp for vectors represents bools as 0 and !0, not 0 and 1.
 341             let val = $fx.bcx.ins().raw_bitcast(vector_ty, val);
 342
 343             $ret.write_cvalue($fx, CValue::by_val(val, $ret.layout()));
 344         } else {
 345             simd_pair_for_each_lane(
 346                 $fx,
 347                 $x,
 348                 $y,
 349                 $ret,
 350                 |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 351                     let res_lane = match lane_layout.ty.kind() {
 352                         ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane),
 353                         ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
 354                         _ => unreachable!("{:?}", lane_layout.ty),
 355                     };
 356                     bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 357                 },
 358             );
 359         }
 360     },
 361     ($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 362         // FIXME use vector icmp when possible
 363         simd_pair_for_each_lane(
 364             $fx,
 365             $x,
 366             $y,
 367             $ret,
 368             |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 369                 let res_lane = match lane_layout.ty.kind() {
 370                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
 371                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
 372                     ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
 373                     _ => unreachable!("{:?}", lane_layout.ty),
 374                 };
 375                 bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 376             },
 377         );
 378     },
 379 }
 380
 381 macro simd_int_binop {
 382     ($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
 383         simd_int_binop!($fx, $op|$op($x, $y) -> $ret);
 384     },
 385     ($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 386         simd_pair_for_each_lane(
 387             $fx,
 388             $x,
 389             $y,
 390             $ret,
 391             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 392                 let res_lane = match lane_layout.ty.kind() {
 393                     ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 394                     ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 395                     _ => unreachable!("{:?}", lane_layout.ty),
 396                 };
 397                 CValue::by_val(res_lane, ret_lane_layout)
 398             },
 399         );
 400     },
 401 }
 402
 403 macro simd_int_flt_binop {
 404     ($fx:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 405         simd_int_flt_binop!($fx, $op|$op|$op_f($x, $y) -> $ret);
 406     },
 407     ($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 408         simd_pair_for_each_lane(
 409             $fx,
 410             $x,
 411             $y,
 412             $ret,
 413             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 414                 let res_lane = match lane_layout.ty.kind() {
 415                     ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 416                     ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 417                     ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
 418                     _ => unreachable!("{:?}", lane_layout.ty),
 419                 };
 420                 CValue::by_val(res_lane, ret_lane_layout)
 421             },
 422         );
 423     },
 424 }
 425
 426 macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
 427     simd_pair_for_each_lane(
 428         $fx,
 429         $x,
 430         $y,
 431         $ret,
 432         |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 433             let res_lane = match lane_layout.ty.kind() {
 434                 ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
 435                 _ => unreachable!("{:?}", lane_layout.ty),
 436             };
 437             CValue::by_val(res_lane, ret_lane_layout)
 438         },
 439     );
 440 }
 441
 442 pub(crate) fn codegen_intrinsic_call<'tcx>(
 443     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 444     instance: Instance<'tcx>,
 445     args: &[mir::Operand<'tcx>],
 446     destination: Option<(CPlace<'tcx>, BasicBlock)>,
 447     span: Span,
 448 ) {
 449     let def_id = instance.def_id();
 450     let substs = instance.substs;
 451
 452     let intrinsic = fx.tcx.item_name(def_id).as_str();
 453     let intrinsic = &intrinsic[..];
 454
 455     let ret = match destination {
 456         Some((place, _)) => place,
 457         None => {
 458             // Insert non returning intrinsics here
 459             match intrinsic {
 460                 "abort" => {
 461                     trap_abort(fx, "Called intrinsic::abort.");
 462                 }
 463                 "unreachable" => {
 464                     trap_unreachable(fx, "[corruption] Called intrinsic::unreachable.");
 465                 }
 466                 "transmute" => {
 467                     crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span);
 468                 }
 469                 _ => unimplemented!("unsupported instrinsic {}", intrinsic),
 470             }
 471             return;
 472         }
 473     };
 474
 475     if intrinsic.starts_with("simd_") {
 476         self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span);
 477         let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
 478         fx.bcx.ins().jump(ret_block, &[]);
 479         return;
 480     }
 481
 482     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 483
 484     call_intrinsic_match! {
 485         fx, intrinsic, substs, ret, destination, args,
 486         expf32(flt) -> f32 => expf,
 487         expf64(flt) -> f64 => exp,
 488         exp2f32(flt) -> f32 => exp2f,
 489         exp2f64(flt) -> f64 => exp2,
 490         sqrtf32(flt) -> f32 => sqrtf,
 491         sqrtf64(flt) -> f64 => sqrt,
 492         powif32(a, x) -> f32 => __powisf2, // compiler-builtins
 493         powif64(a, x) -> f64 => __powidf2, // compiler-builtins
 494         powf32(a, x) -> f32 => powf,
 495         powf64(a, x) -> f64 => pow,
 496         logf32(flt) -> f32 => logf,
 497         logf64(flt) -> f64 => log,
 498         log2f32(flt) -> f32 => log2f,
 499         log2f64(flt) -> f64 => log2,
 500         log10f32(flt) -> f32 => log10f,
 501         log10f64(flt) -> f64 => log10,
 502         fabsf32(flt) -> f32 => fabsf,
 503         fabsf64(flt) -> f64 => fabs,
 504         fmaf32(x, y, z) -> f32 => fmaf,
 505         fmaf64(x, y, z) -> f64 => fma,
 506         copysignf32(x, y) -> f32 => copysignf,
 507         copysignf64(x, y) -> f64 => copysign,
 508
 509         // rounding variants
 510         // FIXME use clif insts
 511         floorf32(flt) -> f32 => floorf,
 512         floorf64(flt) -> f64 => floor,
 513         ceilf32(flt) -> f32 => ceilf,
 514         ceilf64(flt) -> f64 => ceil,
 515         truncf32(flt) -> f32 => truncf,
 516         truncf64(flt) -> f64 => trunc,
 517         roundf32(flt) -> f32 => roundf,
 518         roundf64(flt) -> f64 => round,
 519
 520         // trigonometry
 521         sinf32(flt) -> f32 => sinf,
 522         sinf64(flt) -> f64 => sin,
 523         cosf32(flt) -> f32 => cosf,
 524         cosf64(flt) -> f64 => cos,
 525         tanf32(flt) -> f32 => tanf,
 526         tanf64(flt) -> f64 => tan,
 527     }
 528
 529     intrinsic_match! {
 530         fx, intrinsic, substs, args,
 531         _ => {
 532             fx.tcx.sess.span_fatal(span, &format!("unsupported intrinsic {}", intrinsic));
 533         };
 534
 535         assume, (c _a) {};
 536         likely | unlikely, (c a) {
 537             ret.write_cvalue(fx, a);
 538         };
 539         breakpoint, () {
 540             fx.bcx.ins().debugtrap();
 541         };
 542         copy | copy_nonoverlapping, <elem_ty> (v src, v dst, v count) {
 543             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 544             assert_eq!(args.len(), 3);
 545             let byte_amount = if elem_size != 1 {
 546                 fx.bcx.ins().imul_imm(count, elem_size as i64)
 547             } else {
 548                 count
 549             };
 550
 551             if intrinsic.contains("nonoverlapping") {
 552                 // FIXME emit_small_memcpy
 553                 fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount);
 554             } else {
 555                 // FIXME emit_small_memmove
 556                 fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
 557             }
 558         };
 559         // NOTE: the volatile variants have src and dst swapped
 560         volatile_copy_memory | volatile_copy_nonoverlapping_memory, <elem_ty> (v dst, v src, v count) {
 561             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 562             assert_eq!(args.len(), 3);
 563             let byte_amount = if elem_size != 1 {
 564                 fx.bcx.ins().imul_imm(count, elem_size as i64)
 565             } else {
 566                 count
 567             };
 568
 569             // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
 570             if intrinsic.contains("nonoverlapping") {
 571                 // FIXME emit_small_memcpy
 572                 fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount);
 573             } else {
 574                 // FIXME emit_small_memmove
 575                 fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
 576             }
 577         };
 578         discriminant_value, (c ptr) {
 579             let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 580             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout);
 581             let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout());
 582             ret.write_cvalue(fx, discr);
 583         };
 584         size_of_val, <T> (c ptr) {
 585             let layout = fx.layout_of(T);
 586             let size = if layout.is_unsized() {
 587                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 588                 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 589                 size
 590             } else {
 591                 fx
 592                     .bcx
 593                     .ins()
 594                     .iconst(fx.pointer_type, layout.size.bytes() as i64)
 595             };
 596             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 597         };
 598         min_align_of_val, <T> (c ptr) {
 599             let layout = fx.layout_of(T);
 600             let align = if layout.is_unsized() {
 601                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 602                 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 603                 align
 604             } else {
 605                 fx
 606                     .bcx
 607                     .ins()
 608                     .iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
 609             };
 610             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 611         };
 612
 613         _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) {
 614             // FIXME trap on overflow
 615             let bin_op = match intrinsic {
 616                 "unchecked_add" => BinOp::Add,
 617                 "unchecked_sub" => BinOp::Sub,
 618                 "unchecked_div" | "exact_div" => BinOp::Div,
 619                 "unchecked_rem" => BinOp::Rem,
 620                 "unchecked_shl" => BinOp::Shl,
 621                 "unchecked_shr" => BinOp::Shr,
 622                 _ => unreachable!("intrinsic {}", intrinsic),
 623             };
 624             let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
 625             ret.write_cvalue(fx, res);
 626         };
 627         _ if intrinsic.ends_with("_with_overflow"), (c x, c y) {
 628             assert_eq!(x.layout().ty, y.layout().ty);
 629             let bin_op = match intrinsic {
 630                 "add_with_overflow" => BinOp::Add,
 631                 "sub_with_overflow" => BinOp::Sub,
 632                 "mul_with_overflow" => BinOp::Mul,
 633                 _ => unreachable!("intrinsic {}", intrinsic),
 634             };
 635
 636             let res = crate::num::codegen_checked_int_binop(
 637                 fx,
 638                 bin_op,
 639                 x,
 640                 y,
 641             );
 642             ret.write_cvalue(fx, res);
 643         };
 644         _ if intrinsic.starts_with("wrapping_"), (c x, c y) {
 645             assert_eq!(x.layout().ty, y.layout().ty);
 646             let bin_op = match intrinsic {
 647                 "wrapping_add" => BinOp::Add,
 648                 "wrapping_sub" => BinOp::Sub,
 649                 "wrapping_mul" => BinOp::Mul,
 650                 _ => unreachable!("intrinsic {}", intrinsic),
 651             };
 652             let res = crate::num::codegen_int_binop(
 653                 fx,
 654                 bin_op,
 655                 x,
 656                 y,
 657             );
 658             ret.write_cvalue(fx, res);
 659         };
 660         _ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) {
 661             assert_eq!(lhs.layout().ty, rhs.layout().ty);
 662             let bin_op = match intrinsic {
 663                 "saturating_add" => BinOp::Add,
 664                 "saturating_sub" => BinOp::Sub,
 665                 _ => unreachable!("intrinsic {}", intrinsic),
 666             };
 667
 668             let signed = type_sign(T);
 669
 670             let checked_res = crate::num::codegen_checked_int_binop(
 671                 fx,
 672                 bin_op,
 673                 lhs,
 674                 rhs,
 675             );
 676
 677             let (val, has_overflow) = checked_res.load_scalar_pair(fx);
 678             let clif_ty = fx.clif_type(T).unwrap();
 679
 680             // `select.i8` is not implemented by Cranelift.
 681             let has_overflow = fx.bcx.ins().uextend(types::I32, has_overflow);
 682
 683             let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
 684
 685             let val = match (intrinsic, signed) {
 686                 ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val),
 687                 ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val),
 688                 ("saturating_add", true) => {
 689                     let rhs = rhs.load_scalar(fx);
 690                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 691                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
 692                     fx.bcx.ins().select(has_overflow, sat_val, val)
 693                 }
 694                 ("saturating_sub", true) => {
 695                     let rhs = rhs.load_scalar(fx);
 696                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 697                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
 698                     fx.bcx.ins().select(has_overflow, sat_val, val)
 699                 }
 700                 _ => unreachable!(),
 701             };
 702
 703             let res = CValue::by_val(val, fx.layout_of(T));
 704
 705             ret.write_cvalue(fx, res);
 706         };
 707         rotate_left, <T>(v x, v y) {
 708             let layout = fx.layout_of(T);
 709             let res = fx.bcx.ins().rotl(x, y);
 710             ret.write_cvalue(fx, CValue::by_val(res, layout));
 711         };
 712         rotate_right, <T>(v x, v y) {
 713             let layout = fx.layout_of(T);
 714             let res = fx.bcx.ins().rotr(x, y);
 715             ret.write_cvalue(fx, CValue::by_val(res, layout));
 716         };
 717
 718         // The only difference between offset and arith_offset is regarding UB. Because Cranelift
 719         // doesn't have UB both are codegen'ed the same way
 720         offset | arith_offset, (c base, v offset) {
 721             let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
 722             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 723             let ptr_diff = if pointee_size != 1 {
 724                 fx.bcx.ins().imul_imm(offset, pointee_size as i64)
 725             } else {
 726                 offset
 727             };
 728             let base_val = base.load_scalar(fx);
 729             let res = fx.bcx.ins().iadd(base_val, ptr_diff);
 730             ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
 731         };
 732
 733         transmute, (c from) {
 734             ret.write_cvalue_transmute(fx, from);
 735         };
 736         write_bytes | volatile_set_memory, (c dst, v val, v count) {
 737             let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
 738             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 739             let count = if pointee_size != 1 {
 740                 fx.bcx.ins().imul_imm(count, pointee_size as i64)
 741             } else {
 742                 count
 743             };
 744             let dst_ptr = dst.load_scalar(fx);
 745             // FIXME make the memset actually volatile when switching to emit_small_memset
 746             // FIXME use emit_small_memset
 747             fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count);
 748         };
 749         ctlz | ctlz_nonzero, <T> (v arg) {
 750             // FIXME trap on `ctlz_nonzero` with zero arg.
 751             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 752                 // FIXME verify this algorithm is correct
 753                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 754                 let lsb_lz = fx.bcx.ins().clz(lsb);
 755                 let msb_lz = fx.bcx.ins().clz(msb);
 756                 let msb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, msb, 0);
 757                 let lsb_lz_plus_64 = fx.bcx.ins().iadd_imm(lsb_lz, 64);
 758                 let res = fx.bcx.ins().select(msb_is_zero, lsb_lz_plus_64, msb_lz);
 759                 fx.bcx.ins().uextend(types::I128, res)
 760             } else {
 761                 fx.bcx.ins().clz(arg)
 762             };
 763             let res = CValue::by_val(res, fx.layout_of(T));
 764             ret.write_cvalue(fx, res);
 765         };
 766         cttz | cttz_nonzero, <T> (v arg) {
 767             // FIXME trap on `cttz_nonzero` with zero arg.
 768             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 769                 // FIXME verify this algorithm is correct
 770                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 771                 let lsb_tz = fx.bcx.ins().ctz(lsb);
 772                 let msb_tz = fx.bcx.ins().ctz(msb);
 773                 let lsb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, lsb, 0);
 774                 let msb_tz_plus_64 = fx.bcx.ins().iadd_imm(msb_tz, 64);
 775                 let res = fx.bcx.ins().select(lsb_is_zero, msb_tz_plus_64, lsb_tz);
 776                 fx.bcx.ins().uextend(types::I128, res)
 777             } else {
 778                 fx.bcx.ins().ctz(arg)
 779             };
 780             let res = CValue::by_val(res, fx.layout_of(T));
 781             ret.write_cvalue(fx, res);
 782         };
 783         ctpop, <T> (v arg) {
 784             let res = fx.bcx.ins().popcnt(arg);
 785             let res = CValue::by_val(res, fx.layout_of(T));
 786             ret.write_cvalue(fx, res);
 787         };
 788         bitreverse, <T> (v arg) {
 789             let res = fx.bcx.ins().bitrev(arg);
 790             let res = CValue::by_val(res, fx.layout_of(T));
 791             ret.write_cvalue(fx, res);
 792         };
 793         bswap, <T> (v arg) {
 794             // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
 795             fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value {
 796                 match bcx.func.dfg.value_type(v) {
 797                     types::I8 => v,
 798
 799                     // https://code.woboq.org/gcc/include/bits/byteswap.h.html
 800                     types::I16 => {
 801                         let tmp1 = bcx.ins().ishl_imm(v, 8);
 802                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00);
 803
 804                         let tmp2 = bcx.ins().ushr_imm(v, 8);
 805                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF);
 806
 807                         bcx.ins().bor(n1, n2)
 808                     }
 809                     types::I32 => {
 810                         let tmp1 = bcx.ins().ishl_imm(v, 24);
 811                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000);
 812
 813                         let tmp2 = bcx.ins().ishl_imm(v, 8);
 814                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000);
 815
 816                         let tmp3 = bcx.ins().ushr_imm(v, 8);
 817                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00);
 818
 819                         let tmp4 = bcx.ins().ushr_imm(v, 24);
 820                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF);
 821
 822                         let or_tmp1 = bcx.ins().bor(n1, n2);
 823                         let or_tmp2 = bcx.ins().bor(n3, n4);
 824                         bcx.ins().bor(or_tmp1, or_tmp2)
 825                     }
 826                     types::I64 => {
 827                         let tmp1 = bcx.ins().ishl_imm(v, 56);
 828                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64);
 829
 830                         let tmp2 = bcx.ins().ishl_imm(v, 40);
 831                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64);
 832
 833                         let tmp3 = bcx.ins().ishl_imm(v, 24);
 834                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64);
 835
 836                         let tmp4 = bcx.ins().ishl_imm(v, 8);
 837                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64);
 838
 839                         let tmp5 = bcx.ins().ushr_imm(v, 8);
 840                         let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64);
 841
 842                         let tmp6 = bcx.ins().ushr_imm(v, 24);
 843                         let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64);
 844
 845                         let tmp7 = bcx.ins().ushr_imm(v, 40);
 846                         let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64);
 847
 848                         let tmp8 = bcx.ins().ushr_imm(v, 56);
 849                         let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64);
 850
 851                         let or_tmp1 = bcx.ins().bor(n1, n2);
 852                         let or_tmp2 = bcx.ins().bor(n3, n4);
 853                         let or_tmp3 = bcx.ins().bor(n5, n6);
 854                         let or_tmp4 = bcx.ins().bor(n7, n8);
 855
 856                         let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2);
 857                         let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4);
 858                         bcx.ins().bor(or_tmp5, or_tmp6)
 859                     }
 860                     types::I128 => {
 861                         let (lo, hi) = bcx.ins().isplit(v);
 862                         let lo = swap(bcx, lo);
 863                         let hi = swap(bcx, hi);
 864                         bcx.ins().iconcat(hi, lo)
 865                     }
 866                     ty => unreachable!("bswap {}", ty),
 867                 }
 868             };
 869             let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T));
 870             ret.write_cvalue(fx, res);
 871         };
 872         assert_inhabited | assert_zero_valid | assert_uninit_valid, <T> () {
 873             let layout = fx.layout_of(T);
 874             if layout.abi.is_uninhabited() {
 875                 with_no_trimmed_paths(|| crate::base::codegen_panic(
 876                     fx,
 877                     &format!("attempted to instantiate uninhabited type `{}`", T),
 878                     span,
 879                 ));
 880                 return;
 881             }
 882
 883             if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() {
 884                 with_no_trimmed_paths(|| crate::base::codegen_panic(
 885                     fx,
 886                     &format!("attempted to zero-initialize type `{}`, which is invalid", T),
 887                     span,
 888                 ));
 889                 return;
 890             }
 891
 892             if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() {
 893                 with_no_trimmed_paths(|| crate::base::codegen_panic(
 894                     fx,
 895                     &format!("attempted to leave type `{}` uninitialized, which is invalid", T),
 896                     span,
 897                 ));
 898                 return;
 899             }
 900         };
 901
 902         volatile_load | unaligned_volatile_load, (c ptr) {
 903             // Cranelift treats loads as volatile by default
 904             // FIXME ignore during stack2reg optimization
 905             // FIXME correctly handle unaligned_volatile_load
 906             let inner_layout =
 907                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 908             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 909             ret.write_cvalue(fx, val);
 910         };
 911         volatile_store | unaligned_volatile_store, (v ptr, c val) {
 912             // Cranelift treats stores as volatile by default
 913             // FIXME ignore during stack2reg optimization
 914             // FIXME correctly handle unaligned_volatile_store
 915             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 916             dest.write_cvalue(fx, val);
 917         };
 918
 919         size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () {
 920             let const_val =
 921                 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
 922             let val = crate::constant::codegen_const_value(
 923                 fx,
 924                 const_val,
 925                 ret.layout().ty,
 926             );
 927             ret.write_cvalue(fx, val);
 928         };
 929
 930         ptr_offset_from, <T> (v ptr, v base) {
 931             let isize_layout = fx.layout_of(fx.tcx.types.isize);
 932
 933             let pointee_size: u64 = fx.layout_of(T).size.bytes();
 934             let diff = fx.bcx.ins().isub(ptr, base);
 935             // FIXME this can be an exact division.
 936             let val = CValue::by_val(fx.bcx.ins().sdiv_imm(diff, pointee_size as i64), isize_layout);
 937             ret.write_cvalue(fx, val);
 938         };
 939
 940         ptr_guaranteed_eq, (c a, c b) {
 941             let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b);
 942             ret.write_cvalue(fx, val);
 943         };
 944
 945         ptr_guaranteed_ne, (c a, c b) {
 946             let val = crate::num::codegen_ptr_binop(fx, BinOp::Ne, a, b);
 947             ret.write_cvalue(fx, val);
 948         };
 949
 950         caller_location, () {
 951             let caller_location = fx.get_caller_location(span);
 952             ret.write_cvalue(fx, caller_location);
 953         };
 954
 955         _ if intrinsic.starts_with("atomic_fence"), () {
 956             crate::atomic_shim::lock_global_lock(fx);
 957             crate::atomic_shim::unlock_global_lock(fx);
 958         };
 959         _ if intrinsic.starts_with("atomic_singlethreadfence"), () {
 960             crate::atomic_shim::lock_global_lock(fx);
 961             crate::atomic_shim::unlock_global_lock(fx);
 962         };
 963         _ if intrinsic.starts_with("atomic_load"), (c ptr) {
 964             crate::atomic_shim::lock_global_lock(fx);
 965
 966             let inner_layout =
 967                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 968             validate_atomic_type!(fx, intrinsic, span, inner_layout.ty);
 969             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 970             ret.write_cvalue(fx, val);
 971
 972             crate::atomic_shim::unlock_global_lock(fx);
 973         };
 974         _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
 975             validate_atomic_type!(fx, intrinsic, span, val.layout().ty);
 976
 977             crate::atomic_shim::lock_global_lock(fx);
 978
 979             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 980             dest.write_cvalue(fx, val);
 981
 982             crate::atomic_shim::unlock_global_lock(fx);
 983         };
 984         _ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
 985             validate_atomic_type!(fx, intrinsic, span, T);
 986
 987             crate::atomic_shim::lock_global_lock(fx);
 988
 989             // Read old
 990             let clif_ty = fx.clif_type(T).unwrap();
 991             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 992             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
 993
 994             // Write new
 995             let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout());
 996             dest.write_cvalue(fx, src);
 997
 998             crate::atomic_shim::unlock_global_lock(fx);
 999         };
1000         _ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
1001             validate_atomic_type!(fx, intrinsic, span, T);
1002
1003             let test_old = test_old.load_scalar(fx);
1004             let new = new.load_scalar(fx);
1005
1006             crate::atomic_shim::lock_global_lock(fx);
1007
1008             // Read old
1009             let clif_ty = fx.clif_type(T).unwrap();
1010             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
1011
1012             // Compare
1013             let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
1014             let new = fx.bcx.ins().select(is_eq, new, old); // Keep old if not equal to test_old
1015
1016             // Write new
1017             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
1018
1019             let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
1020             ret.write_cvalue(fx, ret_val);
1021
1022             crate::atomic_shim::unlock_global_lock(fx);
1023         };
1024
1025         _ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, c amount) {
1026             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1027             let amount = amount.load_scalar(fx);
1028             atomic_binop_return_old! (fx, iadd<T>(ptr, amount) -> ret);
1029         };
1030         _ if intrinsic.starts_with("atomic_xsub"), <T> (v ptr, c amount) {
1031             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1032             let amount = amount.load_scalar(fx);
1033             atomic_binop_return_old! (fx, isub<T>(ptr, amount) -> ret);
1034         };
1035         _ if intrinsic.starts_with("atomic_and"), <T> (v ptr, c src) {
1036             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1037             let src = src.load_scalar(fx);
1038             atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
1039         };
1040         _ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, c src) {
1041             validate_atomic_type!(fx, intrinsic, span, T);
1042
1043             let src = src.load_scalar(fx);
1044
1045             crate::atomic_shim::lock_global_lock(fx);
1046
1047             let clif_ty = fx.clif_type(T).unwrap();
1048             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
1049             let and = fx.bcx.ins().band(old, src);
1050             let new = fx.bcx.ins().bnot(and);
1051             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
1052             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
1053
1054             crate::atomic_shim::unlock_global_lock(fx);
1055         };
1056         _ if intrinsic.starts_with("atomic_or"), <T> (v ptr, c src) {
1057             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1058             let src = src.load_scalar(fx);
1059             atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);
1060         };
1061         _ if intrinsic.starts_with("atomic_xor"), <T> (v ptr, c src) {
1062             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1063             let src = src.load_scalar(fx);
1064             atomic_binop_return_old! (fx, bxor<T>(ptr, src) -> ret);
1065         };
1066
1067         _ if intrinsic.starts_with("atomic_max"), <T> (v ptr, c src) {
1068             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1069             let src = src.load_scalar(fx);
1070             atomic_minmax!(fx, IntCC::SignedGreaterThan, <T> (ptr, src) -> ret);
1071         };
1072         _ if intrinsic.starts_with("atomic_umax"), <T> (v ptr, c src) {
1073             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1074             let src = src.load_scalar(fx);
1075             atomic_minmax!(fx, IntCC::UnsignedGreaterThan, <T> (ptr, src) -> ret);
1076         };
1077         _ if intrinsic.starts_with("atomic_min"), <T> (v ptr, c src) {
1078             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1079             let src = src.load_scalar(fx);
1080             atomic_minmax!(fx, IntCC::SignedLessThan, <T> (ptr, src) -> ret);
1081         };
1082         _ if intrinsic.starts_with("atomic_umin"), <T> (v ptr, c src) {
1083             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1084             let src = src.load_scalar(fx);
1085             atomic_minmax!(fx, IntCC::UnsignedLessThan, <T> (ptr, src) -> ret);
1086         };
1087
1088         minnumf32, (v a, v b) {
1089             let val = fx.bcx.ins().fmin(a, b);
1090             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1091             ret.write_cvalue(fx, val);
1092         };
1093         minnumf64, (v a, v b) {
1094             let val = fx.bcx.ins().fmin(a, b);
1095             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1096             ret.write_cvalue(fx, val);
1097         };
1098         maxnumf32, (v a, v b) {
1099             let val = fx.bcx.ins().fmax(a, b);
1100             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1101             ret.write_cvalue(fx, val);
1102         };
1103         maxnumf64, (v a, v b) {
1104             let val = fx.bcx.ins().fmax(a, b);
1105             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1106             ret.write_cvalue(fx, val);
1107         };
1108
1109         try, (v f, v data, v _catch_fn) {
1110             // FIXME once unwinding is supported, change this to actually catch panics
1111             let f_sig = fx.bcx.func.import_signature(Signature {
1112                 call_conv: CallConv::triple_default(fx.triple()),
1113                 params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
1114                 returns: vec![],
1115             });
1116
1117             fx.bcx.ins().call_indirect(f_sig, f, &[data]);
1118
1119             let layout = ret.layout();
1120             let ret_val = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size));
1121             ret.write_cvalue(fx, ret_val);
1122         };
1123
1124         fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) {
1125             let res = crate::num::codegen_float_binop(fx, match intrinsic {
1126                 "fadd_fast" => BinOp::Add,
1127                 "fsub_fast" => BinOp::Sub,
1128                 "fmul_fast" => BinOp::Mul,
1129                 "fdiv_fast" => BinOp::Div,
1130                 "frem_fast" => BinOp::Rem,
1131                 _ => unreachable!(),
1132             }, x, y);
1133             ret.write_cvalue(fx, res);
1134         };
1135         float_to_int_unchecked, (v f) {
1136             let res = crate::cast::clif_int_or_float_cast(
1137                 fx,
1138                 f,
1139                 false,
1140                 fx.clif_type(ret.layout().ty).unwrap(),
1141                 type_sign(ret.layout().ty),
1142             );
1143             ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
1144         };
1145     }
1146
1147     if let Some((_, dest)) = destination {
1148         let ret_block = fx.get_block(dest);
1149         fx.bcx.ins().jump(ret_block, &[]);
1150     } else {
1151         trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
1152     }
1153 }