compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

   1 //! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"`
   2 //! and LLVM intrinsics that have symbol names starting with `llvm.`.
   3
   4 macro_rules! intrinsic_args {
   5     ($fx:expr, $args:expr => ($($arg:tt),*); $intrinsic:expr) => {
   6         #[allow(unused_parens)]
   7         let ($($arg),*) = if let [$($arg),*] = $args {
   8             ($(codegen_operand($fx, $arg)),*)
   9         } else {
  10             $crate::intrinsics::bug_on_incorrect_arg_count($intrinsic);
  11         };
  12     }
  13 }
  14
  15 mod cpuid;
  16 mod llvm;
  17 mod simd;
  18
  19 pub(crate) use cpuid::codegen_cpuid_call;
  20 pub(crate) use llvm::codegen_llvm_intrinsic_call;
  21
  22 use rustc_middle::ty::print::with_no_trimmed_paths;
  23 use rustc_middle::ty::subst::SubstsRef;
  24 use rustc_span::symbol::{kw, sym, Symbol};
  25
  26 use crate::prelude::*;
  27 use cranelift_codegen::ir::AtomicRmwOp;
  28
  29 fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! {
  30     bug!("wrong number of args for intrinsic {}", intrinsic);
  31 }
  32
  33 fn report_atomic_type_validation_error<'tcx>(
  34     fx: &mut FunctionCx<'_, '_, 'tcx>,
  35     intrinsic: Symbol,
  36     span: Span,
  37     ty: Ty<'tcx>,
  38 ) {
  39     fx.tcx.sess.span_err(
  40         span,
  41         &format!(
  42             "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`",
  43             intrinsic, ty
  44         ),
  45     );
  46     // Prevent verifier error
  47     fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
  48 }
  49
  50 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
  51     let (element, count) = match layout.abi {
  52         Abi::Vector { element, count } => (element, count),
  53         _ => unreachable!(),
  54     };
  55
  56     match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) {
  57         // Cranelift currently only implements icmp for 128bit vectors.
  58         Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
  59         _ => None,
  60     }
  61 }
  62
  63 fn simd_for_each_lane<'tcx>(
  64     fx: &mut FunctionCx<'_, '_, 'tcx>,
  65     val: CValue<'tcx>,
  66     ret: CPlace<'tcx>,
  67     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value) -> Value,
  68 ) {
  69     let layout = val.layout();
  70
  71     let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
  72     let lane_layout = fx.layout_of(lane_ty);
  73     let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
  74     let ret_lane_layout = fx.layout_of(ret_lane_ty);
  75     assert_eq!(lane_count, ret_lane_count);
  76
  77     for lane_idx in 0..lane_count {
  78         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
  79
  80         let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, lane);
  81         let res_lane = CValue::by_val(res_lane, ret_lane_layout);
  82
  83         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
  84     }
  85 }
  86
  87 fn simd_pair_for_each_lane<'tcx>(
  88     fx: &mut FunctionCx<'_, '_, 'tcx>,
  89     x: CValue<'tcx>,
  90     y: CValue<'tcx>,
  91     ret: CPlace<'tcx>,
  92     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value) -> Value,
  93 ) {
  94     assert_eq!(x.layout(), y.layout());
  95     let layout = x.layout();
  96
  97     let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
  98     let lane_layout = fx.layout_of(lane_ty);
  99     let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
 100     let ret_lane_layout = fx.layout_of(ret_lane_ty);
 101     assert_eq!(lane_count, ret_lane_count);
 102
 103     for lane_idx in 0..lane_count {
 104         let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx);
 105         let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx);
 106
 107         let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, x_lane, y_lane);
 108         let res_lane = CValue::by_val(res_lane, ret_lane_layout);
 109
 110         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
 111     }
 112 }
 113
 114 fn simd_reduce<'tcx>(
 115     fx: &mut FunctionCx<'_, '_, 'tcx>,
 116     val: CValue<'tcx>,
 117     acc: Option<Value>,
 118     ret: CPlace<'tcx>,
 119     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Value, Value) -> Value,
 120 ) {
 121     let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
 122     let lane_layout = fx.layout_of(lane_ty);
 123     assert_eq!(lane_layout, ret.layout());
 124
 125     let (mut res_val, start_lane) =
 126         if let Some(acc) = acc { (acc, 0) } else { (val.value_lane(fx, 0).load_scalar(fx), 1) };
 127     for lane_idx in start_lane..lane_count {
 128         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
 129         res_val = f(fx, lane_layout.ty, res_val, lane);
 130     }
 131     let res = CValue::by_val(res_val, lane_layout);
 132     ret.write_cvalue(fx, res);
 133 }
 134
 135 // FIXME move all uses to `simd_reduce`
 136 fn simd_reduce_bool<'tcx>(
 137     fx: &mut FunctionCx<'_, '_, 'tcx>,
 138     val: CValue<'tcx>,
 139     ret: CPlace<'tcx>,
 140     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
 141 ) {
 142     let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
 143     assert!(ret.layout().ty.is_bool());
 144
 145     let res_val = val.value_lane(fx, 0).load_scalar(fx);
 146     let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
 147     for lane_idx in 1..lane_count {
 148         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
 149         let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
 150         res_val = f(fx, res_val, lane);
 151     }
 152     let res_val = if fx.bcx.func.dfg.value_type(res_val) != types::I8 {
 153         fx.bcx.ins().ireduce(types::I8, res_val)
 154     } else {
 155         res_val
 156     };
 157     let res = CValue::by_val(res_val, ret.layout());
 158     ret.write_cvalue(fx, res);
 159 }
 160
 161 fn bool_to_zero_or_max_uint<'tcx>(
 162     fx: &mut FunctionCx<'_, '_, 'tcx>,
 163     ty: Ty<'tcx>,
 164     val: Value,
 165 ) -> Value {
 166     let ty = fx.clif_type(ty).unwrap();
 167
 168     let int_ty = match ty {
 169         types::F32 => types::I32,
 170         types::F64 => types::I64,
 171         ty => ty,
 172     };
 173
 174     let val = fx.bcx.ins().bint(int_ty, val);
 175     let mut res = fx.bcx.ins().ineg(val);
 176
 177     if ty.is_float() {
 178         res = fx.bcx.ins().bitcast(ty, res);
 179     }
 180
 181     res
 182 }
 183
 184 pub(crate) fn codegen_intrinsic_call<'tcx>(
 185     fx: &mut FunctionCx<'_, '_, 'tcx>,
 186     instance: Instance<'tcx>,
 187     args: &[mir::Operand<'tcx>],
 188     destination: CPlace<'tcx>,
 189     target: Option<BasicBlock>,
 190     source_info: mir::SourceInfo,
 191 ) {
 192     let intrinsic = fx.tcx.item_name(instance.def_id());
 193     let substs = instance.substs;
 194
 195     let target = if let Some(target) = target {
 196         target
 197     } else {
 198         // Insert non returning intrinsics here
 199         match intrinsic {
 200             sym::abort => {
 201                 fx.bcx.ins().trap(TrapCode::User(0));
 202             }
 203             sym::transmute => {
 204                 crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
 205             }
 206             _ => unimplemented!("unsupported intrinsic {}", intrinsic),
 207         }
 208         return;
 209     };
 210
 211     if intrinsic.as_str().starts_with("simd_") {
 212         self::simd::codegen_simd_intrinsic_call(
 213             fx,
 214             intrinsic,
 215             substs,
 216             args,
 217             destination,
 218             source_info.span,
 219         );
 220         let ret_block = fx.get_block(target);
 221         fx.bcx.ins().jump(ret_block, &[]);
 222     } else if codegen_float_intrinsic_call(fx, intrinsic, args, destination) {
 223         let ret_block = fx.get_block(target);
 224         fx.bcx.ins().jump(ret_block, &[]);
 225     } else {
 226         codegen_regular_intrinsic_call(
 227             fx,
 228             instance,
 229             intrinsic,
 230             substs,
 231             args,
 232             destination,
 233             Some(target),
 234             source_info,
 235         );
 236     }
 237 }
 238
 239 fn codegen_float_intrinsic_call<'tcx>(
 240     fx: &mut FunctionCx<'_, '_, 'tcx>,
 241     intrinsic: Symbol,
 242     args: &[mir::Operand<'tcx>],
 243     ret: CPlace<'tcx>,
 244 ) -> bool {
 245     let (name, arg_count, ty) = match intrinsic {
 246         sym::expf32 => ("expf", 1, fx.tcx.types.f32),
 247         sym::expf64 => ("exp", 1, fx.tcx.types.f64),
 248         sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32),
 249         sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64),
 250         sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32),
 251         sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64),
 252         sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32), // compiler-builtins
 253         sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64), // compiler-builtins
 254         sym::powf32 => ("powf", 2, fx.tcx.types.f32),
 255         sym::powf64 => ("pow", 2, fx.tcx.types.f64),
 256         sym::logf32 => ("logf", 1, fx.tcx.types.f32),
 257         sym::logf64 => ("log", 1, fx.tcx.types.f64),
 258         sym::log2f32 => ("log2f", 1, fx.tcx.types.f32),
 259         sym::log2f64 => ("log2", 1, fx.tcx.types.f64),
 260         sym::log10f32 => ("log10f", 1, fx.tcx.types.f32),
 261         sym::log10f64 => ("log10", 1, fx.tcx.types.f64),
 262         sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32),
 263         sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64),
 264         sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32),
 265         sym::fmaf64 => ("fma", 3, fx.tcx.types.f64),
 266         sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32),
 267         sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64),
 268         sym::floorf32 => ("floorf", 1, fx.tcx.types.f32),
 269         sym::floorf64 => ("floor", 1, fx.tcx.types.f64),
 270         sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32),
 271         sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64),
 272         sym::truncf32 => ("truncf", 1, fx.tcx.types.f32),
 273         sym::truncf64 => ("trunc", 1, fx.tcx.types.f64),
 274         sym::roundf32 => ("roundf", 1, fx.tcx.types.f32),
 275         sym::roundf64 => ("round", 1, fx.tcx.types.f64),
 276         sym::sinf32 => ("sinf", 1, fx.tcx.types.f32),
 277         sym::sinf64 => ("sin", 1, fx.tcx.types.f64),
 278         sym::cosf32 => ("cosf", 1, fx.tcx.types.f32),
 279         sym::cosf64 => ("cos", 1, fx.tcx.types.f64),
 280         _ => return false,
 281     };
 282
 283     if args.len() != arg_count {
 284         bug!("wrong number of args for intrinsic {:?}", intrinsic);
 285     }
 286
 287     let (a, b, c);
 288     let args = match args {
 289         [x] => {
 290             a = [codegen_operand(fx, x)];
 291             &a as &[_]
 292         }
 293         [x, y] => {
 294             b = [codegen_operand(fx, x), codegen_operand(fx, y)];
 295             &b
 296         }
 297         [x, y, z] => {
 298             c = [codegen_operand(fx, x), codegen_operand(fx, y), codegen_operand(fx, z)];
 299             &c
 300         }
 301         _ => unreachable!(),
 302     };
 303
 304     let layout = fx.layout_of(ty);
 305     let res = match intrinsic {
 306         sym::fmaf32 | sym::fmaf64 => {
 307             let a = args[0].load_scalar(fx);
 308             let b = args[1].load_scalar(fx);
 309             let c = args[2].load_scalar(fx);
 310             CValue::by_val(fx.bcx.ins().fma(a, b, c), layout)
 311         }
 312         sym::copysignf32 | sym::copysignf64 => {
 313             let a = args[0].load_scalar(fx);
 314             let b = args[1].load_scalar(fx);
 315             CValue::by_val(fx.bcx.ins().fcopysign(a, b), layout)
 316         }
 317         sym::fabsf32
 318         | sym::fabsf64
 319         | sym::floorf32
 320         | sym::floorf64
 321         | sym::ceilf32
 322         | sym::ceilf64
 323         | sym::truncf32
 324         | sym::truncf64 => {
 325             let a = args[0].load_scalar(fx);
 326
 327             let val = match intrinsic {
 328                 sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(a),
 329                 sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(a),
 330                 sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(a),
 331                 sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(a),
 332                 _ => unreachable!(),
 333             };
 334
 335             CValue::by_val(val, layout)
 336         }
 337         // These intrinsics aren't supported natively by Cranelift.
 338         // Lower them to a libcall.
 339         _ => fx.easy_call(name, &args, ty),
 340     };
 341
 342     ret.write_cvalue(fx, res);
 343
 344     true
 345 }
 346
 347 fn codegen_regular_intrinsic_call<'tcx>(
 348     fx: &mut FunctionCx<'_, '_, 'tcx>,
 349     instance: Instance<'tcx>,
 350     intrinsic: Symbol,
 351     substs: SubstsRef<'tcx>,
 352     args: &[mir::Operand<'tcx>],
 353     ret: CPlace<'tcx>,
 354     destination: Option<BasicBlock>,
 355     source_info: mir::SourceInfo,
 356 ) {
 357     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 358
 359     match intrinsic {
 360         sym::assume => {
 361             intrinsic_args!(fx, args => (_a); intrinsic);
 362         }
 363         sym::likely | sym::unlikely => {
 364             intrinsic_args!(fx, args => (a); intrinsic);
 365
 366             ret.write_cvalue(fx, a);
 367         }
 368         sym::breakpoint => {
 369             intrinsic_args!(fx, args => (); intrinsic);
 370
 371             fx.bcx.ins().debugtrap();
 372         }
 373         sym::copy | sym::copy_nonoverlapping => {
 374             intrinsic_args!(fx, args => (src, dst, count); intrinsic);
 375             let src = src.load_scalar(fx);
 376             let dst = dst.load_scalar(fx);
 377             let count = count.load_scalar(fx);
 378
 379             let elem_ty = substs.type_at(0);
 380             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 381             assert_eq!(args.len(), 3);
 382             let byte_amount =
 383                 if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
 384
 385             if intrinsic == sym::copy_nonoverlapping {
 386                 // FIXME emit_small_memcpy
 387                 fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
 388             } else {
 389                 // FIXME emit_small_memmove
 390                 fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
 391             }
 392         }
 393         sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => {
 394             // NOTE: the volatile variants have src and dst swapped
 395             intrinsic_args!(fx, args => (dst, src, count); intrinsic);
 396             let dst = dst.load_scalar(fx);
 397             let src = src.load_scalar(fx);
 398             let count = count.load_scalar(fx);
 399
 400             let elem_ty = substs.type_at(0);
 401             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 402             assert_eq!(args.len(), 3);
 403             let byte_amount =
 404                 if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
 405
 406             // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
 407             if intrinsic == sym::volatile_copy_nonoverlapping_memory {
 408                 // FIXME emit_small_memcpy
 409                 fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
 410             } else {
 411                 // FIXME emit_small_memmove
 412                 fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
 413             }
 414         }
 415         sym::size_of_val => {
 416             intrinsic_args!(fx, args => (ptr); intrinsic);
 417
 418             let layout = fx.layout_of(substs.type_at(0));
 419             // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
 420             // branch
 421             let size = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
 422                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 423                 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 424                 size
 425             } else {
 426                 fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64)
 427             };
 428             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 429         }
 430         sym::min_align_of_val => {
 431             intrinsic_args!(fx, args => (ptr); intrinsic);
 432
 433             let layout = fx.layout_of(substs.type_at(0));
 434             // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
 435             // branch
 436             let align = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
 437                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 438                 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 439                 align
 440             } else {
 441                 fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
 442             };
 443             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 444         }
 445
 446         sym::vtable_size => {
 447             intrinsic_args!(fx, args => (vtable); intrinsic);
 448             let vtable = vtable.load_scalar(fx);
 449
 450             let size = crate::vtable::size_of_obj(fx, vtable);
 451             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 452         }
 453
 454         sym::vtable_align => {
 455             intrinsic_args!(fx, args => (vtable); intrinsic);
 456             let vtable = vtable.load_scalar(fx);
 457
 458             let align = crate::vtable::min_align_of_obj(fx, vtable);
 459             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 460         }
 461
 462         sym::unchecked_add
 463         | sym::unchecked_sub
 464         | sym::unchecked_mul
 465         | sym::unchecked_div
 466         | sym::exact_div
 467         | sym::unchecked_rem
 468         | sym::unchecked_shl
 469         | sym::unchecked_shr => {
 470             intrinsic_args!(fx, args => (x, y); intrinsic);
 471
 472             // FIXME trap on overflow
 473             let bin_op = match intrinsic {
 474                 sym::unchecked_add => BinOp::Add,
 475                 sym::unchecked_sub => BinOp::Sub,
 476                 sym::unchecked_mul => BinOp::Mul,
 477                 sym::unchecked_div | sym::exact_div => BinOp::Div,
 478                 sym::unchecked_rem => BinOp::Rem,
 479                 sym::unchecked_shl => BinOp::Shl,
 480                 sym::unchecked_shr => BinOp::Shr,
 481                 _ => unreachable!(),
 482             };
 483             let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
 484             ret.write_cvalue(fx, res);
 485         }
 486         sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
 487             intrinsic_args!(fx, args => (x, y); intrinsic);
 488
 489             assert_eq!(x.layout().ty, y.layout().ty);
 490             let bin_op = match intrinsic {
 491                 sym::add_with_overflow => BinOp::Add,
 492                 sym::sub_with_overflow => BinOp::Sub,
 493                 sym::mul_with_overflow => BinOp::Mul,
 494                 _ => unreachable!(),
 495             };
 496
 497             let res = crate::num::codegen_checked_int_binop(fx, bin_op, x, y);
 498             ret.write_cvalue(fx, res);
 499         }
 500         sym::saturating_add | sym::saturating_sub => {
 501             intrinsic_args!(fx, args => (lhs, rhs); intrinsic);
 502
 503             assert_eq!(lhs.layout().ty, rhs.layout().ty);
 504             let bin_op = match intrinsic {
 505                 sym::saturating_add => BinOp::Add,
 506                 sym::saturating_sub => BinOp::Sub,
 507                 _ => unreachable!(),
 508             };
 509
 510             let signed = type_sign(lhs.layout().ty);
 511
 512             let checked_res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs);
 513
 514             let (val, has_overflow) = checked_res.load_scalar_pair(fx);
 515             let clif_ty = fx.clif_type(lhs.layout().ty).unwrap();
 516
 517             let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
 518
 519             let val = match (intrinsic, signed) {
 520                 (sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val),
 521                 (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val),
 522                 (sym::saturating_add, true) => {
 523                     let rhs = rhs.load_scalar(fx);
 524                     let rhs_ge_zero =
 525                         fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 526                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
 527                     fx.bcx.ins().select(has_overflow, sat_val, val)
 528                 }
 529                 (sym::saturating_sub, true) => {
 530                     let rhs = rhs.load_scalar(fx);
 531                     let rhs_ge_zero =
 532                         fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 533                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
 534                     fx.bcx.ins().select(has_overflow, sat_val, val)
 535                 }
 536                 _ => unreachable!(),
 537             };
 538
 539             let res = CValue::by_val(val, lhs.layout());
 540
 541             ret.write_cvalue(fx, res);
 542         }
 543         sym::rotate_left => {
 544             intrinsic_args!(fx, args => (x, y); intrinsic);
 545             let y = y.load_scalar(fx);
 546
 547             let layout = x.layout();
 548             let x = x.load_scalar(fx);
 549             let res = fx.bcx.ins().rotl(x, y);
 550             ret.write_cvalue(fx, CValue::by_val(res, layout));
 551         }
 552         sym::rotate_right => {
 553             intrinsic_args!(fx, args => (x, y); intrinsic);
 554             let y = y.load_scalar(fx);
 555
 556             let layout = x.layout();
 557             let x = x.load_scalar(fx);
 558             let res = fx.bcx.ins().rotr(x, y);
 559             ret.write_cvalue(fx, CValue::by_val(res, layout));
 560         }
 561
 562         // The only difference between offset and arith_offset is regarding UB. Because Cranelift
 563         // doesn't have UB both are codegen'ed the same way
 564         sym::offset | sym::arith_offset => {
 565             intrinsic_args!(fx, args => (base, offset); intrinsic);
 566             let offset = offset.load_scalar(fx);
 567
 568             let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
 569             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 570             let ptr_diff = if pointee_size != 1 {
 571                 fx.bcx.ins().imul_imm(offset, pointee_size as i64)
 572             } else {
 573                 offset
 574             };
 575             let base_val = base.load_scalar(fx);
 576             let res = fx.bcx.ins().iadd(base_val, ptr_diff);
 577             ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
 578         }
 579
 580         sym::ptr_mask => {
 581             intrinsic_args!(fx, args => (ptr, mask); intrinsic);
 582             let ptr = ptr.load_scalar(fx);
 583             let mask = mask.load_scalar(fx);
 584             fx.bcx.ins().band(ptr, mask);
 585         }
 586
 587         sym::transmute => {
 588             intrinsic_args!(fx, args => (from); intrinsic);
 589
 590             ret.write_cvalue_transmute(fx, from);
 591         }
 592         sym::write_bytes | sym::volatile_set_memory => {
 593             intrinsic_args!(fx, args => (dst, val, count); intrinsic);
 594             let val = val.load_scalar(fx);
 595             let count = count.load_scalar(fx);
 596
 597             let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
 598             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 599             let count = if pointee_size != 1 {
 600                 fx.bcx.ins().imul_imm(count, pointee_size as i64)
 601             } else {
 602                 count
 603             };
 604             let dst_ptr = dst.load_scalar(fx);
 605             // FIXME make the memset actually volatile when switching to emit_small_memset
 606             // FIXME use emit_small_memset
 607             fx.bcx.call_memset(fx.target_config, dst_ptr, val, count);
 608         }
 609         sym::ctlz | sym::ctlz_nonzero => {
 610             intrinsic_args!(fx, args => (arg); intrinsic);
 611             let val = arg.load_scalar(fx);
 612
 613             // FIXME trap on `ctlz_nonzero` with zero arg.
 614             let res = fx.bcx.ins().clz(val);
 615             let res = CValue::by_val(res, arg.layout());
 616             ret.write_cvalue(fx, res);
 617         }
 618         sym::cttz | sym::cttz_nonzero => {
 619             intrinsic_args!(fx, args => (arg); intrinsic);
 620             let val = arg.load_scalar(fx);
 621
 622             // FIXME trap on `cttz_nonzero` with zero arg.
 623             let res = fx.bcx.ins().ctz(val);
 624             let res = CValue::by_val(res, arg.layout());
 625             ret.write_cvalue(fx, res);
 626         }
 627         sym::ctpop => {
 628             intrinsic_args!(fx, args => (arg); intrinsic);
 629             let val = arg.load_scalar(fx);
 630
 631             let res = fx.bcx.ins().popcnt(val);
 632             let res = CValue::by_val(res, arg.layout());
 633             ret.write_cvalue(fx, res);
 634         }
 635         sym::bitreverse => {
 636             intrinsic_args!(fx, args => (arg); intrinsic);
 637             let val = arg.load_scalar(fx);
 638
 639             let res = fx.bcx.ins().bitrev(val);
 640             let res = CValue::by_val(res, arg.layout());
 641             ret.write_cvalue(fx, res);
 642         }
 643         sym::bswap => {
 644             // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
 645             fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value {
 646                 match bcx.func.dfg.value_type(v) {
 647                     types::I8 => v,
 648
 649                     // https://code.woboq.org/gcc/include/bits/byteswap.h.html
 650                     types::I16 => {
 651                         let tmp1 = bcx.ins().ishl_imm(v, 8);
 652                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00);
 653
 654                         let tmp2 = bcx.ins().ushr_imm(v, 8);
 655                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF);
 656
 657                         bcx.ins().bor(n1, n2)
 658                     }
 659                     types::I32 => {
 660                         let tmp1 = bcx.ins().ishl_imm(v, 24);
 661                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000);
 662
 663                         let tmp2 = bcx.ins().ishl_imm(v, 8);
 664                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000);
 665
 666                         let tmp3 = bcx.ins().ushr_imm(v, 8);
 667                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00);
 668
 669                         let tmp4 = bcx.ins().ushr_imm(v, 24);
 670                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF);
 671
 672                         let or_tmp1 = bcx.ins().bor(n1, n2);
 673                         let or_tmp2 = bcx.ins().bor(n3, n4);
 674                         bcx.ins().bor(or_tmp1, or_tmp2)
 675                     }
 676                     types::I64 => {
 677                         let tmp1 = bcx.ins().ishl_imm(v, 56);
 678                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64);
 679
 680                         let tmp2 = bcx.ins().ishl_imm(v, 40);
 681                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64);
 682
 683                         let tmp3 = bcx.ins().ishl_imm(v, 24);
 684                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64);
 685
 686                         let tmp4 = bcx.ins().ishl_imm(v, 8);
 687                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64);
 688
 689                         let tmp5 = bcx.ins().ushr_imm(v, 8);
 690                         let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64);
 691
 692                         let tmp6 = bcx.ins().ushr_imm(v, 24);
 693                         let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64);
 694
 695                         let tmp7 = bcx.ins().ushr_imm(v, 40);
 696                         let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64);
 697
 698                         let tmp8 = bcx.ins().ushr_imm(v, 56);
 699                         let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64);
 700
 701                         let or_tmp1 = bcx.ins().bor(n1, n2);
 702                         let or_tmp2 = bcx.ins().bor(n3, n4);
 703                         let or_tmp3 = bcx.ins().bor(n5, n6);
 704                         let or_tmp4 = bcx.ins().bor(n7, n8);
 705
 706                         let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2);
 707                         let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4);
 708                         bcx.ins().bor(or_tmp5, or_tmp6)
 709                     }
 710                     types::I128 => {
 711                         let (lo, hi) = bcx.ins().isplit(v);
 712                         let lo = swap(bcx, lo);
 713                         let hi = swap(bcx, hi);
 714                         bcx.ins().iconcat(hi, lo)
 715                     }
 716                     ty => unreachable!("bswap {}", ty),
 717                 }
 718             }
 719             intrinsic_args!(fx, args => (arg); intrinsic);
 720             let val = arg.load_scalar(fx);
 721
 722             let res = CValue::by_val(swap(&mut fx.bcx, val), arg.layout());
 723             ret.write_cvalue(fx, res);
 724         }
 725         sym::assert_inhabited | sym::assert_zero_valid | sym::assert_uninit_valid => {
 726             intrinsic_args!(fx, args => (); intrinsic);
 727
 728             let layout = fx.layout_of(substs.type_at(0));
 729             if layout.abi.is_uninhabited() {
 730                 with_no_trimmed_paths!({
 731                     crate::base::codegen_panic(
 732                         fx,
 733                         &format!("attempted to instantiate uninhabited type `{}`", layout.ty),
 734                         source_info,
 735                     )
 736                 });
 737                 return;
 738             }
 739
 740             if intrinsic == sym::assert_zero_valid && !fx.tcx.permits_zero_init(layout) {
 741                 with_no_trimmed_paths!({
 742                     crate::base::codegen_panic(
 743                         fx,
 744                         &format!(
 745                             "attempted to zero-initialize type `{}`, which is invalid",
 746                             layout.ty
 747                         ),
 748                         source_info,
 749                     );
 750                 });
 751                 return;
 752             }
 753
 754             if intrinsic == sym::assert_uninit_valid && !fx.tcx.permits_uninit_init(layout) {
 755                 with_no_trimmed_paths!({
 756                     crate::base::codegen_panic(
 757                         fx,
 758                         &format!(
 759                             "attempted to leave type `{}` uninitialized, which is invalid",
 760                             layout.ty
 761                         ),
 762                         source_info,
 763                     )
 764                 });
 765                 return;
 766             }
 767         }
 768
 769         sym::volatile_load | sym::unaligned_volatile_load => {
 770             intrinsic_args!(fx, args => (ptr); intrinsic);
 771
 772             // Cranelift treats loads as volatile by default
 773             // FIXME correctly handle unaligned_volatile_load
 774             let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 775             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 776             ret.write_cvalue(fx, val);
 777         }
 778         sym::volatile_store | sym::unaligned_volatile_store => {
 779             intrinsic_args!(fx, args => (ptr, val); intrinsic);
 780             let ptr = ptr.load_scalar(fx);
 781
 782             // Cranelift treats stores as volatile by default
 783             // FIXME correctly handle unaligned_volatile_store
 784             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 785             dest.write_cvalue(fx, val);
 786         }
 787
 788         sym::pref_align_of
 789         | sym::needs_drop
 790         | sym::type_id
 791         | sym::type_name
 792         | sym::variant_count => {
 793             intrinsic_args!(fx, args => (); intrinsic);
 794
 795             let const_val =
 796                 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
 797             let val = crate::constant::codegen_const_value(fx, const_val, ret.layout().ty);
 798             ret.write_cvalue(fx, val);
 799         }
 800
 801         sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
 802             intrinsic_args!(fx, args => (ptr, base); intrinsic);
 803             let ptr = ptr.load_scalar(fx);
 804             let base = base.load_scalar(fx);
 805             let ty = substs.type_at(0);
 806
 807             let pointee_size: u64 = fx.layout_of(ty).size.bytes();
 808             let diff_bytes = fx.bcx.ins().isub(ptr, base);
 809             // FIXME this can be an exact division.
 810             let val = if intrinsic == sym::ptr_offset_from_unsigned {
 811                 let usize_layout = fx.layout_of(fx.tcx.types.usize);
 812                 // Because diff_bytes ULE isize::MAX, this would be fine as signed,
 813                 // but unsigned is slightly easier to codegen, so might as well.
 814                 CValue::by_val(fx.bcx.ins().udiv_imm(diff_bytes, pointee_size as i64), usize_layout)
 815             } else {
 816                 let isize_layout = fx.layout_of(fx.tcx.types.isize);
 817                 CValue::by_val(fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64), isize_layout)
 818             };
 819             ret.write_cvalue(fx, val);
 820         }
 821
 822         sym::ptr_guaranteed_eq => {
 823             intrinsic_args!(fx, args => (a, b); intrinsic);
 824
 825             let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b);
 826             ret.write_cvalue(fx, val);
 827         }
 828
 829         sym::ptr_guaranteed_ne => {
 830             intrinsic_args!(fx, args => (a, b); intrinsic);
 831
 832             let val = crate::num::codegen_ptr_binop(fx, BinOp::Ne, a, b);
 833             ret.write_cvalue(fx, val);
 834         }
 835
 836         sym::caller_location => {
 837             intrinsic_args!(fx, args => (); intrinsic);
 838
 839             let caller_location = fx.get_caller_location(source_info);
 840             ret.write_cvalue(fx, caller_location);
 841         }
 842
 843         _ if intrinsic.as_str().starts_with("atomic_fence") => {
 844             intrinsic_args!(fx, args => (); intrinsic);
 845
 846             fx.bcx.ins().fence();
 847         }
 848         _ if intrinsic.as_str().starts_with("atomic_singlethreadfence") => {
 849             intrinsic_args!(fx, args => (); intrinsic);
 850
 851             // FIXME use a compiler fence once Cranelift supports it
 852             fx.bcx.ins().fence();
 853         }
 854         _ if intrinsic.as_str().starts_with("atomic_load") => {
 855             intrinsic_args!(fx, args => (ptr); intrinsic);
 856             let ptr = ptr.load_scalar(fx);
 857
 858             let ty = substs.type_at(0);
 859             match ty.kind() {
 860                 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
 861                     // FIXME implement 128bit atomics
 862                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
 863                         // special case for compiler-builtins to avoid having to patch it
 864                         crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
 865                         return;
 866                     } else {
 867                         fx.tcx
 868                             .sess
 869                             .span_fatal(source_info.span, "128bit atomics not yet supported");
 870                     }
 871                 }
 872                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 873                 _ => {
 874                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
 875                     return;
 876                 }
 877             }
 878             let clif_ty = fx.clif_type(ty).unwrap();
 879
 880             let val = fx.bcx.ins().atomic_load(clif_ty, MemFlags::trusted(), ptr);
 881
 882             let val = CValue::by_val(val, fx.layout_of(ty));
 883             ret.write_cvalue(fx, val);
 884         }
 885         _ if intrinsic.as_str().starts_with("atomic_store") => {
 886             intrinsic_args!(fx, args => (ptr, val); intrinsic);
 887             let ptr = ptr.load_scalar(fx);
 888
 889             let ty = substs.type_at(0);
 890             match ty.kind() {
 891                 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
 892                     // FIXME implement 128bit atomics
 893                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
 894                         // special case for compiler-builtins to avoid having to patch it
 895                         crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
 896                         return;
 897                     } else {
 898                         fx.tcx
 899                             .sess
 900                             .span_fatal(source_info.span, "128bit atomics not yet supported");
 901                     }
 902                 }
 903                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 904                 _ => {
 905                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
 906                     return;
 907                 }
 908             }
 909
 910             let val = val.load_scalar(fx);
 911
 912             fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
 913         }
 914         _ if intrinsic.as_str().starts_with("atomic_xchg") => {
 915             intrinsic_args!(fx, args => (ptr, new); intrinsic);
 916             let ptr = ptr.load_scalar(fx);
 917
 918             let layout = new.layout();
 919             match layout.ty.kind() {
 920                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 921                 _ => {
 922                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 923                     return;
 924                 }
 925             }
 926             let ty = fx.clif_type(layout.ty).unwrap();
 927
 928             let new = new.load_scalar(fx);
 929
 930             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new);
 931
 932             let old = CValue::by_val(old, layout);
 933             ret.write_cvalue(fx, old);
 934         }
 935         _ if intrinsic.as_str().starts_with("atomic_cxchg") => {
 936             // both atomic_cxchg_* and atomic_cxchgweak_*
 937             intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic);
 938             let ptr = ptr.load_scalar(fx);
 939
 940             let layout = new.layout();
 941             match layout.ty.kind() {
 942                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 943                 _ => {
 944                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 945                     return;
 946                 }
 947             }
 948
 949             let test_old = test_old.load_scalar(fx);
 950             let new = new.load_scalar(fx);
 951
 952             let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new);
 953             let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
 954
 955             let ret_val =
 956                 CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
 957             ret.write_cvalue(fx, ret_val)
 958         }
 959
 960         _ if intrinsic.as_str().starts_with("atomic_xadd") => {
 961             intrinsic_args!(fx, args => (ptr, amount); intrinsic);
 962             let ptr = ptr.load_scalar(fx);
 963
 964             let layout = amount.layout();
 965             match layout.ty.kind() {
 966                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 967                 _ => {
 968                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 969                     return;
 970                 }
 971             }
 972             let ty = fx.clif_type(layout.ty).unwrap();
 973
 974             let amount = amount.load_scalar(fx);
 975
 976             let old =
 977                 fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
 978
 979             let old = CValue::by_val(old, layout);
 980             ret.write_cvalue(fx, old);
 981         }
 982         _ if intrinsic.as_str().starts_with("atomic_xsub") => {
 983             intrinsic_args!(fx, args => (ptr, amount); intrinsic);
 984             let ptr = ptr.load_scalar(fx);
 985
 986             let layout = amount.layout();
 987             match layout.ty.kind() {
 988                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 989                 _ => {
 990                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 991                     return;
 992                 }
 993             }
 994             let ty = fx.clif_type(layout.ty).unwrap();
 995
 996             let amount = amount.load_scalar(fx);
 997
 998             let old =
 999                 fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
1000
1001             let old = CValue::by_val(old, layout);
1002             ret.write_cvalue(fx, old);
1003         }
1004         _ if intrinsic.as_str().starts_with("atomic_and") => {
1005             intrinsic_args!(fx, args => (ptr, src); intrinsic);
1006             let ptr = ptr.load_scalar(fx);
1007
1008             let layout = src.layout();
1009             match layout.ty.kind() {
1010                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1011                 _ => {
1012                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1013                     return;
1014                 }
1015             }
1016             let ty = fx.clif_type(layout.ty).unwrap();
1017
1018             let src = src.load_scalar(fx);
1019
1020             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src);
1021
1022             let old = CValue::by_val(old, layout);
1023             ret.write_cvalue(fx, old);
1024         }
1025         _ if intrinsic.as_str().starts_with("atomic_or") => {
1026             intrinsic_args!(fx, args => (ptr, src); intrinsic);
1027             let ptr = ptr.load_scalar(fx);
1028
1029             let layout = src.layout();
1030             match layout.ty.kind() {
1031                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1032                 _ => {
1033                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1034                     return;
1035                 }
1036             }
1037             let ty = fx.clif_type(layout.ty).unwrap();
1038
1039             let src = src.load_scalar(fx);
1040
1041             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src);
1042
1043             let old = CValue::by_val(old, layout);
1044             ret.write_cvalue(fx, old);
1045         }
1046         _ if intrinsic.as_str().starts_with("atomic_xor") => {
1047             intrinsic_args!(fx, args => (ptr, src); intrinsic);
1048             let ptr = ptr.load_scalar(fx);
1049
1050             let layout = src.layout();
1051             match layout.ty.kind() {
1052                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1053                 _ => {
1054                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1055                     return;
1056                 }
1057             }
1058             let ty = fx.clif_type(layout.ty).unwrap();
1059
1060             let src = src.load_scalar(fx);
1061
1062             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src);
1063
1064             let old = CValue::by_val(old, layout);
1065             ret.write_cvalue(fx, old);
1066         }
1067         _ if intrinsic.as_str().starts_with("atomic_nand") => {
1068             intrinsic_args!(fx, args => (ptr, src); intrinsic);
1069             let ptr = ptr.load_scalar(fx);
1070
1071             let layout = src.layout();
1072             match layout.ty.kind() {
1073                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1074                 _ => {
1075                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1076                     return;
1077                 }
1078             }
1079             let ty = fx.clif_type(layout.ty).unwrap();
1080
1081             let src = src.load_scalar(fx);
1082
1083             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src);
1084
1085             let old = CValue::by_val(old, layout);
1086             ret.write_cvalue(fx, old);
1087         }
1088         _ if intrinsic.as_str().starts_with("atomic_max") => {
1089             intrinsic_args!(fx, args => (ptr, src); intrinsic);
1090             let ptr = ptr.load_scalar(fx);
1091
1092             let layout = src.layout();
1093             match layout.ty.kind() {
1094                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1095                 _ => {
1096                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1097                     return;
1098                 }
1099             }
1100             let ty = fx.clif_type(layout.ty).unwrap();
1101
1102             let src = src.load_scalar(fx);
1103
1104             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src);
1105
1106             let old = CValue::by_val(old, layout);
1107             ret.write_cvalue(fx, old);
1108         }
1109         _ if intrinsic.as_str().starts_with("atomic_umax") => {
1110             intrinsic_args!(fx, args => (ptr, src); intrinsic);
1111             let ptr = ptr.load_scalar(fx);
1112
1113             let layout = src.layout();
1114             match layout.ty.kind() {
1115                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1116                 _ => {
1117                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1118                     return;
1119                 }
1120             }
1121             let ty = fx.clif_type(layout.ty).unwrap();
1122
1123             let src = src.load_scalar(fx);
1124
1125             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src);
1126
1127             let old = CValue::by_val(old, layout);
1128             ret.write_cvalue(fx, old);
1129         }
1130         _ if intrinsic.as_str().starts_with("atomic_min") => {
1131             intrinsic_args!(fx, args => (ptr, src); intrinsic);
1132             let ptr = ptr.load_scalar(fx);
1133
1134             let layout = src.layout();
1135             match layout.ty.kind() {
1136                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1137                 _ => {
1138                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1139                     return;
1140                 }
1141             }
1142             let ty = fx.clif_type(layout.ty).unwrap();
1143
1144             let src = src.load_scalar(fx);
1145
1146             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src);
1147
1148             let old = CValue::by_val(old, layout);
1149             ret.write_cvalue(fx, old);
1150         }
1151         _ if intrinsic.as_str().starts_with("atomic_umin") => {
1152             intrinsic_args!(fx, args => (ptr, src); intrinsic);
1153             let ptr = ptr.load_scalar(fx);
1154
1155             let layout = src.layout();
1156             match layout.ty.kind() {
1157                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1158                 _ => {
1159                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1160                     return;
1161                 }
1162             }
1163             let ty = fx.clif_type(layout.ty).unwrap();
1164
1165             let src = src.load_scalar(fx);
1166
1167             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src);
1168
1169             let old = CValue::by_val(old, layout);
1170             ret.write_cvalue(fx, old);
1171         }
1172
1173         sym::minnumf32 => {
1174             intrinsic_args!(fx, args => (a, b); intrinsic);
1175             let a = a.load_scalar(fx);
1176             let b = b.load_scalar(fx);
1177
1178             let val = crate::num::codegen_float_min(fx, a, b);
1179             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1180             ret.write_cvalue(fx, val);
1181         }
1182         sym::minnumf64 => {
1183             intrinsic_args!(fx, args => (a, b); intrinsic);
1184             let a = a.load_scalar(fx);
1185             let b = b.load_scalar(fx);
1186
1187             let val = crate::num::codegen_float_min(fx, a, b);
1188             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1189             ret.write_cvalue(fx, val);
1190         }
1191         sym::maxnumf32 => {
1192             intrinsic_args!(fx, args => (a, b); intrinsic);
1193             let a = a.load_scalar(fx);
1194             let b = b.load_scalar(fx);
1195
1196             let val = crate::num::codegen_float_max(fx, a, b);
1197             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1198             ret.write_cvalue(fx, val);
1199         }
1200         sym::maxnumf64 => {
1201             intrinsic_args!(fx, args => (a, b); intrinsic);
1202             let a = a.load_scalar(fx);
1203             let b = b.load_scalar(fx);
1204
1205             let val = crate::num::codegen_float_max(fx, a, b);
1206             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1207             ret.write_cvalue(fx, val);
1208         }
1209
1210         kw::Try => {
1211             intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic);
1212             let f = f.load_scalar(fx);
1213             let data = data.load_scalar(fx);
1214             let _catch_fn = catch_fn.load_scalar(fx);
1215
1216             // FIXME once unwinding is supported, change this to actually catch panics
1217             let f_sig = fx.bcx.func.import_signature(Signature {
1218                 call_conv: fx.target_config.default_call_conv,
1219                 params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
1220                 returns: vec![],
1221             });
1222
1223             fx.bcx.ins().call_indirect(f_sig, f, &[data]);
1224
1225             let layout = ret.layout();
1226             let ret_val = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size));
1227             ret.write_cvalue(fx, ret_val);
1228         }
1229
1230         sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
1231             intrinsic_args!(fx, args => (x, y); intrinsic);
1232
1233             let res = crate::num::codegen_float_binop(
1234                 fx,
1235                 match intrinsic {
1236                     sym::fadd_fast => BinOp::Add,
1237                     sym::fsub_fast => BinOp::Sub,
1238                     sym::fmul_fast => BinOp::Mul,
1239                     sym::fdiv_fast => BinOp::Div,
1240                     sym::frem_fast => BinOp::Rem,
1241                     _ => unreachable!(),
1242                 },
1243                 x,
1244                 y,
1245             );
1246             ret.write_cvalue(fx, res);
1247         }
1248         sym::float_to_int_unchecked => {
1249             intrinsic_args!(fx, args => (f); intrinsic);
1250             let f = f.load_scalar(fx);
1251
1252             let res = crate::cast::clif_int_or_float_cast(
1253                 fx,
1254                 f,
1255                 false,
1256                 fx.clif_type(ret.layout().ty).unwrap(),
1257                 type_sign(ret.layout().ty),
1258             );
1259             ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
1260         }
1261
1262         sym::raw_eq => {
1263             intrinsic_args!(fx, args => (lhs_ref, rhs_ref); intrinsic);
1264             let lhs_ref = lhs_ref.load_scalar(fx);
1265             let rhs_ref = rhs_ref.load_scalar(fx);
1266
1267             let size = fx.layout_of(substs.type_at(0)).layout.size();
1268             // FIXME add and use emit_small_memcmp
1269             let is_eq_value = if size == Size::ZERO {
1270                 // No bytes means they're trivially equal
1271                 fx.bcx.ins().iconst(types::I8, 1)
1272             } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) {
1273                 // Can't use `trusted` for these loads; they could be unaligned.
1274                 let mut flags = MemFlags::new();
1275                 flags.set_notrap();
1276                 let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0);
1277                 let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0);
1278                 let eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val);
1279                 fx.bcx.ins().bint(types::I8, eq)
1280             } else {
1281                 // Just call `memcmp` (like slices do in core) when the
1282                 // size is too large or it's not a power-of-two.
1283                 let signed_bytes = i64::try_from(size.bytes()).unwrap();
1284                 let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes);
1285                 let params = vec![AbiParam::new(fx.pointer_type); 3];
1286                 let returns = vec![AbiParam::new(types::I32)];
1287                 let args = &[lhs_ref, rhs_ref, bytes_val];
1288                 let cmp = fx.lib_call("memcmp", params, returns, args)[0];
1289                 let eq = fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0);
1290                 fx.bcx.ins().bint(types::I8, eq)
1291             };
1292             ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout()));
1293         }
1294
1295         sym::const_allocate => {
1296             intrinsic_args!(fx, args => (_size, _align); intrinsic);
1297
1298             // returns a null pointer at runtime.
1299             let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
1300             ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
1301         }
1302
1303         sym::const_deallocate => {
1304             intrinsic_args!(fx, args => (_ptr, _size, _align); intrinsic);
1305             // nop at runtime.
1306         }
1307
1308         sym::black_box => {
1309             intrinsic_args!(fx, args => (a); intrinsic);
1310
1311             // FIXME implement black_box semantics
1312             ret.write_cvalue(fx, a);
1313         }
1314
1315         // FIXME implement variadics in cranelift
1316         sym::va_copy | sym::va_arg | sym::va_end => {
1317             fx.tcx.sess.span_fatal(
1318                 source_info.span,
1319                 "Defining variadic functions is not yet supported by Cranelift",
1320             );
1321         }
1322
1323         _ => {
1324             fx.tcx
1325                 .sess
1326                 .span_fatal(source_info.span, &format!("unsupported intrinsic {}", intrinsic));
1327         }
1328     }
1329
1330     let ret_block = fx.get_block(destination.unwrap());
1331     fx.bcx.ins().jump(ret_block, &[]);
1332 }