1 //! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"`
2 //! and LLVM intrinsics that have symbol names starting with `llvm.`.
4 macro_rules! intrinsic_args {
5 ($fx:expr, $args:expr => ($($arg:tt),*); $intrinsic:expr) => {
6 #[allow(unused_parens)]
7 let ($($arg),*) = if let [$($arg),*] = $args {
8 ($(codegen_operand($fx, $arg)),*)
10 $crate::intrinsics::bug_on_incorrect_arg_count($intrinsic);
19 pub(crate) use cpuid::codegen_cpuid_call;
20 pub(crate) use llvm::codegen_llvm_intrinsic_call;
22 use rustc_middle::ty::print::with_no_trimmed_paths;
23 use rustc_middle::ty::subst::SubstsRef;
24 use rustc_span::symbol::{kw, sym, Symbol};
26 use crate::prelude::*;
27 use cranelift_codegen::ir::AtomicRmwOp;
29 fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! {
30 bug!("wrong number of args for intrinsic {}", intrinsic);
33 fn report_atomic_type_validation_error<'tcx>(
34 fx: &mut FunctionCx<'_, '_, 'tcx>,
42 "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`",
46 // Prevent verifier error
47 fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
50 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
51 let (element, count) = match layout.abi {
52 Abi::Vector { element, count } => (element, count),
56 match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) {
57 // Cranelift currently only implements icmp for 128bit vectors.
58 Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
63 fn simd_for_each_lane<'tcx>(
64 fx: &mut FunctionCx<'_, '_, 'tcx>,
67 f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value) -> Value,
69 let layout = val.layout();
71 let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
72 let lane_layout = fx.layout_of(lane_ty);
73 let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
74 let ret_lane_layout = fx.layout_of(ret_lane_ty);
75 assert_eq!(lane_count, ret_lane_count);
77 for lane_idx in 0..lane_count {
78 let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
80 let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, lane);
81 let res_lane = CValue::by_val(res_lane, ret_lane_layout);
83 ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
87 fn simd_pair_for_each_lane_typed<'tcx>(
88 fx: &mut FunctionCx<'_, '_, 'tcx>,
92 f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, CValue<'tcx>, CValue<'tcx>) -> CValue<'tcx>,
94 assert_eq!(x.layout(), y.layout());
95 let layout = x.layout();
97 let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
98 let (ret_lane_count, _ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
99 assert_eq!(lane_count, ret_lane_count);
101 for lane_idx in 0..lane_count {
102 let x_lane = x.value_lane(fx, lane_idx);
103 let y_lane = y.value_lane(fx, lane_idx);
105 let res_lane = f(fx, x_lane, y_lane);
107 ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
111 fn simd_pair_for_each_lane<'tcx>(
112 fx: &mut FunctionCx<'_, '_, 'tcx>,
116 f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value) -> Value,
118 assert_eq!(x.layout(), y.layout());
119 let layout = x.layout();
121 let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
122 let lane_layout = fx.layout_of(lane_ty);
123 let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
124 let ret_lane_layout = fx.layout_of(ret_lane_ty);
125 assert_eq!(lane_count, ret_lane_count);
127 for lane_idx in 0..lane_count {
128 let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx);
129 let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx);
131 let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, x_lane, y_lane);
132 let res_lane = CValue::by_val(res_lane, ret_lane_layout);
134 ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
138 fn simd_reduce<'tcx>(
139 fx: &mut FunctionCx<'_, '_, 'tcx>,
143 f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Value, Value) -> Value,
145 let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
146 let lane_layout = fx.layout_of(lane_ty);
147 assert_eq!(lane_layout, ret.layout());
149 let (mut res_val, start_lane) =
150 if let Some(acc) = acc { (acc, 0) } else { (val.value_lane(fx, 0).load_scalar(fx), 1) };
151 for lane_idx in start_lane..lane_count {
152 let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
153 res_val = f(fx, lane_layout.ty, res_val, lane);
155 let res = CValue::by_val(res_val, lane_layout);
156 ret.write_cvalue(fx, res);
159 // FIXME move all uses to `simd_reduce`
160 fn simd_reduce_bool<'tcx>(
161 fx: &mut FunctionCx<'_, '_, 'tcx>,
164 f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
166 let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
167 assert!(ret.layout().ty.is_bool());
169 let res_val = val.value_lane(fx, 0).load_scalar(fx);
170 let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
171 for lane_idx in 1..lane_count {
172 let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
173 let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
174 res_val = f(fx, res_val, lane);
176 let res_val = if fx.bcx.func.dfg.value_type(res_val) != types::I8 {
177 fx.bcx.ins().ireduce(types::I8, res_val)
181 let res = CValue::by_val(res_val, ret.layout());
182 ret.write_cvalue(fx, res);
185 fn bool_to_zero_or_max_uint<'tcx>(
186 fx: &mut FunctionCx<'_, '_, 'tcx>,
190 let ty = fx.clif_type(ty).unwrap();
192 let int_ty = match ty {
193 types::F32 => types::I32,
194 types::F64 => types::I64,
198 let val = fx.bcx.ins().bint(int_ty, val);
199 let mut res = fx.bcx.ins().ineg(val);
202 res = fx.bcx.ins().bitcast(ty, res);
208 pub(crate) fn codegen_intrinsic_call<'tcx>(
209 fx: &mut FunctionCx<'_, '_, 'tcx>,
210 instance: Instance<'tcx>,
211 args: &[mir::Operand<'tcx>],
212 destination: CPlace<'tcx>,
213 target: Option<BasicBlock>,
214 source_info: mir::SourceInfo,
216 let intrinsic = fx.tcx.item_name(instance.def_id());
217 let substs = instance.substs;
219 let target = if let Some(target) = target {
222 // Insert non returning intrinsics here
225 fx.bcx.ins().trap(TrapCode::User(0));
228 crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
230 _ => unimplemented!("unsupported intrinsic {}", intrinsic),
235 if intrinsic.as_str().starts_with("simd_") {
236 self::simd::codegen_simd_intrinsic_call(
244 let ret_block = fx.get_block(target);
245 fx.bcx.ins().jump(ret_block, &[]);
246 } else if codegen_float_intrinsic_call(fx, intrinsic, args, destination) {
247 let ret_block = fx.get_block(target);
248 fx.bcx.ins().jump(ret_block, &[]);
250 codegen_regular_intrinsic_call(
263 fn codegen_float_intrinsic_call<'tcx>(
264 fx: &mut FunctionCx<'_, '_, 'tcx>,
266 args: &[mir::Operand<'tcx>],
269 let (name, arg_count, ty) = match intrinsic {
270 sym::expf32 => ("expf", 1, fx.tcx.types.f32),
271 sym::expf64 => ("exp", 1, fx.tcx.types.f64),
272 sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32),
273 sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64),
274 sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32),
275 sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64),
276 sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32), // compiler-builtins
277 sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64), // compiler-builtins
278 sym::powf32 => ("powf", 2, fx.tcx.types.f32),
279 sym::powf64 => ("pow", 2, fx.tcx.types.f64),
280 sym::logf32 => ("logf", 1, fx.tcx.types.f32),
281 sym::logf64 => ("log", 1, fx.tcx.types.f64),
282 sym::log2f32 => ("log2f", 1, fx.tcx.types.f32),
283 sym::log2f64 => ("log2", 1, fx.tcx.types.f64),
284 sym::log10f32 => ("log10f", 1, fx.tcx.types.f32),
285 sym::log10f64 => ("log10", 1, fx.tcx.types.f64),
286 sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32),
287 sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64),
288 sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32),
289 sym::fmaf64 => ("fma", 3, fx.tcx.types.f64),
290 sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32),
291 sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64),
292 sym::floorf32 => ("floorf", 1, fx.tcx.types.f32),
293 sym::floorf64 => ("floor", 1, fx.tcx.types.f64),
294 sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32),
295 sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64),
296 sym::truncf32 => ("truncf", 1, fx.tcx.types.f32),
297 sym::truncf64 => ("trunc", 1, fx.tcx.types.f64),
298 sym::roundf32 => ("roundf", 1, fx.tcx.types.f32),
299 sym::roundf64 => ("round", 1, fx.tcx.types.f64),
300 sym::sinf32 => ("sinf", 1, fx.tcx.types.f32),
301 sym::sinf64 => ("sin", 1, fx.tcx.types.f64),
302 sym::cosf32 => ("cosf", 1, fx.tcx.types.f32),
303 sym::cosf64 => ("cos", 1, fx.tcx.types.f64),
307 if args.len() != arg_count {
308 bug!("wrong number of args for intrinsic {:?}", intrinsic);
312 let args = match args {
314 a = [codegen_operand(fx, x)];
318 b = [codegen_operand(fx, x), codegen_operand(fx, y)];
322 c = [codegen_operand(fx, x), codegen_operand(fx, y), codegen_operand(fx, z)];
328 let layout = fx.layout_of(ty);
329 let res = match intrinsic {
330 sym::fmaf32 | sym::fmaf64 => {
331 let a = args[0].load_scalar(fx);
332 let b = args[1].load_scalar(fx);
333 let c = args[2].load_scalar(fx);
334 CValue::by_val(fx.bcx.ins().fma(a, b, c), layout)
336 sym::copysignf32 | sym::copysignf64 => {
337 let a = args[0].load_scalar(fx);
338 let b = args[1].load_scalar(fx);
339 CValue::by_val(fx.bcx.ins().fcopysign(a, b), layout)
349 let a = args[0].load_scalar(fx);
351 let val = match intrinsic {
352 sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(a),
353 sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(a),
354 sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(a),
355 sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(a),
359 CValue::by_val(val, layout)
361 // These intrinsics aren't supported natively by Cranelift.
362 // Lower them to a libcall.
363 _ => fx.easy_call(name, &args, ty),
366 ret.write_cvalue(fx, res);
371 fn codegen_regular_intrinsic_call<'tcx>(
372 fx: &mut FunctionCx<'_, '_, 'tcx>,
373 instance: Instance<'tcx>,
375 substs: SubstsRef<'tcx>,
376 args: &[mir::Operand<'tcx>],
378 destination: Option<BasicBlock>,
379 source_info: mir::SourceInfo,
381 let usize_layout = fx.layout_of(fx.tcx.types.usize);
384 sym::likely | sym::unlikely => {
385 intrinsic_args!(fx, args => (a); intrinsic);
387 ret.write_cvalue(fx, a);
390 intrinsic_args!(fx, args => (); intrinsic);
392 fx.bcx.ins().debugtrap();
394 sym::copy | sym::copy_nonoverlapping => {
395 intrinsic_args!(fx, args => (src, dst, count); intrinsic);
396 let src = src.load_scalar(fx);
397 let dst = dst.load_scalar(fx);
398 let count = count.load_scalar(fx);
400 let elem_ty = substs.type_at(0);
401 let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
402 assert_eq!(args.len(), 3);
404 if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
406 if intrinsic == sym::copy_nonoverlapping {
407 // FIXME emit_small_memcpy
408 fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
410 // FIXME emit_small_memmove
411 fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
414 sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => {
415 // NOTE: the volatile variants have src and dst swapped
416 intrinsic_args!(fx, args => (dst, src, count); intrinsic);
417 let dst = dst.load_scalar(fx);
418 let src = src.load_scalar(fx);
419 let count = count.load_scalar(fx);
421 let elem_ty = substs.type_at(0);
422 let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
423 assert_eq!(args.len(), 3);
425 if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
427 // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
428 if intrinsic == sym::volatile_copy_nonoverlapping_memory {
429 // FIXME emit_small_memcpy
430 fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
432 // FIXME emit_small_memmove
433 fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
436 sym::size_of_val => {
437 intrinsic_args!(fx, args => (ptr); intrinsic);
439 let layout = fx.layout_of(substs.type_at(0));
440 // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
442 let size = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
443 let (_ptr, info) = ptr.load_scalar_pair(fx);
444 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
447 fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64)
449 ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
451 sym::min_align_of_val => {
452 intrinsic_args!(fx, args => (ptr); intrinsic);
454 let layout = fx.layout_of(substs.type_at(0));
455 // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
457 let align = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
458 let (_ptr, info) = ptr.load_scalar_pair(fx);
459 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
462 fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
464 ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
467 sym::vtable_size => {
468 intrinsic_args!(fx, args => (vtable); intrinsic);
469 let vtable = vtable.load_scalar(fx);
471 let size = crate::vtable::size_of_obj(fx, vtable);
472 ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
475 sym::vtable_align => {
476 intrinsic_args!(fx, args => (vtable); intrinsic);
477 let vtable = vtable.load_scalar(fx);
479 let align = crate::vtable::min_align_of_obj(fx, vtable);
480 ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
490 | sym::unchecked_shr => {
491 intrinsic_args!(fx, args => (x, y); intrinsic);
493 // FIXME trap on overflow
494 let bin_op = match intrinsic {
495 sym::unchecked_add => BinOp::Add,
496 sym::unchecked_sub => BinOp::Sub,
497 sym::unchecked_mul => BinOp::Mul,
498 sym::unchecked_div | sym::exact_div => BinOp::Div,
499 sym::unchecked_rem => BinOp::Rem,
500 sym::unchecked_shl => BinOp::Shl,
501 sym::unchecked_shr => BinOp::Shr,
504 let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
505 ret.write_cvalue(fx, res);
507 sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
508 intrinsic_args!(fx, args => (x, y); intrinsic);
510 assert_eq!(x.layout().ty, y.layout().ty);
511 let bin_op = match intrinsic {
512 sym::add_with_overflow => BinOp::Add,
513 sym::sub_with_overflow => BinOp::Sub,
514 sym::mul_with_overflow => BinOp::Mul,
518 let res = crate::num::codegen_checked_int_binop(fx, bin_op, x, y);
519 ret.write_cvalue(fx, res);
521 sym::saturating_add | sym::saturating_sub => {
522 intrinsic_args!(fx, args => (lhs, rhs); intrinsic);
524 assert_eq!(lhs.layout().ty, rhs.layout().ty);
525 let bin_op = match intrinsic {
526 sym::saturating_add => BinOp::Add,
527 sym::saturating_sub => BinOp::Sub,
531 let res = crate::num::codegen_saturating_int_binop(fx, bin_op, lhs, rhs);
532 ret.write_cvalue(fx, res);
534 sym::rotate_left => {
535 intrinsic_args!(fx, args => (x, y); intrinsic);
536 let y = y.load_scalar(fx);
538 let layout = x.layout();
539 let x = x.load_scalar(fx);
540 let res = fx.bcx.ins().rotl(x, y);
541 ret.write_cvalue(fx, CValue::by_val(res, layout));
543 sym::rotate_right => {
544 intrinsic_args!(fx, args => (x, y); intrinsic);
545 let y = y.load_scalar(fx);
547 let layout = x.layout();
548 let x = x.load_scalar(fx);
549 let res = fx.bcx.ins().rotr(x, y);
550 ret.write_cvalue(fx, CValue::by_val(res, layout));
553 // The only difference between offset and arith_offset is regarding UB. Because Cranelift
554 // doesn't have UB both are codegen'ed the same way
555 sym::offset | sym::arith_offset => {
556 intrinsic_args!(fx, args => (base, offset); intrinsic);
557 let offset = offset.load_scalar(fx);
559 let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
560 let pointee_size = fx.layout_of(pointee_ty).size.bytes();
561 let ptr_diff = if pointee_size != 1 {
562 fx.bcx.ins().imul_imm(offset, pointee_size as i64)
566 let base_val = base.load_scalar(fx);
567 let res = fx.bcx.ins().iadd(base_val, ptr_diff);
568 ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
572 intrinsic_args!(fx, args => (ptr, mask); intrinsic);
573 let ptr = ptr.load_scalar(fx);
574 let mask = mask.load_scalar(fx);
575 fx.bcx.ins().band(ptr, mask);
579 intrinsic_args!(fx, args => (from); intrinsic);
581 ret.write_cvalue_transmute(fx, from);
583 sym::write_bytes | sym::volatile_set_memory => {
584 intrinsic_args!(fx, args => (dst, val, count); intrinsic);
585 let val = val.load_scalar(fx);
586 let count = count.load_scalar(fx);
588 let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
589 let pointee_size = fx.layout_of(pointee_ty).size.bytes();
590 let count = if pointee_size != 1 {
591 fx.bcx.ins().imul_imm(count, pointee_size as i64)
595 let dst_ptr = dst.load_scalar(fx);
596 // FIXME make the memset actually volatile when switching to emit_small_memset
597 // FIXME use emit_small_memset
598 fx.bcx.call_memset(fx.target_config, dst_ptr, val, count);
600 sym::ctlz | sym::ctlz_nonzero => {
601 intrinsic_args!(fx, args => (arg); intrinsic);
602 let val = arg.load_scalar(fx);
604 // FIXME trap on `ctlz_nonzero` with zero arg.
605 let res = fx.bcx.ins().clz(val);
606 let res = CValue::by_val(res, arg.layout());
607 ret.write_cvalue(fx, res);
609 sym::cttz | sym::cttz_nonzero => {
610 intrinsic_args!(fx, args => (arg); intrinsic);
611 let val = arg.load_scalar(fx);
613 // FIXME trap on `cttz_nonzero` with zero arg.
614 let res = fx.bcx.ins().ctz(val);
615 let res = CValue::by_val(res, arg.layout());
616 ret.write_cvalue(fx, res);
619 intrinsic_args!(fx, args => (arg); intrinsic);
620 let val = arg.load_scalar(fx);
622 let res = fx.bcx.ins().popcnt(val);
623 let res = CValue::by_val(res, arg.layout());
624 ret.write_cvalue(fx, res);
627 intrinsic_args!(fx, args => (arg); intrinsic);
628 let val = arg.load_scalar(fx);
630 let res = fx.bcx.ins().bitrev(val);
631 let res = CValue::by_val(res, arg.layout());
632 ret.write_cvalue(fx, res);
635 // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
636 fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value {
637 match bcx.func.dfg.value_type(v) {
640 // https://code.woboq.org/gcc/include/bits/byteswap.h.html
642 let tmp1 = bcx.ins().ishl_imm(v, 8);
643 let n1 = bcx.ins().band_imm(tmp1, 0xFF00);
645 let tmp2 = bcx.ins().ushr_imm(v, 8);
646 let n2 = bcx.ins().band_imm(tmp2, 0x00FF);
648 bcx.ins().bor(n1, n2)
651 let tmp1 = bcx.ins().ishl_imm(v, 24);
652 let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000);
654 let tmp2 = bcx.ins().ishl_imm(v, 8);
655 let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000);
657 let tmp3 = bcx.ins().ushr_imm(v, 8);
658 let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00);
660 let tmp4 = bcx.ins().ushr_imm(v, 24);
661 let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF);
663 let or_tmp1 = bcx.ins().bor(n1, n2);
664 let or_tmp2 = bcx.ins().bor(n3, n4);
665 bcx.ins().bor(or_tmp1, or_tmp2)
668 let tmp1 = bcx.ins().ishl_imm(v, 56);
669 let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64);
671 let tmp2 = bcx.ins().ishl_imm(v, 40);
672 let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64);
674 let tmp3 = bcx.ins().ishl_imm(v, 24);
675 let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64);
677 let tmp4 = bcx.ins().ishl_imm(v, 8);
678 let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64);
680 let tmp5 = bcx.ins().ushr_imm(v, 8);
681 let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64);
683 let tmp6 = bcx.ins().ushr_imm(v, 24);
684 let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64);
686 let tmp7 = bcx.ins().ushr_imm(v, 40);
687 let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64);
689 let tmp8 = bcx.ins().ushr_imm(v, 56);
690 let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64);
692 let or_tmp1 = bcx.ins().bor(n1, n2);
693 let or_tmp2 = bcx.ins().bor(n3, n4);
694 let or_tmp3 = bcx.ins().bor(n5, n6);
695 let or_tmp4 = bcx.ins().bor(n7, n8);
697 let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2);
698 let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4);
699 bcx.ins().bor(or_tmp5, or_tmp6)
702 let (lo, hi) = bcx.ins().isplit(v);
703 let lo = swap(bcx, lo);
704 let hi = swap(bcx, hi);
705 bcx.ins().iconcat(hi, lo)
707 ty => unreachable!("bswap {}", ty),
710 intrinsic_args!(fx, args => (arg); intrinsic);
711 let val = arg.load_scalar(fx);
713 let res = CValue::by_val(swap(&mut fx.bcx, val), arg.layout());
714 ret.write_cvalue(fx, res);
716 sym::assert_inhabited | sym::assert_zero_valid | sym::assert_uninit_valid => {
717 intrinsic_args!(fx, args => (); intrinsic);
719 let layout = fx.layout_of(substs.type_at(0));
720 if layout.abi.is_uninhabited() {
721 with_no_trimmed_paths!({
722 crate::base::codegen_panic(
724 &format!("attempted to instantiate uninhabited type `{}`", layout.ty),
731 if intrinsic == sym::assert_zero_valid && !fx.tcx.permits_zero_init(layout) {
732 with_no_trimmed_paths!({
733 crate::base::codegen_panic(
736 "attempted to zero-initialize type `{}`, which is invalid",
745 if intrinsic == sym::assert_uninit_valid && !fx.tcx.permits_uninit_init(layout) {
746 with_no_trimmed_paths!({
747 crate::base::codegen_panic(
750 "attempted to leave type `{}` uninitialized, which is invalid",
760 sym::volatile_load | sym::unaligned_volatile_load => {
761 intrinsic_args!(fx, args => (ptr); intrinsic);
763 // Cranelift treats loads as volatile by default
764 // FIXME correctly handle unaligned_volatile_load
765 let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
766 let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
767 ret.write_cvalue(fx, val);
769 sym::volatile_store | sym::unaligned_volatile_store => {
770 intrinsic_args!(fx, args => (ptr, val); intrinsic);
771 let ptr = ptr.load_scalar(fx);
773 // Cranelift treats stores as volatile by default
774 // FIXME correctly handle unaligned_volatile_store
775 let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
776 dest.write_cvalue(fx, val);
783 | sym::variant_count => {
784 intrinsic_args!(fx, args => (); intrinsic);
787 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
788 let val = crate::constant::codegen_const_value(fx, const_val, ret.layout().ty);
789 ret.write_cvalue(fx, val);
792 sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
793 intrinsic_args!(fx, args => (ptr, base); intrinsic);
794 let ptr = ptr.load_scalar(fx);
795 let base = base.load_scalar(fx);
796 let ty = substs.type_at(0);
798 let pointee_size: u64 = fx.layout_of(ty).size.bytes();
799 let diff_bytes = fx.bcx.ins().isub(ptr, base);
800 // FIXME this can be an exact division.
801 let val = if intrinsic == sym::ptr_offset_from_unsigned {
802 let usize_layout = fx.layout_of(fx.tcx.types.usize);
803 // Because diff_bytes ULE isize::MAX, this would be fine as signed,
804 // but unsigned is slightly easier to codegen, so might as well.
805 CValue::by_val(fx.bcx.ins().udiv_imm(diff_bytes, pointee_size as i64), usize_layout)
807 let isize_layout = fx.layout_of(fx.tcx.types.isize);
808 CValue::by_val(fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64), isize_layout)
810 ret.write_cvalue(fx, val);
813 sym::ptr_guaranteed_cmp => {
814 intrinsic_args!(fx, args => (a, b); intrinsic);
816 let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b).load_scalar(fx);
817 ret.write_cvalue(fx, CValue::by_val(val, fx.layout_of(fx.tcx.types.u8)));
820 sym::caller_location => {
821 intrinsic_args!(fx, args => (); intrinsic);
823 let caller_location = fx.get_caller_location(source_info);
824 ret.write_cvalue(fx, caller_location);
827 _ if intrinsic.as_str().starts_with("atomic_fence") => {
828 intrinsic_args!(fx, args => (); intrinsic);
830 fx.bcx.ins().fence();
832 _ if intrinsic.as_str().starts_with("atomic_singlethreadfence") => {
833 intrinsic_args!(fx, args => (); intrinsic);
835 // FIXME use a compiler fence once Cranelift supports it
836 fx.bcx.ins().fence();
838 _ if intrinsic.as_str().starts_with("atomic_load") => {
839 intrinsic_args!(fx, args => (ptr); intrinsic);
840 let ptr = ptr.load_scalar(fx);
842 let ty = substs.type_at(0);
844 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
845 // FIXME implement 128bit atomics
846 if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
847 // special case for compiler-builtins to avoid having to patch it
848 crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
853 .span_fatal(source_info.span, "128bit atomics not yet supported");
856 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
858 report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
862 let clif_ty = fx.clif_type(ty).unwrap();
864 let val = fx.bcx.ins().atomic_load(clif_ty, MemFlags::trusted(), ptr);
866 let val = CValue::by_val(val, fx.layout_of(ty));
867 ret.write_cvalue(fx, val);
869 _ if intrinsic.as_str().starts_with("atomic_store") => {
870 intrinsic_args!(fx, args => (ptr, val); intrinsic);
871 let ptr = ptr.load_scalar(fx);
873 let ty = substs.type_at(0);
875 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
876 // FIXME implement 128bit atomics
877 if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
878 // special case for compiler-builtins to avoid having to patch it
879 crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
884 .span_fatal(source_info.span, "128bit atomics not yet supported");
887 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
889 report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
894 let val = val.load_scalar(fx);
896 fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
898 _ if intrinsic.as_str().starts_with("atomic_xchg") => {
899 intrinsic_args!(fx, args => (ptr, new); intrinsic);
900 let ptr = ptr.load_scalar(fx);
902 let layout = new.layout();
903 match layout.ty.kind() {
904 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
906 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
910 let ty = fx.clif_type(layout.ty).unwrap();
912 let new = new.load_scalar(fx);
914 let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new);
916 let old = CValue::by_val(old, layout);
917 ret.write_cvalue(fx, old);
919 _ if intrinsic.as_str().starts_with("atomic_cxchg") => {
920 // both atomic_cxchg_* and atomic_cxchgweak_*
921 intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic);
922 let ptr = ptr.load_scalar(fx);
924 let layout = new.layout();
925 match layout.ty.kind() {
926 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
928 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
933 let test_old = test_old.load_scalar(fx);
934 let new = new.load_scalar(fx);
936 let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new);
937 let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
940 CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
941 ret.write_cvalue(fx, ret_val)
944 _ if intrinsic.as_str().starts_with("atomic_xadd") => {
945 intrinsic_args!(fx, args => (ptr, amount); intrinsic);
946 let ptr = ptr.load_scalar(fx);
948 let layout = amount.layout();
949 match layout.ty.kind() {
950 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
952 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
956 let ty = fx.clif_type(layout.ty).unwrap();
958 let amount = amount.load_scalar(fx);
961 fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
963 let old = CValue::by_val(old, layout);
964 ret.write_cvalue(fx, old);
966 _ if intrinsic.as_str().starts_with("atomic_xsub") => {
967 intrinsic_args!(fx, args => (ptr, amount); intrinsic);
968 let ptr = ptr.load_scalar(fx);
970 let layout = amount.layout();
971 match layout.ty.kind() {
972 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
974 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
978 let ty = fx.clif_type(layout.ty).unwrap();
980 let amount = amount.load_scalar(fx);
983 fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
985 let old = CValue::by_val(old, layout);
986 ret.write_cvalue(fx, old);
988 _ if intrinsic.as_str().starts_with("atomic_and") => {
989 intrinsic_args!(fx, args => (ptr, src); intrinsic);
990 let ptr = ptr.load_scalar(fx);
992 let layout = src.layout();
993 match layout.ty.kind() {
994 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
996 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1000 let ty = fx.clif_type(layout.ty).unwrap();
1002 let src = src.load_scalar(fx);
1004 let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src);
1006 let old = CValue::by_val(old, layout);
1007 ret.write_cvalue(fx, old);
1009 _ if intrinsic.as_str().starts_with("atomic_or") => {
1010 intrinsic_args!(fx, args => (ptr, src); intrinsic);
1011 let ptr = ptr.load_scalar(fx);
1013 let layout = src.layout();
1014 match layout.ty.kind() {
1015 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1017 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1021 let ty = fx.clif_type(layout.ty).unwrap();
1023 let src = src.load_scalar(fx);
1025 let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src);
1027 let old = CValue::by_val(old, layout);
1028 ret.write_cvalue(fx, old);
1030 _ if intrinsic.as_str().starts_with("atomic_xor") => {
1031 intrinsic_args!(fx, args => (ptr, src); intrinsic);
1032 let ptr = ptr.load_scalar(fx);
1034 let layout = src.layout();
1035 match layout.ty.kind() {
1036 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1038 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1042 let ty = fx.clif_type(layout.ty).unwrap();
1044 let src = src.load_scalar(fx);
1046 let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src);
1048 let old = CValue::by_val(old, layout);
1049 ret.write_cvalue(fx, old);
1051 _ if intrinsic.as_str().starts_with("atomic_nand") => {
1052 intrinsic_args!(fx, args => (ptr, src); intrinsic);
1053 let ptr = ptr.load_scalar(fx);
1055 let layout = src.layout();
1056 match layout.ty.kind() {
1057 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1059 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1063 let ty = fx.clif_type(layout.ty).unwrap();
1065 let src = src.load_scalar(fx);
1067 let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src);
1069 let old = CValue::by_val(old, layout);
1070 ret.write_cvalue(fx, old);
1072 _ if intrinsic.as_str().starts_with("atomic_max") => {
1073 intrinsic_args!(fx, args => (ptr, src); intrinsic);
1074 let ptr = ptr.load_scalar(fx);
1076 let layout = src.layout();
1077 match layout.ty.kind() {
1078 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1080 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1084 let ty = fx.clif_type(layout.ty).unwrap();
1086 let src = src.load_scalar(fx);
1088 let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src);
1090 let old = CValue::by_val(old, layout);
1091 ret.write_cvalue(fx, old);
1093 _ if intrinsic.as_str().starts_with("atomic_umax") => {
1094 intrinsic_args!(fx, args => (ptr, src); intrinsic);
1095 let ptr = ptr.load_scalar(fx);
1097 let layout = src.layout();
1098 match layout.ty.kind() {
1099 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1101 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1105 let ty = fx.clif_type(layout.ty).unwrap();
1107 let src = src.load_scalar(fx);
1109 let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src);
1111 let old = CValue::by_val(old, layout);
1112 ret.write_cvalue(fx, old);
1114 _ if intrinsic.as_str().starts_with("atomic_min") => {
1115 intrinsic_args!(fx, args => (ptr, src); intrinsic);
1116 let ptr = ptr.load_scalar(fx);
1118 let layout = src.layout();
1119 match layout.ty.kind() {
1120 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1122 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1126 let ty = fx.clif_type(layout.ty).unwrap();
1128 let src = src.load_scalar(fx);
1130 let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src);
1132 let old = CValue::by_val(old, layout);
1133 ret.write_cvalue(fx, old);
1135 _ if intrinsic.as_str().starts_with("atomic_umin") => {
1136 intrinsic_args!(fx, args => (ptr, src); intrinsic);
1137 let ptr = ptr.load_scalar(fx);
1139 let layout = src.layout();
1140 match layout.ty.kind() {
1141 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1143 report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1147 let ty = fx.clif_type(layout.ty).unwrap();
1149 let src = src.load_scalar(fx);
1151 let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src);
1153 let old = CValue::by_val(old, layout);
1154 ret.write_cvalue(fx, old);
1158 intrinsic_args!(fx, args => (a, b); intrinsic);
1159 let a = a.load_scalar(fx);
1160 let b = b.load_scalar(fx);
1162 let val = crate::num::codegen_float_min(fx, a, b);
1163 let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1164 ret.write_cvalue(fx, val);
1167 intrinsic_args!(fx, args => (a, b); intrinsic);
1168 let a = a.load_scalar(fx);
1169 let b = b.load_scalar(fx);
1171 let val = crate::num::codegen_float_min(fx, a, b);
1172 let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1173 ret.write_cvalue(fx, val);
1176 intrinsic_args!(fx, args => (a, b); intrinsic);
1177 let a = a.load_scalar(fx);
1178 let b = b.load_scalar(fx);
1180 let val = crate::num::codegen_float_max(fx, a, b);
1181 let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1182 ret.write_cvalue(fx, val);
1185 intrinsic_args!(fx, args => (a, b); intrinsic);
1186 let a = a.load_scalar(fx);
1187 let b = b.load_scalar(fx);
1189 let val = crate::num::codegen_float_max(fx, a, b);
1190 let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1191 ret.write_cvalue(fx, val);
1195 intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic);
1196 let f = f.load_scalar(fx);
1197 let data = data.load_scalar(fx);
1198 let _catch_fn = catch_fn.load_scalar(fx);
1200 // FIXME once unwinding is supported, change this to actually catch panics
1201 let f_sig = fx.bcx.func.import_signature(Signature {
1202 call_conv: fx.target_config.default_call_conv,
1203 params: vec![AbiParam::new(pointer_ty(fx.tcx))],
1207 fx.bcx.ins().call_indirect(f_sig, f, &[data]);
1209 let layout = ret.layout();
1210 let ret_val = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size));
1211 ret.write_cvalue(fx, ret_val);
1214 sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
1215 intrinsic_args!(fx, args => (x, y); intrinsic);
1217 let res = crate::num::codegen_float_binop(
1220 sym::fadd_fast => BinOp::Add,
1221 sym::fsub_fast => BinOp::Sub,
1222 sym::fmul_fast => BinOp::Mul,
1223 sym::fdiv_fast => BinOp::Div,
1224 sym::frem_fast => BinOp::Rem,
1225 _ => unreachable!(),
1230 ret.write_cvalue(fx, res);
1232 sym::float_to_int_unchecked => {
1233 intrinsic_args!(fx, args => (f); intrinsic);
1234 let f = f.load_scalar(fx);
1236 let res = crate::cast::clif_int_or_float_cast(
1240 fx.clif_type(ret.layout().ty).unwrap(),
1241 type_sign(ret.layout().ty),
1243 ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
1247 intrinsic_args!(fx, args => (lhs_ref, rhs_ref); intrinsic);
1248 let lhs_ref = lhs_ref.load_scalar(fx);
1249 let rhs_ref = rhs_ref.load_scalar(fx);
1251 let size = fx.layout_of(substs.type_at(0)).layout.size();
1252 // FIXME add and use emit_small_memcmp
1253 let is_eq_value = if size == Size::ZERO {
1254 // No bytes means they're trivially equal
1255 fx.bcx.ins().iconst(types::I8, 1)
1256 } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) {
1257 // Can't use `trusted` for these loads; they could be unaligned.
1258 let mut flags = MemFlags::new();
1260 let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0);
1261 let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0);
1262 let eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val);
1263 fx.bcx.ins().bint(types::I8, eq)
1265 // Just call `memcmp` (like slices do in core) when the
1266 // size is too large or it's not a power-of-two.
1267 let signed_bytes = i64::try_from(size.bytes()).unwrap();
1268 let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes);
1269 let params = vec![AbiParam::new(fx.pointer_type); 3];
1270 let returns = vec![AbiParam::new(types::I32)];
1271 let args = &[lhs_ref, rhs_ref, bytes_val];
1272 let cmp = fx.lib_call("memcmp", params, returns, args)[0];
1273 let eq = fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0);
1274 fx.bcx.ins().bint(types::I8, eq)
1276 ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout()));
1279 sym::const_allocate => {
1280 intrinsic_args!(fx, args => (_size, _align); intrinsic);
1282 // returns a null pointer at runtime.
1283 let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
1284 ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
1287 sym::const_deallocate => {
1288 intrinsic_args!(fx, args => (_ptr, _size, _align); intrinsic);
1293 intrinsic_args!(fx, args => (a); intrinsic);
1295 // FIXME implement black_box semantics
1296 ret.write_cvalue(fx, a);
1299 // FIXME implement variadics in cranelift
1300 sym::va_copy | sym::va_arg | sym::va_end => {
1301 fx.tcx.sess.span_fatal(
1303 "Defining variadic functions is not yet supported by Cranelift",
1310 .span_fatal(source_info.span, &format!("unsupported intrinsic {}", intrinsic));
1314 let ret_block = fx.get_block(destination.unwrap());
1315 fx.bcx.ins().jump(ret_block, &[]);