1 use super::operand::{OperandRef, OperandValue};
2 use super::place::PlaceRef;
4 use crate::common::{span_invalid_monomorphization_error, IntPredicate};
9 use rustc_middle::ty::{self, Ty, TyCtxt};
10 use rustc_span::{sym, Span};
11 use rustc_target::abi::call::{FnAbi, PassMode};
13 fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
22 let layout = bx.layout_of(ty);
23 let size = layout.size;
24 let align = layout.align.abi;
25 let size = bx.mul(bx.const_usize(size.bytes()), count);
26 let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() };
28 bx.memmove(dst, align, src, align, size, flags);
30 bx.memcpy(dst, align, src, align, size, flags);
34 fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
42 let layout = bx.layout_of(ty);
43 let size = layout.size;
44 let align = layout.align.abi;
45 let size = bx.mul(bx.const_usize(size.bytes()), count);
46 let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() };
47 bx.memset(dst, val, size, align, flags);
50 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
51 pub fn codegen_intrinsic_call(
53 instance: ty::Instance<'tcx>,
54 fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
55 args: &[OperandRef<'tcx, Bx::Value>],
59 let callee_ty = instance.ty(bx.tcx(), ty::ParamEnv::reveal_all());
61 let ty::FnDef(def_id, substs) = *callee_ty.kind() else {
62 bug!("expected fn item type, found {}", callee_ty);
65 let sig = callee_ty.fn_sig(bx.tcx());
66 let sig = bx.tcx().normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig);
67 let arg_tys = sig.inputs();
68 let ret_ty = sig.output();
69 let name = bx.tcx().item_name(def_id);
70 let name_str = name.as_str();
72 let llret_ty = bx.backend_type(bx.layout_of(ret_ty));
73 let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
75 let llval = match name {
77 bx.assume(args[0].immediate());
85 sym::va_start => bx.va_start(args[0].immediate()),
86 sym::va_end => bx.va_end(args[0].immediate()),
88 let tp_ty = substs.type_at(0);
89 if let OperandValue::Pair(_, meta) = args[0].val {
90 let (llsize, _) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta));
93 bx.const_usize(bx.layout_of(tp_ty).size.bytes())
96 sym::min_align_of_val => {
97 let tp_ty = substs.type_at(0);
98 if let OperandValue::Pair(_, meta) = args[0].val {
99 let (_, llalign) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta));
102 bx.const_usize(bx.layout_of(tp_ty).align.abi.bytes())
109 | sym::variant_count => {
112 .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None)
114 OperandRef::from_const(bx, value, ret_ty).immediate_or_packed_pair(bx)
117 let ty = substs.type_at(0);
118 let layout = bx.layout_of(ty);
119 let ptr = args[0].immediate();
120 let offset = args[1].immediate();
121 bx.inbounds_gep(bx.backend_type(layout), ptr, &[offset])
123 sym::arith_offset => {
124 let ty = substs.type_at(0);
125 let layout = bx.layout_of(ty);
126 let ptr = args[0].immediate();
127 let offset = args[1].immediate();
128 bx.gep(bx.backend_type(layout), ptr, &[offset])
142 sym::write_bytes => {
154 sym::volatile_copy_nonoverlapping_memory => {
166 sym::volatile_copy_memory => {
178 sym::volatile_set_memory => {
189 sym::volatile_store => {
190 let dst = args[0].deref(bx.cx());
191 args[1].val.volatile_store(bx, dst);
194 sym::unaligned_volatile_store => {
195 let dst = args[0].deref(bx.cx());
196 args[1].val.unaligned_volatile_store(bx, dst);
199 sym::add_with_overflow
200 | sym::sub_with_overflow
201 | sym::mul_with_overflow
209 | sym::exact_div => {
211 match int_type_width_signed(ty, bx.tcx()) {
212 Some((_width, signed)) => match name {
213 sym::add_with_overflow
214 | sym::sub_with_overflow
215 | sym::mul_with_overflow => {
216 let op = match name {
217 sym::add_with_overflow => OverflowOp::Add,
218 sym::sub_with_overflow => OverflowOp::Sub,
219 sym::mul_with_overflow => OverflowOp::Mul,
222 let (val, overflow) =
223 bx.checked_binop(op, ty, args[0].immediate(), args[1].immediate());
224 // Convert `i1` to a `bool`, and write it to the out parameter
225 let val = bx.from_immediate(val);
226 let overflow = bx.from_immediate(overflow);
228 let dest = result.project_field(bx, 0);
229 bx.store(val, dest.llval, dest.align);
230 let dest = result.project_field(bx, 1);
231 bx.store(overflow, dest.llval, dest.align);
237 bx.exactsdiv(args[0].immediate(), args[1].immediate())
239 bx.exactudiv(args[0].immediate(), args[1].immediate())
242 sym::unchecked_div => {
244 bx.sdiv(args[0].immediate(), args[1].immediate())
246 bx.udiv(args[0].immediate(), args[1].immediate())
249 sym::unchecked_rem => {
251 bx.srem(args[0].immediate(), args[1].immediate())
253 bx.urem(args[0].immediate(), args[1].immediate())
256 sym::unchecked_shl => bx.shl(args[0].immediate(), args[1].immediate()),
257 sym::unchecked_shr => {
259 bx.ashr(args[0].immediate(), args[1].immediate())
261 bx.lshr(args[0].immediate(), args[1].immediate())
264 sym::unchecked_add => {
266 bx.unchecked_sadd(args[0].immediate(), args[1].immediate())
268 bx.unchecked_uadd(args[0].immediate(), args[1].immediate())
271 sym::unchecked_sub => {
273 bx.unchecked_ssub(args[0].immediate(), args[1].immediate())
275 bx.unchecked_usub(args[0].immediate(), args[1].immediate())
278 sym::unchecked_mul => {
280 bx.unchecked_smul(args[0].immediate(), args[1].immediate())
282 bx.unchecked_umul(args[0].immediate(), args[1].immediate())
288 span_invalid_monomorphization_error(
292 "invalid monomorphization of `{}` intrinsic: \
293 expected basic integer type, found `{}`",
301 sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
302 match float_type_width(arg_tys[0]) {
303 Some(_width) => match name {
304 sym::fadd_fast => bx.fadd_fast(args[0].immediate(), args[1].immediate()),
305 sym::fsub_fast => bx.fsub_fast(args[0].immediate(), args[1].immediate()),
306 sym::fmul_fast => bx.fmul_fast(args[0].immediate(), args[1].immediate()),
307 sym::fdiv_fast => bx.fdiv_fast(args[0].immediate(), args[1].immediate()),
308 sym::frem_fast => bx.frem_fast(args[0].immediate(), args[1].immediate()),
312 span_invalid_monomorphization_error(
316 "invalid monomorphization of `{}` intrinsic: \
317 expected basic float type, found `{}`",
326 sym::float_to_int_unchecked => {
327 if float_type_width(arg_tys[0]).is_none() {
328 span_invalid_monomorphization_error(
332 "invalid monomorphization of `float_to_int_unchecked` \
333 intrinsic: expected basic float type, \
340 let Some((_width, signed)) = int_type_width_signed(ret_ty, bx.tcx()) else {
341 span_invalid_monomorphization_error(
345 "invalid monomorphization of `float_to_int_unchecked` \
346 intrinsic: expected basic integer type, \
354 bx.fptosi(args[0].immediate(), llret_ty)
356 bx.fptoui(args[0].immediate(), llret_ty)
360 sym::discriminant_value => {
361 if ret_ty.is_integral() {
362 args[0].deref(bx.cx()).codegen_get_discr(bx, ret_ty)
364 span_bug!(span, "Invalid discriminant type for `{:?}`", arg_tys[0])
368 sym::const_allocate => {
369 // returns a null pointer at runtime.
370 bx.const_null(bx.type_i8p())
373 sym::const_deallocate => {
378 // This requires that atomic intrinsics follow a specific naming pattern:
379 // "atomic_<operation>[_<ordering>]"
380 name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
381 use crate::common::AtomicOrdering::*;
382 use crate::common::{AtomicRmwBinOp, SynchronizationScope};
384 let Some((instruction, ordering)) = atomic.split_once('_') else {
385 bx.sess().fatal("Atomic intrinsic missing memory ordering");
388 let parse_ordering = |bx: &Bx, s| match s {
389 "unordered" => Unordered,
390 "relaxed" => Relaxed,
391 "acquire" => Acquire,
392 "release" => Release,
393 "acqrel" => AcquireRelease,
394 "seqcst" => SequentiallyConsistent,
395 _ => bx.sess().fatal("unknown ordering in atomic intrinsic"),
398 let invalid_monomorphization = |ty| {
399 span_invalid_monomorphization_error(
403 "invalid monomorphization of `{}` intrinsic: \
404 expected basic integer type, found `{}`",
411 "cxchg" | "cxchgweak" => {
412 let Some((success, failure)) = ordering.split_once('_') else {
413 bx.sess().fatal("Atomic compare-exchange intrinsic missing failure memory ordering");
415 let ty = substs.type_at(0);
416 if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
417 let weak = instruction == "cxchgweak";
418 let mut dst = args[0].immediate();
419 let mut cmp = args[1].immediate();
420 let mut src = args[2].immediate();
421 if ty.is_unsafe_ptr() {
422 // Some platforms do not support atomic operations on pointers,
423 // so we cast to integer first.
424 let ptr_llty = bx.type_ptr_to(bx.type_isize());
425 dst = bx.pointercast(dst, ptr_llty);
426 cmp = bx.ptrtoint(cmp, bx.type_isize());
427 src = bx.ptrtoint(src, bx.type_isize());
429 let pair = bx.atomic_cmpxchg(dst, cmp, src, parse_ordering(bx, success), parse_ordering(bx, failure), weak);
430 let val = bx.extract_value(pair, 0);
431 let success = bx.extract_value(pair, 1);
432 let val = bx.from_immediate(val);
433 let success = bx.from_immediate(success);
435 let dest = result.project_field(bx, 0);
436 bx.store(val, dest.llval, dest.align);
437 let dest = result.project_field(bx, 1);
438 bx.store(success, dest.llval, dest.align);
441 return invalid_monomorphization(ty);
446 let ty = substs.type_at(0);
447 if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
448 let layout = bx.layout_of(ty);
449 let size = layout.size;
450 let mut source = args[0].immediate();
451 if ty.is_unsafe_ptr() {
452 // Some platforms do not support atomic operations on pointers,
453 // so we cast to integer first...
454 let llty = bx.type_isize();
455 let ptr_llty = bx.type_ptr_to(llty);
456 source = bx.pointercast(source, ptr_llty);
457 let result = bx.atomic_load(llty, source, parse_ordering(bx, ordering), size);
458 // ... and then cast the result back to a pointer
459 bx.inttoptr(result, bx.backend_type(layout))
461 bx.atomic_load(bx.backend_type(layout), source, parse_ordering(bx, ordering), size)
464 return invalid_monomorphization(ty);
469 let ty = substs.type_at(0);
470 if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
471 let size = bx.layout_of(ty).size;
472 let mut val = args[1].immediate();
473 let mut ptr = args[0].immediate();
474 if ty.is_unsafe_ptr() {
475 // Some platforms do not support atomic operations on pointers,
476 // so we cast to integer first.
477 let ptr_llty = bx.type_ptr_to(bx.type_isize());
478 ptr = bx.pointercast(ptr, ptr_llty);
479 val = bx.ptrtoint(val, bx.type_isize());
481 bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size);
484 return invalid_monomorphization(ty);
489 bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::CrossThread);
493 "singlethreadfence" => {
494 bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::SingleThread);
498 // These are all AtomicRMW ops
500 let atom_op = match op {
501 "xchg" => AtomicRmwBinOp::AtomicXchg,
502 "xadd" => AtomicRmwBinOp::AtomicAdd,
503 "xsub" => AtomicRmwBinOp::AtomicSub,
504 "and" => AtomicRmwBinOp::AtomicAnd,
505 "nand" => AtomicRmwBinOp::AtomicNand,
506 "or" => AtomicRmwBinOp::AtomicOr,
507 "xor" => AtomicRmwBinOp::AtomicXor,
508 "max" => AtomicRmwBinOp::AtomicMax,
509 "min" => AtomicRmwBinOp::AtomicMin,
510 "umax" => AtomicRmwBinOp::AtomicUMax,
511 "umin" => AtomicRmwBinOp::AtomicUMin,
512 _ => bx.sess().fatal("unknown atomic operation"),
515 let ty = substs.type_at(0);
516 if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
517 let mut ptr = args[0].immediate();
518 let mut val = args[1].immediate();
519 if ty.is_unsafe_ptr() {
520 // Some platforms do not support atomic operations on pointers,
521 // so we cast to integer first.
522 let ptr_llty = bx.type_ptr_to(bx.type_isize());
523 ptr = bx.pointercast(ptr, ptr_llty);
524 val = bx.ptrtoint(val, bx.type_isize());
526 bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
528 return invalid_monomorphization(ty);
534 sym::nontemporal_store => {
535 let dst = args[0].deref(bx.cx());
536 args[1].val.nontemporal_store(bx, dst);
540 sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => {
541 let a = args[0].immediate();
542 let b = args[1].immediate();
543 if name == sym::ptr_guaranteed_eq {
544 bx.icmp(IntPredicate::IntEQ, a, b)
546 bx.icmp(IntPredicate::IntNE, a, b)
550 sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
551 let ty = substs.type_at(0);
552 let pointee_size = bx.layout_of(ty).size;
554 let a = args[0].immediate();
555 let b = args[1].immediate();
556 let a = bx.ptrtoint(a, bx.type_isize());
557 let b = bx.ptrtoint(b, bx.type_isize());
558 let pointee_size = bx.const_usize(pointee_size.bytes());
559 if name == sym::ptr_offset_from {
560 // This is the same sequence that Clang emits for pointer subtraction.
561 // It can be neither `nsw` nor `nuw` because the input is treated as
562 // unsigned but then the output is treated as signed, so neither works.
563 let d = bx.sub(a, b);
564 // this is where the signed magic happens (notice the `s` in `exactsdiv`)
565 bx.exactsdiv(d, pointee_size)
567 // The `_unsigned` version knows the relative ordering of the pointers,
568 // so can use `sub nuw` and `udiv exact` instead of dealing in signed.
569 let d = bx.unchecked_usub(a, b);
570 bx.exactudiv(d, pointee_size)
575 // Need to use backend-specific things in the implementation.
576 bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span);
581 if !fn_abi.ret.is_ignore() {
582 if let PassMode::Cast(ty) = fn_abi.ret.mode {
583 let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(&ty));
584 let ptr = bx.pointercast(result.llval, ptr_llty);
585 bx.store(llval, ptr, result.align);
587 OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout)
595 // Returns the width of an int Ty, and if it's signed or not
596 // Returns None if the type is not an integer
597 // FIXME: there’s multiple of this functions, investigate using some of the already existing
599 fn int_type_width_signed(ty: Ty<'_>, tcx: TyCtxt<'_>) -> Option<(u64, bool)> {
602 Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), true))
605 Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), false))
611 // Returns the width of a float Ty
612 // Returns None if the type is not a float
613 fn float_type_width(ty: Ty<'_>) -> Option<u64> {
615 ty::Float(t) => Some(t.bit_width()),