]> git.lizzy.rs Git - rust.git/blob - compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
Auto merge of #101442 - joboet:null_check_tcs, r=thomcc
[rust.git] / compiler / rustc_codegen_ssa / src / mir / intrinsic.rs
1 use super::operand::{OperandRef, OperandValue};
2 use super::place::PlaceRef;
3 use super::FunctionCx;
4 use crate::common::{span_invalid_monomorphization_error, IntPredicate};
5 use crate::glue;
6 use crate::meth;
7 use crate::traits::*;
8 use crate::MemFlags;
9
10 use rustc_middle::ty::{self, Ty, TyCtxt};
11 use rustc_span::{sym, Span};
12 use rustc_target::abi::{
13     call::{FnAbi, PassMode},
14     WrappingRange,
15 };
16
17 fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
18     bx: &mut Bx,
19     allow_overlap: bool,
20     volatile: bool,
21     ty: Ty<'tcx>,
22     dst: Bx::Value,
23     src: Bx::Value,
24     count: Bx::Value,
25 ) {
26     let layout = bx.layout_of(ty);
27     let size = layout.size;
28     let align = layout.align.abi;
29     let size = bx.mul(bx.const_usize(size.bytes()), count);
30     let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() };
31     if allow_overlap {
32         bx.memmove(dst, align, src, align, size, flags);
33     } else {
34         bx.memcpy(dst, align, src, align, size, flags);
35     }
36 }
37
38 fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
39     bx: &mut Bx,
40     volatile: bool,
41     ty: Ty<'tcx>,
42     dst: Bx::Value,
43     val: Bx::Value,
44     count: Bx::Value,
45 ) {
46     let layout = bx.layout_of(ty);
47     let size = layout.size;
48     let align = layout.align.abi;
49     let size = bx.mul(bx.const_usize(size.bytes()), count);
50     let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() };
51     bx.memset(dst, val, size, align, flags);
52 }
53
54 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
55     pub fn codegen_intrinsic_call(
56         bx: &mut Bx,
57         instance: ty::Instance<'tcx>,
58         fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
59         args: &[OperandRef<'tcx, Bx::Value>],
60         llresult: Bx::Value,
61         span: Span,
62     ) {
63         let callee_ty = instance.ty(bx.tcx(), ty::ParamEnv::reveal_all());
64
65         let ty::FnDef(def_id, substs) = *callee_ty.kind() else {
66             bug!("expected fn item type, found {}", callee_ty);
67         };
68
69         let sig = callee_ty.fn_sig(bx.tcx());
70         let sig = bx.tcx().normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig);
71         let arg_tys = sig.inputs();
72         let ret_ty = sig.output();
73         let name = bx.tcx().item_name(def_id);
74         let name_str = name.as_str();
75
76         let llret_ty = bx.backend_type(bx.layout_of(ret_ty));
77         let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
78
79         let llval = match name {
80             sym::abort => {
81                 bx.abort();
82                 return;
83             }
84
85             sym::va_start => bx.va_start(args[0].immediate()),
86             sym::va_end => bx.va_end(args[0].immediate()),
87             sym::size_of_val => {
88                 let tp_ty = substs.type_at(0);
89                 if let OperandValue::Pair(_, meta) = args[0].val {
90                     let (llsize, _) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta));
91                     llsize
92                 } else {
93                     bx.const_usize(bx.layout_of(tp_ty).size.bytes())
94                 }
95             }
96             sym::min_align_of_val => {
97                 let tp_ty = substs.type_at(0);
98                 if let OperandValue::Pair(_, meta) = args[0].val {
99                     let (_, llalign) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta));
100                     llalign
101                 } else {
102                     bx.const_usize(bx.layout_of(tp_ty).align.abi.bytes())
103                 }
104             }
105             sym::vtable_size | sym::vtable_align => {
106                 let vtable = args[0].immediate();
107                 let idx = match name {
108                     sym::vtable_size => ty::COMMON_VTABLE_ENTRIES_SIZE,
109                     sym::vtable_align => ty::COMMON_VTABLE_ENTRIES_ALIGN,
110                     _ => bug!(),
111                 };
112                 let value = meth::VirtualIndex::from_index(idx).get_usize(bx, vtable);
113                 if name == sym::vtable_align {
114                     // Alignment is always nonzero.
115                     bx.range_metadata(value, WrappingRange { start: 1, end: !0 });
116                 };
117                 value
118             }
119             sym::pref_align_of
120             | sym::needs_drop
121             | sym::type_id
122             | sym::type_name
123             | sym::variant_count => {
124                 let value = bx
125                     .tcx()
126                     .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None)
127                     .unwrap();
128                 OperandRef::from_const(bx, value, ret_ty).immediate_or_packed_pair(bx)
129             }
130             sym::offset => {
131                 let ty = substs.type_at(0);
132                 let layout = bx.layout_of(ty);
133                 let ptr = args[0].immediate();
134                 let offset = args[1].immediate();
135                 bx.inbounds_gep(bx.backend_type(layout), ptr, &[offset])
136             }
137             sym::arith_offset => {
138                 let ty = substs.type_at(0);
139                 let layout = bx.layout_of(ty);
140                 let ptr = args[0].immediate();
141                 let offset = args[1].immediate();
142                 bx.gep(bx.backend_type(layout), ptr, &[offset])
143             }
144             sym::copy => {
145                 copy_intrinsic(
146                     bx,
147                     true,
148                     false,
149                     substs.type_at(0),
150                     args[1].immediate(),
151                     args[0].immediate(),
152                     args[2].immediate(),
153                 );
154                 return;
155             }
156             sym::write_bytes => {
157                 memset_intrinsic(
158                     bx,
159                     false,
160                     substs.type_at(0),
161                     args[0].immediate(),
162                     args[1].immediate(),
163                     args[2].immediate(),
164                 );
165                 return;
166             }
167
168             sym::volatile_copy_nonoverlapping_memory => {
169                 copy_intrinsic(
170                     bx,
171                     false,
172                     true,
173                     substs.type_at(0),
174                     args[0].immediate(),
175                     args[1].immediate(),
176                     args[2].immediate(),
177                 );
178                 return;
179             }
180             sym::volatile_copy_memory => {
181                 copy_intrinsic(
182                     bx,
183                     true,
184                     true,
185                     substs.type_at(0),
186                     args[0].immediate(),
187                     args[1].immediate(),
188                     args[2].immediate(),
189                 );
190                 return;
191             }
192             sym::volatile_set_memory => {
193                 memset_intrinsic(
194                     bx,
195                     true,
196                     substs.type_at(0),
197                     args[0].immediate(),
198                     args[1].immediate(),
199                     args[2].immediate(),
200                 );
201                 return;
202             }
203             sym::volatile_store => {
204                 let dst = args[0].deref(bx.cx());
205                 args[1].val.volatile_store(bx, dst);
206                 return;
207             }
208             sym::unaligned_volatile_store => {
209                 let dst = args[0].deref(bx.cx());
210                 args[1].val.unaligned_volatile_store(bx, dst);
211                 return;
212             }
213             sym::add_with_overflow
214             | sym::sub_with_overflow
215             | sym::mul_with_overflow
216             | sym::unchecked_div
217             | sym::unchecked_rem
218             | sym::unchecked_shl
219             | sym::unchecked_shr
220             | sym::unchecked_add
221             | sym::unchecked_sub
222             | sym::unchecked_mul
223             | sym::exact_div => {
224                 let ty = arg_tys[0];
225                 match int_type_width_signed(ty, bx.tcx()) {
226                     Some((_width, signed)) => match name {
227                         sym::add_with_overflow
228                         | sym::sub_with_overflow
229                         | sym::mul_with_overflow => {
230                             let op = match name {
231                                 sym::add_with_overflow => OverflowOp::Add,
232                                 sym::sub_with_overflow => OverflowOp::Sub,
233                                 sym::mul_with_overflow => OverflowOp::Mul,
234                                 _ => bug!(),
235                             };
236                             let (val, overflow) =
237                                 bx.checked_binop(op, ty, args[0].immediate(), args[1].immediate());
238                             // Convert `i1` to a `bool`, and write it to the out parameter
239                             let val = bx.from_immediate(val);
240                             let overflow = bx.from_immediate(overflow);
241
242                             let dest = result.project_field(bx, 0);
243                             bx.store(val, dest.llval, dest.align);
244                             let dest = result.project_field(bx, 1);
245                             bx.store(overflow, dest.llval, dest.align);
246
247                             return;
248                         }
249                         sym::exact_div => {
250                             if signed {
251                                 bx.exactsdiv(args[0].immediate(), args[1].immediate())
252                             } else {
253                                 bx.exactudiv(args[0].immediate(), args[1].immediate())
254                             }
255                         }
256                         sym::unchecked_div => {
257                             if signed {
258                                 bx.sdiv(args[0].immediate(), args[1].immediate())
259                             } else {
260                                 bx.udiv(args[0].immediate(), args[1].immediate())
261                             }
262                         }
263                         sym::unchecked_rem => {
264                             if signed {
265                                 bx.srem(args[0].immediate(), args[1].immediate())
266                             } else {
267                                 bx.urem(args[0].immediate(), args[1].immediate())
268                             }
269                         }
270                         sym::unchecked_shl => bx.shl(args[0].immediate(), args[1].immediate()),
271                         sym::unchecked_shr => {
272                             if signed {
273                                 bx.ashr(args[0].immediate(), args[1].immediate())
274                             } else {
275                                 bx.lshr(args[0].immediate(), args[1].immediate())
276                             }
277                         }
278                         sym::unchecked_add => {
279                             if signed {
280                                 bx.unchecked_sadd(args[0].immediate(), args[1].immediate())
281                             } else {
282                                 bx.unchecked_uadd(args[0].immediate(), args[1].immediate())
283                             }
284                         }
285                         sym::unchecked_sub => {
286                             if signed {
287                                 bx.unchecked_ssub(args[0].immediate(), args[1].immediate())
288                             } else {
289                                 bx.unchecked_usub(args[0].immediate(), args[1].immediate())
290                             }
291                         }
292                         sym::unchecked_mul => {
293                             if signed {
294                                 bx.unchecked_smul(args[0].immediate(), args[1].immediate())
295                             } else {
296                                 bx.unchecked_umul(args[0].immediate(), args[1].immediate())
297                             }
298                         }
299                         _ => bug!(),
300                     },
301                     None => {
302                         span_invalid_monomorphization_error(
303                             bx.tcx().sess,
304                             span,
305                             &format!(
306                                 "invalid monomorphization of `{}` intrinsic: \
307                                       expected basic integer type, found `{}`",
308                                 name, ty
309                             ),
310                         );
311                         return;
312                     }
313                 }
314             }
315             sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
316                 match float_type_width(arg_tys[0]) {
317                     Some(_width) => match name {
318                         sym::fadd_fast => bx.fadd_fast(args[0].immediate(), args[1].immediate()),
319                         sym::fsub_fast => bx.fsub_fast(args[0].immediate(), args[1].immediate()),
320                         sym::fmul_fast => bx.fmul_fast(args[0].immediate(), args[1].immediate()),
321                         sym::fdiv_fast => bx.fdiv_fast(args[0].immediate(), args[1].immediate()),
322                         sym::frem_fast => bx.frem_fast(args[0].immediate(), args[1].immediate()),
323                         _ => bug!(),
324                     },
325                     None => {
326                         span_invalid_monomorphization_error(
327                             bx.tcx().sess,
328                             span,
329                             &format!(
330                                 "invalid monomorphization of `{}` intrinsic: \
331                                       expected basic float type, found `{}`",
332                                 name, arg_tys[0]
333                             ),
334                         );
335                         return;
336                     }
337                 }
338             }
339
340             sym::float_to_int_unchecked => {
341                 if float_type_width(arg_tys[0]).is_none() {
342                     span_invalid_monomorphization_error(
343                         bx.tcx().sess,
344                         span,
345                         &format!(
346                             "invalid monomorphization of `float_to_int_unchecked` \
347                                   intrinsic: expected basic float type, \
348                                   found `{}`",
349                             arg_tys[0]
350                         ),
351                     );
352                     return;
353                 }
354                 let Some((_width, signed)) = int_type_width_signed(ret_ty, bx.tcx()) else {
355                     span_invalid_monomorphization_error(
356                         bx.tcx().sess,
357                         span,
358                         &format!(
359                             "invalid monomorphization of `float_to_int_unchecked` \
360                                     intrinsic:  expected basic integer type, \
361                                     found `{}`",
362                             ret_ty
363                         ),
364                     );
365                     return;
366                 };
367                 if signed {
368                     bx.fptosi(args[0].immediate(), llret_ty)
369                 } else {
370                     bx.fptoui(args[0].immediate(), llret_ty)
371                 }
372             }
373
374             sym::discriminant_value => {
375                 if ret_ty.is_integral() {
376                     args[0].deref(bx.cx()).codegen_get_discr(bx, ret_ty)
377                 } else {
378                     span_bug!(span, "Invalid discriminant type for `{:?}`", arg_tys[0])
379                 }
380             }
381
382             sym::const_allocate => {
383                 // returns a null pointer at runtime.
384                 bx.const_null(bx.type_i8p())
385             }
386
387             sym::const_deallocate => {
388                 // nop at runtime.
389                 return;
390             }
391
392             // This requires that atomic intrinsics follow a specific naming pattern:
393             // "atomic_<operation>[_<ordering>]"
394             name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
395                 use crate::common::AtomicOrdering::*;
396                 use crate::common::{AtomicRmwBinOp, SynchronizationScope};
397
398                 let Some((instruction, ordering)) = atomic.split_once('_') else {
399                     bx.sess().fatal("Atomic intrinsic missing memory ordering");
400                 };
401
402                 let parse_ordering = |bx: &Bx, s| match s {
403                     "unordered" => Unordered,
404                     "relaxed" => Relaxed,
405                     "acquire" => Acquire,
406                     "release" => Release,
407                     "acqrel" => AcquireRelease,
408                     "seqcst" => SequentiallyConsistent,
409                     _ => bx.sess().fatal("unknown ordering in atomic intrinsic"),
410                 };
411
412                 let invalid_monomorphization = |ty| {
413                     span_invalid_monomorphization_error(
414                         bx.tcx().sess,
415                         span,
416                         &format!(
417                             "invalid monomorphization of `{}` intrinsic: \
418                                   expected basic integer type, found `{}`",
419                             name, ty
420                         ),
421                     );
422                 };
423
424                 match instruction {
425                     "cxchg" | "cxchgweak" => {
426                         let Some((success, failure)) = ordering.split_once('_') else {
427                             bx.sess().fatal("Atomic compare-exchange intrinsic missing failure memory ordering");
428                         };
429                         let ty = substs.type_at(0);
430                         if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
431                             let weak = instruction == "cxchgweak";
432                             let mut dst = args[0].immediate();
433                             let mut cmp = args[1].immediate();
434                             let mut src = args[2].immediate();
435                             if ty.is_unsafe_ptr() {
436                                 // Some platforms do not support atomic operations on pointers,
437                                 // so we cast to integer first.
438                                 let ptr_llty = bx.type_ptr_to(bx.type_isize());
439                                 dst = bx.pointercast(dst, ptr_llty);
440                                 cmp = bx.ptrtoint(cmp, bx.type_isize());
441                                 src = bx.ptrtoint(src, bx.type_isize());
442                             }
443                             let pair = bx.atomic_cmpxchg(dst, cmp, src, parse_ordering(bx, success), parse_ordering(bx, failure), weak);
444                             let val = bx.extract_value(pair, 0);
445                             let success = bx.extract_value(pair, 1);
446                             let val = bx.from_immediate(val);
447                             let success = bx.from_immediate(success);
448
449                             let dest = result.project_field(bx, 0);
450                             bx.store(val, dest.llval, dest.align);
451                             let dest = result.project_field(bx, 1);
452                             bx.store(success, dest.llval, dest.align);
453                             return;
454                         } else {
455                             return invalid_monomorphization(ty);
456                         }
457                     }
458
459                     "load" => {
460                         let ty = substs.type_at(0);
461                         if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
462                             let layout = bx.layout_of(ty);
463                             let size = layout.size;
464                             let mut source = args[0].immediate();
465                             if ty.is_unsafe_ptr() {
466                                 // Some platforms do not support atomic operations on pointers,
467                                 // so we cast to integer first...
468                                 let llty = bx.type_isize();
469                                 let ptr_llty = bx.type_ptr_to(llty);
470                                 source = bx.pointercast(source, ptr_llty);
471                                 let result = bx.atomic_load(llty, source, parse_ordering(bx, ordering), size);
472                                 // ... and then cast the result back to a pointer
473                                 bx.inttoptr(result, bx.backend_type(layout))
474                             } else {
475                                 bx.atomic_load(bx.backend_type(layout), source, parse_ordering(bx, ordering), size)
476                             }
477                         } else {
478                             return invalid_monomorphization(ty);
479                         }
480                     }
481
482                     "store" => {
483                         let ty = substs.type_at(0);
484                         if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
485                             let size = bx.layout_of(ty).size;
486                             let mut val = args[1].immediate();
487                             let mut ptr = args[0].immediate();
488                             if ty.is_unsafe_ptr() {
489                                 // Some platforms do not support atomic operations on pointers,
490                                 // so we cast to integer first.
491                                 let ptr_llty = bx.type_ptr_to(bx.type_isize());
492                                 ptr = bx.pointercast(ptr, ptr_llty);
493                                 val = bx.ptrtoint(val, bx.type_isize());
494                             }
495                             bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size);
496                             return;
497                         } else {
498                             return invalid_monomorphization(ty);
499                         }
500                     }
501
502                     "fence" => {
503                         bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::CrossThread);
504                         return;
505                     }
506
507                     "singlethreadfence" => {
508                         bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::SingleThread);
509                         return;
510                     }
511
512                     // These are all AtomicRMW ops
513                     op => {
514                         let atom_op = match op {
515                             "xchg" => AtomicRmwBinOp::AtomicXchg,
516                             "xadd" => AtomicRmwBinOp::AtomicAdd,
517                             "xsub" => AtomicRmwBinOp::AtomicSub,
518                             "and" => AtomicRmwBinOp::AtomicAnd,
519                             "nand" => AtomicRmwBinOp::AtomicNand,
520                             "or" => AtomicRmwBinOp::AtomicOr,
521                             "xor" => AtomicRmwBinOp::AtomicXor,
522                             "max" => AtomicRmwBinOp::AtomicMax,
523                             "min" => AtomicRmwBinOp::AtomicMin,
524                             "umax" => AtomicRmwBinOp::AtomicUMax,
525                             "umin" => AtomicRmwBinOp::AtomicUMin,
526                             _ => bx.sess().fatal("unknown atomic operation"),
527                         };
528
529                         let ty = substs.type_at(0);
530                         if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
531                             let mut ptr = args[0].immediate();
532                             let mut val = args[1].immediate();
533                             if ty.is_unsafe_ptr() {
534                                 // Some platforms do not support atomic operations on pointers,
535                                 // so we cast to integer first.
536                                 let ptr_llty = bx.type_ptr_to(bx.type_isize());
537                                 ptr = bx.pointercast(ptr, ptr_llty);
538                                 val = bx.ptrtoint(val, bx.type_isize());
539                             }
540                             bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
541                         } else {
542                             return invalid_monomorphization(ty);
543                         }
544                     }
545                 }
546             }
547
548             sym::nontemporal_store => {
549                 let dst = args[0].deref(bx.cx());
550                 args[1].val.nontemporal_store(bx, dst);
551                 return;
552             }
553
554             sym::ptr_guaranteed_cmp => {
555                 let a = args[0].immediate();
556                 let b = args[1].immediate();
557                 bx.icmp(IntPredicate::IntEQ, a, b)
558             }
559
560             sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
561                 let ty = substs.type_at(0);
562                 let pointee_size = bx.layout_of(ty).size;
563
564                 let a = args[0].immediate();
565                 let b = args[1].immediate();
566                 let a = bx.ptrtoint(a, bx.type_isize());
567                 let b = bx.ptrtoint(b, bx.type_isize());
568                 let pointee_size = bx.const_usize(pointee_size.bytes());
569                 if name == sym::ptr_offset_from {
570                     // This is the same sequence that Clang emits for pointer subtraction.
571                     // It can be neither `nsw` nor `nuw` because the input is treated as
572                     // unsigned but then the output is treated as signed, so neither works.
573                     let d = bx.sub(a, b);
574                     // this is where the signed magic happens (notice the `s` in `exactsdiv`)
575                     bx.exactsdiv(d, pointee_size)
576                 } else {
577                     // The `_unsigned` version knows the relative ordering of the pointers,
578                     // so can use `sub nuw` and `udiv exact` instead of dealing in signed.
579                     let d = bx.unchecked_usub(a, b);
580                     bx.exactudiv(d, pointee_size)
581                 }
582             }
583
584             _ => {
585                 // Need to use backend-specific things in the implementation.
586                 bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span);
587                 return;
588             }
589         };
590
591         if !fn_abi.ret.is_ignore() {
592             if let PassMode::Cast(ty, _) = &fn_abi.ret.mode {
593                 let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(ty));
594                 let ptr = bx.pointercast(result.llval, ptr_llty);
595                 bx.store(llval, ptr, result.align);
596             } else {
597                 OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout)
598                     .val
599                     .store(bx, result);
600             }
601         }
602     }
603 }
604
605 // Returns the width of an int Ty, and if it's signed or not
606 // Returns None if the type is not an integer
607 // FIXME: there’s multiple of this functions, investigate using some of the already existing
608 // stuffs.
609 fn int_type_width_signed(ty: Ty<'_>, tcx: TyCtxt<'_>) -> Option<(u64, bool)> {
610     match ty.kind() {
611         ty::Int(t) => {
612             Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), true))
613         }
614         ty::Uint(t) => {
615             Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), false))
616         }
617         _ => None,
618     }
619 }
620
621 // Returns the width of a float Ty
622 // Returns None if the type is not a float
623 fn float_type_width(ty: Ty<'_>) -> Option<u64> {
624     match ty.kind() {
625         ty::Float(t) => Some(t.bit_width()),
626         _ => None,
627     }
628 }