2 use rustc_hir::lang_items::LangItem;
3 use rustc_middle::ty::layout::{
4 fn_can_unwind, FnAbiError, HasParamEnv, HasTyCtxt, LayoutCx, LayoutOf, TyAndLayout,
6 use rustc_middle::ty::{self, Ty, TyCtxt};
7 use rustc_session::config::OptLevel;
8 use rustc_span::def_id::DefId;
9 use rustc_target::abi::call::{
10 ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, Conv, FnAbi, PassMode, Reg, RegKind,
12 use rustc_target::abi::*;
13 use rustc_target::spec::abi::Abi as SpecAbi;
17 pub fn provide(providers: &mut ty::query::Providers) {
18 *providers = ty::query::Providers { fn_abi_of_fn_ptr, fn_abi_of_instance, ..*providers };
21 // NOTE(eddyb) this is private to avoid using it from outside of
22 // `fn_abi_of_instance` - any other uses are either too high-level
23 // for `Instance` (e.g. typeck would use `Ty::fn_sig` instead),
24 // or should go through `FnAbi` instead, to avoid losing any
25 // adjustments `fn_abi_of_instance` might be performing.
26 #[tracing::instrument(level = "debug", skip(tcx, param_env))]
27 fn fn_sig_for_fn_abi<'tcx>(
29 instance: ty::Instance<'tcx>,
30 param_env: ty::ParamEnv<'tcx>,
31 ) -> ty::PolyFnSig<'tcx> {
32 let ty = instance.ty(tcx, param_env);
35 // HACK(davidtwco,eddyb): This is a workaround for polymorphization considering
36 // parameters unused if they show up in the signature, but not in the `mir::Body`
37 // (i.e. due to being inside a projection that got normalized, see
38 // `tests/ui/polymorphization/normalized_sig_types.rs`), and codegen not keeping
39 // track of a polymorphization `ParamEnv` to allow normalizing later.
41 // We normalize the `fn_sig` again after substituting at a later point.
42 let mut sig = match *ty.kind() {
43 ty::FnDef(def_id, substs) => tcx
46 tcx.normalize_erasing_regions(tcx.param_env(def_id), fn_sig)
52 if let ty::InstanceDef::VTableShim(..) = instance.def {
53 // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`.
54 sig = sig.map_bound(|mut sig| {
55 let mut inputs_and_output = sig.inputs_and_output.to_vec();
56 inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]);
57 sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
63 ty::Closure(def_id, substs) => {
64 let sig = substs.as_closure().sig();
66 let bound_vars = tcx.mk_bound_variable_kinds(
67 sig.bound_vars().iter().chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))),
69 let br = ty::BoundRegion {
70 var: ty::BoundVar::from_usize(bound_vars.len() - 1),
71 kind: ty::BoundRegionKind::BrEnv,
73 let env_region = ty::ReLateBound(ty::INNERMOST, br);
74 let env_ty = tcx.closure_env_ty(def_id, substs, env_region).unwrap();
76 let sig = sig.skip_binder();
77 ty::Binder::bind_with_vars(
79 iter::once(env_ty).chain(sig.inputs().iter().cloned()),
88 ty::Generator(did, substs, _) => {
89 let sig = substs.as_generator().poly_sig();
91 let bound_vars = tcx.mk_bound_variable_kinds(
92 sig.bound_vars().iter().chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))),
94 let br = ty::BoundRegion {
95 var: ty::BoundVar::from_usize(bound_vars.len() - 1),
96 kind: ty::BoundRegionKind::BrEnv,
98 let env_region = ty::ReLateBound(ty::INNERMOST, br);
99 let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty);
101 let pin_did = tcx.require_lang_item(LangItem::Pin, None);
102 let pin_adt_ref = tcx.adt_def(pin_did);
103 let pin_substs = tcx.intern_substs(&[env_ty.into()]);
104 let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs);
106 let sig = sig.skip_binder();
107 // The `FnSig` and the `ret_ty` here is for a generators main
108 // `Generator::resume(...) -> GeneratorState` function in case we
109 // have an ordinary generator, or the `Future::poll(...) -> Poll`
110 // function in case this is a special generator backing an async construct.
111 let ret_ty = if tcx.generator_is_async(did) {
112 let state_did = tcx.require_lang_item(LangItem::Poll, None);
113 let state_adt_ref = tcx.adt_def(state_did);
114 let state_substs = tcx.intern_substs(&[sig.return_ty.into()]);
115 tcx.mk_adt(state_adt_ref, state_substs)
117 let state_did = tcx.require_lang_item(LangItem::GeneratorState, None);
118 let state_adt_ref = tcx.adt_def(state_did);
119 let state_substs = tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]);
120 tcx.mk_adt(state_adt_ref, state_substs)
123 ty::Binder::bind_with_vars(
125 [env_ty, sig.resume_ty].iter(),
128 hir::Unsafety::Normal,
129 rustc_target::spec::abi::Abi::Rust,
134 _ => bug!("unexpected type {:?} in Instance::fn_sig", ty),
139 fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: SpecAbi) -> Conv {
140 use rustc_target::spec::abi::Abi::*;
141 match tcx.sess.target.adjust_abi(abi) {
142 RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust,
143 RustCold => Conv::RustCold,
145 // It's the ABI's job to select this, not ours.
146 System { .. } => bug!("system abi should be selected elsewhere"),
147 EfiApi => bug!("eficall abi should be selected elsewhere"),
149 Stdcall { .. } => Conv::X86Stdcall,
150 Fastcall { .. } => Conv::X86Fastcall,
151 Vectorcall { .. } => Conv::X86VectorCall,
152 Thiscall { .. } => Conv::X86ThisCall,
154 Unadjusted => Conv::C,
155 Win64 { .. } => Conv::X86_64Win64,
156 SysV64 { .. } => Conv::X86_64SysV,
157 Aapcs { .. } => Conv::ArmAapcs,
158 CCmseNonSecureCall => Conv::CCmseNonSecureCall,
159 PtxKernel => Conv::PtxKernel,
160 Msp430Interrupt => Conv::Msp430Intr,
161 X86Interrupt => Conv::X86Intr,
162 AmdGpuKernel => Conv::AmdGpuKernel,
163 AvrInterrupt => Conv::AvrInterrupt,
164 AvrNonBlockingInterrupt => Conv::AvrNonBlockingInterrupt,
167 // These API constants ought to be more specific...
168 Cdecl { .. } => Conv::C,
172 fn fn_abi_of_fn_ptr<'tcx>(
174 query: ty::ParamEnvAnd<'tcx, (ty::PolyFnSig<'tcx>, &'tcx ty::List<Ty<'tcx>>)>,
175 ) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> {
176 let (param_env, (sig, extra_args)) = query.into_parts();
178 let cx = LayoutCx { tcx, param_env };
179 fn_abi_new_uncached(&cx, sig, extra_args, None, None, false)
182 fn fn_abi_of_instance<'tcx>(
184 query: ty::ParamEnvAnd<'tcx, (ty::Instance<'tcx>, &'tcx ty::List<Ty<'tcx>>)>,
185 ) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> {
186 let (param_env, (instance, extra_args)) = query.into_parts();
188 let sig = fn_sig_for_fn_abi(tcx, instance, param_env);
190 let caller_location = if instance.def.requires_caller_location(tcx) {
191 Some(tcx.caller_location_ty())
197 &LayoutCx { tcx, param_env },
201 Some(instance.def_id()),
202 matches!(instance.def, ty::InstanceDef::Virtual(..)),
206 // Handle safe Rust thin and fat pointers.
207 fn adjust_for_rust_scalar<'tcx>(
208 cx: LayoutCx<'tcx, TyCtxt<'tcx>>,
209 attrs: &mut ArgAttributes,
211 layout: TyAndLayout<'tcx>,
215 // Booleans are always a noundef i1 that needs to be zero-extended.
216 if scalar.is_bool() {
217 attrs.ext(ArgExtension::Zext);
218 attrs.set(ArgAttribute::NoUndef);
222 if !scalar.is_uninit_valid() {
223 attrs.set(ArgAttribute::NoUndef);
226 // Only pointer types handled below.
227 let Scalar::Initialized { value: Pointer, valid_range} = scalar else { return };
229 if !valid_range.contains(0) {
230 attrs.set(ArgAttribute::NonNull);
233 if let Some(pointee) = layout.pointee_info_at(&cx, offset) {
234 if let Some(kind) = pointee.safe {
235 attrs.pointee_align = Some(pointee.align);
237 // `Box` (`UniqueBorrowed`) are not necessarily dereferenceable
238 // for the entire duration of the function as they can be deallocated
239 // at any time. Same for shared mutable references. If LLVM had a
240 // way to say "dereferenceable on entry" we could use it here.
241 attrs.pointee_size = match kind {
242 PointerKind::UniqueBorrowed
243 | PointerKind::UniqueBorrowedPinned
244 | PointerKind::Frozen => pointee.size,
245 PointerKind::SharedMutable | PointerKind::UniqueOwned => Size::ZERO,
248 // The aliasing rules for `Box<T>` are still not decided, but currently we emit
249 // `noalias` for it. This can be turned off using an unstable flag.
250 // See https://github.com/rust-lang/unsafe-code-guidelines/issues/326
251 let noalias_for_box = cx.tcx.sess.opts.unstable_opts.box_noalias;
253 // LLVM prior to version 12 had known miscompiles in the presence of noalias attributes
254 // (see #54878), so it was conditionally disabled, but we don't support earlier
255 // versions at all anymore. We still support turning it off using -Zmutable-noalias.
256 let noalias_mut_ref = cx.tcx.sess.opts.unstable_opts.mutable_noalias;
258 // `&mut` pointer parameters never alias other parameters,
259 // or mutable global data
261 // `&T` where `T` contains no `UnsafeCell<U>` is immutable,
262 // and can be marked as both `readonly` and `noalias`, as
263 // LLVM's definition of `noalias` is based solely on memory
264 // dependencies rather than pointer equality
265 let no_alias = match kind {
266 PointerKind::SharedMutable | PointerKind::UniqueBorrowedPinned => false,
267 PointerKind::UniqueBorrowed => noalias_mut_ref,
268 PointerKind::UniqueOwned => noalias_for_box,
269 PointerKind::Frozen => true,
271 // We can never add `noalias` in return position; that LLVM attribute has some very surprising semantics
272 // (see <https://github.com/rust-lang/unsafe-code-guidelines/issues/385#issuecomment-1368055745>).
273 if no_alias && !is_return {
274 attrs.set(ArgAttribute::NoAlias);
277 if kind == PointerKind::Frozen && !is_return {
278 attrs.set(ArgAttribute::ReadOnly);
284 // FIXME(eddyb) perhaps group the signature/type-containing (or all of them?)
285 // arguments of this method, into a separate `struct`.
286 #[tracing::instrument(level = "debug", skip(cx, caller_location, fn_def_id, force_thin_self_ptr))]
287 fn fn_abi_new_uncached<'tcx>(
288 cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
289 sig: ty::PolyFnSig<'tcx>,
290 extra_args: &[Ty<'tcx>],
291 caller_location: Option<Ty<'tcx>>,
292 fn_def_id: Option<DefId>,
293 // FIXME(eddyb) replace this with something typed, like an `enum`.
294 force_thin_self_ptr: bool,
295 ) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> {
296 let sig = cx.tcx.normalize_erasing_late_bound_regions(cx.param_env, sig);
298 let conv = conv_from_spec_abi(cx.tcx(), sig.abi);
300 let mut inputs = sig.inputs();
301 let extra_args = if sig.abi == RustCall {
302 assert!(!sig.c_variadic && extra_args.is_empty());
304 if let Some(input) = sig.inputs().last() {
305 if let ty::Tuple(tupled_arguments) = input.kind() {
306 inputs = &sig.inputs()[0..sig.inputs().len() - 1];
310 "argument to function with \"rust-call\" ABI \
316 "argument to function with \"rust-call\" ABI \
321 assert!(sig.c_variadic || extra_args.is_empty());
325 let target = &cx.tcx.sess.target;
326 let target_env_gnu_like = matches!(&target.env[..], "gnu" | "musl" | "uclibc");
327 let win_x64_gnu = target.os == "windows" && target.arch == "x86_64" && target.env == "gnu";
328 let linux_s390x_gnu_like =
329 target.os == "linux" && target.arch == "s390x" && target_env_gnu_like;
330 let linux_sparc64_gnu_like =
331 target.os == "linux" && target.arch == "sparc64" && target_env_gnu_like;
332 let linux_powerpc_gnu_like =
333 target.os == "linux" && target.arch == "powerpc" && target_env_gnu_like;
335 let rust_abi = matches!(sig.abi, RustIntrinsic | PlatformIntrinsic | Rust | RustCall);
337 let arg_of = |ty: Ty<'tcx>, arg_idx: Option<usize>| -> Result<_, FnAbiError<'tcx>> {
338 let span = tracing::debug_span!("arg_of");
339 let _entered = span.enter();
340 let is_return = arg_idx.is_none();
342 let layout = cx.layout_of(ty)?;
343 let layout = if force_thin_self_ptr && arg_idx == Some(0) {
344 // Don't pass the vtable, it's not an argument of the virtual fn.
345 // Instead, pass just the data pointer, but give it the type `*const/mut dyn Trait`
346 // or `&/&mut dyn Trait` because this is special-cased elsewhere in codegen
347 make_thin_self_ptr(cx, layout)
352 let mut arg = ArgAbi::new(cx, layout, |layout, scalar, offset| {
353 let mut attrs = ArgAttributes::new();
354 adjust_for_rust_scalar(*cx, &mut attrs, scalar, *layout, offset, is_return);
358 if arg.layout.is_zst() {
359 // For some forsaken reason, x86_64-pc-windows-gnu
360 // doesn't ignore zero-sized struct arguments.
361 // The same is true for {s390x,sparc64,powerpc}-unknown-linux-{gnu,musl,uclibc}.
365 && !linux_s390x_gnu_like
366 && !linux_sparc64_gnu_like
367 && !linux_powerpc_gnu_like)
369 arg.mode = PassMode::Ignore;
376 let mut fn_abi = FnAbi {
377 ret: arg_of(sig.output(), None)?,
381 .chain(extra_args.iter().copied())
382 .chain(caller_location)
384 .map(|(i, ty)| arg_of(ty, Some(i)))
385 .collect::<Result<_, _>>()?,
386 c_variadic: sig.c_variadic,
387 fixed_count: inputs.len() as u32,
389 can_unwind: fn_can_unwind(cx.tcx(), fn_def_id, sig.abi),
391 fn_abi_adjust_for_abi(cx, &mut fn_abi, sig.abi, fn_def_id)?;
392 debug!("fn_abi_new_uncached = {:?}", fn_abi);
393 Ok(cx.tcx.arena.alloc(fn_abi))
396 #[tracing::instrument(level = "trace", skip(cx))]
397 fn fn_abi_adjust_for_abi<'tcx>(
398 cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
399 fn_abi: &mut FnAbi<'tcx, Ty<'tcx>>,
401 fn_def_id: Option<DefId>,
402 ) -> Result<(), FnAbiError<'tcx>> {
403 if abi == SpecAbi::Unadjusted {
407 if abi == SpecAbi::Rust
408 || abi == SpecAbi::RustCall
409 || abi == SpecAbi::RustIntrinsic
410 || abi == SpecAbi::PlatformIntrinsic
412 // Look up the deduced parameter attributes for this function, if we have its def ID and
413 // we're optimizing in non-incremental mode. We'll tag its parameters with those attributes
415 let deduced_param_attrs = if cx.tcx.sess.opts.optimize != OptLevel::No
416 && cx.tcx.sess.opts.incremental.is_none()
418 fn_def_id.map(|fn_def_id| cx.tcx.deduced_param_attrs(fn_def_id)).unwrap_or_default()
423 let fixup = |arg: &mut ArgAbi<'tcx, Ty<'tcx>>, arg_idx: Option<usize>| {
428 match arg.layout.abi {
429 Abi::Aggregate { .. } => {}
431 // This is a fun case! The gist of what this is doing is
432 // that we want callers and callees to always agree on the
433 // ABI of how they pass SIMD arguments. If we were to *not*
434 // make these arguments indirect then they'd be immediates
435 // in LLVM, which means that they'd used whatever the
436 // appropriate ABI is for the callee and the caller. That
437 // means, for example, if the caller doesn't have AVX
438 // enabled but the callee does, then passing an AVX argument
439 // across this boundary would cause corrupt data to show up.
441 // This problem is fixed by unconditionally passing SIMD
442 // arguments through memory between callers and callees
443 // which should get them all to agree on ABI regardless of
444 // target feature sets. Some more information about this
445 // issue can be found in #44367.
447 // Note that the platform intrinsic ABI is exempt here as
448 // that's how we connect up to LLVM and it's unstable
449 // anyway, we control all calls to it in libstd.
451 if abi != SpecAbi::PlatformIntrinsic
452 && cx.tcx.sess.target.simd_types_indirect =>
461 let size = arg.layout.size;
462 if arg.layout.is_unsized() || size > Pointer.size(cx) {
465 // We want to pass small aggregates as immediates, but using
466 // a LLVM aggregate type for this leads to bad optimizations,
467 // so we pick an appropriately sized integer type instead.
468 arg.cast_to(Reg { kind: RegKind::Integer, size });
471 // If we deduced that this parameter was read-only, add that to the attribute list now.
473 // The `readonly` parameter only applies to pointers, so we can only do this if the
474 // argument was passed indirectly. (If the argument is passed directly, it's an SSA
475 // value, so it's implicitly immutable.)
476 if let (Some(arg_idx), &mut PassMode::Indirect { ref mut attrs, .. }) =
477 (arg_idx, &mut arg.mode)
479 // The `deduced_param_attrs` list could be empty if this is a type of function
480 // we can't deduce any parameters for, so make sure the argument index is in
482 if let Some(deduced_param_attrs) = deduced_param_attrs.get(arg_idx) {
483 if deduced_param_attrs.read_only {
484 attrs.regular.insert(ArgAttribute::ReadOnly);
485 debug!("added deduced read-only attribute");
491 fixup(&mut fn_abi.ret, None);
492 for (arg_idx, arg) in fn_abi.args.iter_mut().enumerate() {
493 fixup(arg, Some(arg_idx));
496 fn_abi.adjust_for_foreign_abi(cx, abi)?;
502 #[tracing::instrument(level = "debug", skip(cx))]
503 fn make_thin_self_ptr<'tcx>(
504 cx: &(impl HasTyCtxt<'tcx> + HasParamEnv<'tcx>),
505 layout: TyAndLayout<'tcx>,
506 ) -> TyAndLayout<'tcx> {
508 let fat_pointer_ty = if layout.is_unsized() {
509 // unsized `self` is passed as a pointer to `self`
510 // FIXME (mikeyhew) change this to use &own if it is ever added to the language
511 tcx.mk_mut_ptr(layout.ty)
514 Abi::ScalarPair(..) | Abi::Scalar(..) => (),
515 _ => bug!("receiver type has unsupported layout: {:?}", layout),
518 // In the case of Rc<Self>, we need to explicitly pass a *mut RcBox<Self>
519 // with a Scalar (not ScalarPair) ABI. This is a hack that is understood
520 // elsewhere in the compiler as a method on a `dyn Trait`.
521 // To get the type `*mut RcBox<Self>`, we just keep unwrapping newtypes until we
522 // get a built-in pointer type
523 let mut fat_pointer_layout = layout;
524 'descend_newtypes: while !fat_pointer_layout.ty.is_unsafe_ptr()
525 && !fat_pointer_layout.ty.is_region_ptr()
527 for i in 0..fat_pointer_layout.fields.count() {
528 let field_layout = fat_pointer_layout.field(cx, i);
530 if !field_layout.is_zst() {
531 fat_pointer_layout = field_layout;
532 continue 'descend_newtypes;
536 bug!("receiver has no non-zero-sized fields {:?}", fat_pointer_layout);
539 fat_pointer_layout.ty
542 // we now have a type like `*mut RcBox<dyn Trait>`
543 // change its layout to that of `*mut ()`, a thin pointer, but keep the same type
544 // this is understood as a special case elsewhere in the compiler
545 let unit_ptr_ty = tcx.mk_mut_ptr(tcx.mk_unit());
550 // NOTE(eddyb) using an empty `ParamEnv`, and `unwrap`-ing the `Result`
551 // should always work because the type is always `*mut ()`.
552 ..tcx.layout_of(ty::ParamEnv::reveal_all().and(unit_ptr_ty)).unwrap()