use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
-use crate::mir::{GeneratorLayout, GeneratorSavedLocal};
use crate::ty::normalize_erasing_regions::NormalizationError;
-use crate::ty::{
- self, layout_sanity_check::sanity_check_layout, subst::SubstsRef, EarlyBinder, ReprOptions, Ty,
- TyCtxt, TypeVisitable,
-};
+use crate::ty::{self, ReprOptions, Ty, TyCtxt, TypeVisitable};
use rustc_ast as ast;
use rustc_attr as attr;
use rustc_errors::{DiagnosticBuilder, Handler, IntoDiagnostic};
use rustc_hir as hir;
use rustc_hir::def_id::DefId;
-use rustc_hir::lang_items::LangItem;
-use rustc_index::bit_set::BitSet;
-use rustc_index::vec::{Idx, IndexVec};
-use rustc_session::{config::OptLevel, DataTypeKind, FieldInfo, SizeKind, VariantInfo};
-use rustc_span::symbol::Symbol;
+use rustc_index::vec::Idx;
+use rustc_session::config::OptLevel;
use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::call::{
- ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, Conv, FnAbi, PassMode, Reg, RegKind,
-};
+use rustc_target::abi::call::FnAbi;
use rustc_target::abi::*;
use rustc_target::spec::{abi::Abi as SpecAbi, HasTargetSpec, PanicStrategy, Target};
-use std::cmp::{self, Ordering};
+use std::cmp::{self};
use std::fmt;
-use std::iter;
use std::num::NonZeroUsize;
use std::ops::Bound;
-use rand::{seq::SliceRandom, SeedableRng};
-use rand_xoshiro::Xoshiro128StarStar;
-
-pub fn provide(providers: &mut ty::query::Providers) {
- *providers =
- ty::query::Providers { layout_of, fn_abi_of_fn_ptr, fn_abi_of_instance, ..*providers };
-}
-
pub trait IntegerExt {
fn to_ty<'tcx>(&self, tcx: TyCtxt<'tcx>, signed: bool) -> Ty<'tcx>;
fn from_attr<C: HasDataLayout>(cx: &C, ity: attr::IntType) -> Integer;
}
}
-#[instrument(skip(tcx, query), level = "debug")]
-fn layout_of<'tcx>(
- tcx: TyCtxt<'tcx>,
- query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
-) -> Result<TyAndLayout<'tcx>, LayoutError<'tcx>> {
- let (param_env, ty) = query.into_parts();
- debug!(?ty);
-
- let param_env = param_env.with_reveal_all_normalized(tcx);
- let unnormalized_ty = ty;
-
- // FIXME: We might want to have two different versions of `layout_of`:
- // One that can be called after typecheck has completed and can use
- // `normalize_erasing_regions` here and another one that can be called
- // before typecheck has completed and uses `try_normalize_erasing_regions`.
- let ty = match tcx.try_normalize_erasing_regions(param_env, ty) {
- Ok(t) => t,
- Err(normalization_error) => {
- return Err(LayoutError::NormalizationFailure(ty, normalization_error));
- }
- };
-
- if ty != unnormalized_ty {
- // Ensure this layout is also cached for the normalized type.
- return tcx.layout_of(param_env.and(ty));
- }
-
- let cx = LayoutCx { tcx, param_env };
-
- let layout = cx.layout_of_uncached(ty)?;
- let layout = TyAndLayout { ty, layout };
-
- cx.record_layout_for_printing(layout);
-
- sanity_check_layout(&cx, &layout);
-
- Ok(layout)
-}
-
#[derive(Clone, Copy)]
pub struct LayoutCx<'tcx, C> {
pub tcx: C,
pub param_env: ty::ParamEnv<'tcx>,
}
-#[derive(Copy, Clone, Debug)]
-enum StructKind {
- /// A tuple, closure, or univariant which cannot be coerced to unsized.
- AlwaysSized,
- /// A univariant, the last field of which may be coerced to unsized.
- MaybeUnsized,
- /// A univariant, but with a prefix of an arbitrary size & alignment (e.g., enum tag).
- Prefixed(Size, Align),
-}
-
-// Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`.
-// This is used to go between `memory_index` (source field order to memory order)
-// and `inverse_memory_index` (memory order to source field order).
-// See also `FieldsShape::Arbitrary::memory_index` for more details.
-// FIXME(eddyb) build a better abstraction for permutations, if possible.
-fn invert_mapping(map: &[u32]) -> Vec<u32> {
- let mut inverse = vec![0; map.len()];
- for i in 0..map.len() {
- inverse[map[i] as usize] = i as u32;
- }
- inverse
-}
-
-impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
- fn scalar_pair(&self, a: Scalar, b: Scalar) -> LayoutS<'tcx> {
- let dl = self.data_layout();
- let b_align = b.align(dl);
- let align = a.align(dl).max(b_align).max(dl.aggregate_align);
- let b_offset = a.size(dl).align_to(b_align.abi);
- let size = (b_offset + b.size(dl)).align_to(align.abi);
-
- // HACK(nox): We iter on `b` and then `a` because `max_by_key`
- // returns the last maximum.
- let largest_niche = Niche::from_scalar(dl, b_offset, b)
- .into_iter()
- .chain(Niche::from_scalar(dl, Size::ZERO, a))
- .max_by_key(|niche| niche.available(dl));
-
- LayoutS {
- variants: Variants::Single { index: VariantIdx::new(0) },
- fields: FieldsShape::Arbitrary {
- offsets: vec![Size::ZERO, b_offset],
- memory_index: vec![0, 1],
- },
- abi: Abi::ScalarPair(a, b),
- largest_niche,
- align,
- size,
- }
- }
-
- fn univariant_uninterned(
- &self,
- ty: Ty<'tcx>,
- fields: &[TyAndLayout<'_>],
- repr: &ReprOptions,
- kind: StructKind,
- ) -> Result<LayoutS<'tcx>, LayoutError<'tcx>> {
- let dl = self.data_layout();
- let pack = repr.pack;
- if pack.is_some() && repr.align.is_some() {
- self.tcx.sess.delay_span_bug(DUMMY_SP, "struct cannot be packed and aligned");
- return Err(LayoutError::Unknown(ty));
- }
-
- let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
-
- let mut inverse_memory_index: Vec<u32> = (0..fields.len() as u32).collect();
-
- let optimize = !repr.inhibit_struct_field_reordering_opt();
- if optimize {
- let end =
- if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
- let optimizing = &mut inverse_memory_index[..end];
- let field_align = |f: &TyAndLayout<'_>| {
- if let Some(pack) = pack { f.align.abi.min(pack) } else { f.align.abi }
- };
-
- // If `-Z randomize-layout` was enabled for the type definition we can shuffle
- // the field ordering to try and catch some code making assumptions about layouts
- // we don't guarantee
- if repr.can_randomize_type_layout() {
- // `ReprOptions.layout_seed` is a deterministic seed that we can use to
- // randomize field ordering with
- let mut rng = Xoshiro128StarStar::seed_from_u64(repr.field_shuffle_seed);
-
- // Shuffle the ordering of the fields
- optimizing.shuffle(&mut rng);
-
- // Otherwise we just leave things alone and actually optimize the type's fields
- } else {
- match kind {
- StructKind::AlwaysSized | StructKind::MaybeUnsized => {
- optimizing.sort_by_key(|&x| {
- // Place ZSTs first to avoid "interesting offsets",
- // especially with only one or two non-ZST fields.
- let f = &fields[x as usize];
- (!f.is_zst(), cmp::Reverse(field_align(f)))
- });
- }
-
- StructKind::Prefixed(..) => {
- // Sort in ascending alignment so that the layout stays optimal
- // regardless of the prefix
- optimizing.sort_by_key(|&x| field_align(&fields[x as usize]));
- }
- }
-
- // FIXME(Kixiron): We can always shuffle fields within a given alignment class
- // regardless of the status of `-Z randomize-layout`
- }
- }
-
- // inverse_memory_index holds field indices by increasing memory offset.
- // That is, if field 5 has offset 0, the first element of inverse_memory_index is 5.
- // We now write field offsets to the corresponding offset slot;
- // field 5 with offset 0 puts 0 in offsets[5].
- // At the bottom of this function, we invert `inverse_memory_index` to
- // produce `memory_index` (see `invert_mapping`).
-
- let mut sized = true;
- let mut offsets = vec![Size::ZERO; fields.len()];
- let mut offset = Size::ZERO;
- let mut largest_niche = None;
- let mut largest_niche_available = 0;
-
- if let StructKind::Prefixed(prefix_size, prefix_align) = kind {
- let prefix_align =
- if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align };
- align = align.max(AbiAndPrefAlign::new(prefix_align));
- offset = prefix_size.align_to(prefix_align);
- }
-
- for &i in &inverse_memory_index {
- let field = fields[i as usize];
- if !sized {
- self.tcx.sess.delay_span_bug(
- DUMMY_SP,
- &format!(
- "univariant: field #{} of `{}` comes after unsized field",
- offsets.len(),
- ty
- ),
- );
- }
-
- if field.is_unsized() {
- sized = false;
- }
-
- // Invariant: offset < dl.obj_size_bound() <= 1<<61
- let field_align = if let Some(pack) = pack {
- field.align.min(AbiAndPrefAlign::new(pack))
- } else {
- field.align
- };
- offset = offset.align_to(field_align.abi);
- align = align.max(field_align);
-
- debug!("univariant offset: {:?} field: {:#?}", offset, field);
- offsets[i as usize] = offset;
-
- if let Some(mut niche) = field.largest_niche {
- let available = niche.available(dl);
- if available > largest_niche_available {
- largest_niche_available = available;
- niche.offset += offset;
- largest_niche = Some(niche);
- }
- }
-
- offset = offset.checked_add(field.size, dl).ok_or(LayoutError::SizeOverflow(ty))?;
- }
-
- if let Some(repr_align) = repr.align {
- align = align.max(AbiAndPrefAlign::new(repr_align));
- }
-
- debug!("univariant min_size: {:?}", offset);
- let min_size = offset;
-
- // As stated above, inverse_memory_index holds field indices by increasing offset.
- // This makes it an already-sorted view of the offsets vec.
- // To invert it, consider:
- // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
- // Field 5 would be the first element, so memory_index is i:
- // Note: if we didn't optimize, it's already right.
-
- let memory_index =
- if optimize { invert_mapping(&inverse_memory_index) } else { inverse_memory_index };
-
- let size = min_size.align_to(align.abi);
- let mut abi = Abi::Aggregate { sized };
-
- // Unpack newtype ABIs and find scalar pairs.
- if sized && size.bytes() > 0 {
- // All other fields must be ZSTs.
- let mut non_zst_fields = fields.iter().enumerate().filter(|&(_, f)| !f.is_zst());
-
- match (non_zst_fields.next(), non_zst_fields.next(), non_zst_fields.next()) {
- // We have exactly one non-ZST field.
- (Some((i, field)), None, None) => {
- // Field fills the struct and it has a scalar or scalar pair ABI.
- if offsets[i].bytes() == 0 && align.abi == field.align.abi && size == field.size
- {
- match field.abi {
- // For plain scalars, or vectors of them, we can't unpack
- // newtypes for `#[repr(C)]`, as that affects C ABIs.
- Abi::Scalar(_) | Abi::Vector { .. } if optimize => {
- abi = field.abi;
- }
- // But scalar pairs are Rust-specific and get
- // treated as aggregates by C ABIs anyway.
- Abi::ScalarPair(..) => {
- abi = field.abi;
- }
- _ => {}
- }
- }
- }
-
- // Two non-ZST fields, and they're both scalars.
- (Some((i, a)), Some((j, b)), None) => {
- match (a.abi, b.abi) {
- (Abi::Scalar(a), Abi::Scalar(b)) => {
- // Order by the memory placement, not source order.
- let ((i, a), (j, b)) = if offsets[i] < offsets[j] {
- ((i, a), (j, b))
- } else {
- ((j, b), (i, a))
- };
- let pair = self.scalar_pair(a, b);
- let pair_offsets = match pair.fields {
- FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
- assert_eq!(memory_index, &[0, 1]);
- offsets
- }
- _ => bug!(),
- };
- if offsets[i] == pair_offsets[0]
- && offsets[j] == pair_offsets[1]
- && align == pair.align
- && size == pair.size
- {
- // We can use `ScalarPair` only when it matches our
- // already computed layout (including `#[repr(C)]`).
- abi = pair.abi;
- }
- }
- _ => {}
- }
- }
-
- _ => {}
- }
- }
-
- if fields.iter().any(|f| f.abi.is_uninhabited()) {
- abi = Abi::Uninhabited;
- }
-
- Ok(LayoutS {
- variants: Variants::Single { index: VariantIdx::new(0) },
- fields: FieldsShape::Arbitrary { offsets, memory_index },
- abi,
- largest_niche,
- align,
- size,
- })
- }
-
- fn layout_of_uncached(&self, ty: Ty<'tcx>) -> Result<Layout<'tcx>, LayoutError<'tcx>> {
- let tcx = self.tcx;
- let param_env = self.param_env;
- let dl = self.data_layout();
- let scalar_unit = |value: Primitive| {
- let size = value.size(dl);
- assert!(size.bits() <= 128);
- Scalar::Initialized { value, valid_range: WrappingRange::full(size) }
- };
- let scalar =
- |value: Primitive| tcx.intern_layout(LayoutS::scalar(self, scalar_unit(value)));
-
- let univariant = |fields: &[TyAndLayout<'_>], repr: &ReprOptions, kind| {
- Ok(tcx.intern_layout(self.univariant_uninterned(ty, fields, repr, kind)?))
- };
- debug_assert!(!ty.has_infer_types_or_consts());
-
- Ok(match *ty.kind() {
- // Basic scalars.
- ty::Bool => tcx.intern_layout(LayoutS::scalar(
- self,
- Scalar::Initialized {
- value: Int(I8, false),
- valid_range: WrappingRange { start: 0, end: 1 },
- },
- )),
- ty::Char => tcx.intern_layout(LayoutS::scalar(
- self,
- Scalar::Initialized {
- value: Int(I32, false),
- valid_range: WrappingRange { start: 0, end: 0x10FFFF },
- },
- )),
- ty::Int(ity) => scalar(Int(Integer::from_int_ty(dl, ity), true)),
- ty::Uint(ity) => scalar(Int(Integer::from_uint_ty(dl, ity), false)),
- ty::Float(fty) => scalar(match fty {
- ty::FloatTy::F32 => F32,
- ty::FloatTy::F64 => F64,
- }),
- ty::FnPtr(_) => {
- let mut ptr = scalar_unit(Pointer);
- ptr.valid_range_mut().start = 1;
- tcx.intern_layout(LayoutS::scalar(self, ptr))
- }
-
- // The never type.
- ty::Never => tcx.intern_layout(LayoutS {
- variants: Variants::Single { index: VariantIdx::new(0) },
- fields: FieldsShape::Primitive,
- abi: Abi::Uninhabited,
- largest_niche: None,
- align: dl.i8_align,
- size: Size::ZERO,
- }),
-
- // Potentially-wide pointers.
- ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => {
- let mut data_ptr = scalar_unit(Pointer);
- if !ty.is_unsafe_ptr() {
- data_ptr.valid_range_mut().start = 1;
- }
-
- let pointee = tcx.normalize_erasing_regions(param_env, pointee);
- if pointee.is_sized(tcx.at(DUMMY_SP), param_env) {
- return Ok(tcx.intern_layout(LayoutS::scalar(self, data_ptr)));
- }
-
- let unsized_part = tcx.struct_tail_erasing_lifetimes(pointee, param_env);
- let metadata = match unsized_part.kind() {
- ty::Foreign(..) => {
- return Ok(tcx.intern_layout(LayoutS::scalar(self, data_ptr)));
- }
- ty::Slice(_) | ty::Str => scalar_unit(Int(dl.ptr_sized_integer(), false)),
- ty::Dynamic(..) => {
- let mut vtable = scalar_unit(Pointer);
- vtable.valid_range_mut().start = 1;
- vtable
- }
- _ => return Err(LayoutError::Unknown(unsized_part)),
- };
-
- // Effectively a (ptr, meta) tuple.
- tcx.intern_layout(self.scalar_pair(data_ptr, metadata))
- }
-
- ty::Dynamic(_, _, ty::DynStar) => {
- let mut data = scalar_unit(Int(dl.ptr_sized_integer(), false));
- data.valid_range_mut().start = 0;
- let mut vtable = scalar_unit(Pointer);
- vtable.valid_range_mut().start = 1;
- tcx.intern_layout(self.scalar_pair(data, vtable))
- }
-
- // Arrays and slices.
- ty::Array(element, mut count) => {
- if count.has_projections() {
- count = tcx.normalize_erasing_regions(param_env, count);
- if count.has_projections() {
- return Err(LayoutError::Unknown(ty));
- }
- }
-
- let count = count.try_eval_usize(tcx, param_env).ok_or(LayoutError::Unknown(ty))?;
- let element = self.layout_of(element)?;
- let size =
- element.size.checked_mul(count, dl).ok_or(LayoutError::SizeOverflow(ty))?;
-
- let abi =
- if count != 0 && tcx.conservative_is_privately_uninhabited(param_env.and(ty)) {
- Abi::Uninhabited
- } else {
- Abi::Aggregate { sized: true }
- };
-
- let largest_niche = if count != 0 { element.largest_niche } else { None };
-
- tcx.intern_layout(LayoutS {
- variants: Variants::Single { index: VariantIdx::new(0) },
- fields: FieldsShape::Array { stride: element.size, count },
- abi,
- largest_niche,
- align: element.align,
- size,
- })
- }
- ty::Slice(element) => {
- let element = self.layout_of(element)?;
- tcx.intern_layout(LayoutS {
- variants: Variants::Single { index: VariantIdx::new(0) },
- fields: FieldsShape::Array { stride: element.size, count: 0 },
- abi: Abi::Aggregate { sized: false },
- largest_niche: None,
- align: element.align,
- size: Size::ZERO,
- })
- }
- ty::Str => tcx.intern_layout(LayoutS {
- variants: Variants::Single { index: VariantIdx::new(0) },
- fields: FieldsShape::Array { stride: Size::from_bytes(1), count: 0 },
- abi: Abi::Aggregate { sized: false },
- largest_niche: None,
- align: dl.i8_align,
- size: Size::ZERO,
- }),
-
- // Odd unit types.
- ty::FnDef(..) => univariant(&[], &ReprOptions::default(), StructKind::AlwaysSized)?,
- ty::Dynamic(_, _, ty::Dyn) | ty::Foreign(..) => {
- let mut unit = self.univariant_uninterned(
- ty,
- &[],
- &ReprOptions::default(),
- StructKind::AlwaysSized,
- )?;
- match unit.abi {
- Abi::Aggregate { ref mut sized } => *sized = false,
- _ => bug!(),
- }
- tcx.intern_layout(unit)
- }
-
- ty::Generator(def_id, substs, _) => self.generator_layout(ty, def_id, substs)?,
-
- ty::Closure(_, ref substs) => {
- let tys = substs.as_closure().upvar_tys();
- univariant(
- &tys.map(|ty| self.layout_of(ty)).collect::<Result<Vec<_>, _>>()?,
- &ReprOptions::default(),
- StructKind::AlwaysSized,
- )?
- }
-
- ty::Tuple(tys) => {
- let kind =
- if tys.len() == 0 { StructKind::AlwaysSized } else { StructKind::MaybeUnsized };
-
- univariant(
- &tys.iter().map(|k| self.layout_of(k)).collect::<Result<Vec<_>, _>>()?,
- &ReprOptions::default(),
- kind,
- )?
- }
-
- // SIMD vector types.
- ty::Adt(def, substs) if def.repr().simd() => {
- if !def.is_struct() {
- // Should have yielded E0517 by now.
- tcx.sess.delay_span_bug(
- DUMMY_SP,
- "#[repr(simd)] was applied to an ADT that is not a struct",
- );
- return Err(LayoutError::Unknown(ty));
- }
-
- // Supported SIMD vectors are homogeneous ADTs with at least one field:
- //
- // * #[repr(simd)] struct S(T, T, T, T);
- // * #[repr(simd)] struct S { x: T, y: T, z: T, w: T }
- // * #[repr(simd)] struct S([T; 4])
- //
- // where T is a primitive scalar (integer/float/pointer).
-
- // SIMD vectors with zero fields are not supported.
- // (should be caught by typeck)
- if def.non_enum_variant().fields.is_empty() {
- tcx.sess.fatal(&format!("monomorphising SIMD type `{}` of zero length", ty));
- }
-
- // Type of the first ADT field:
- let f0_ty = def.non_enum_variant().fields[0].ty(tcx, substs);
-
- // Heterogeneous SIMD vectors are not supported:
- // (should be caught by typeck)
- for fi in &def.non_enum_variant().fields {
- if fi.ty(tcx, substs) != f0_ty {
- tcx.sess.fatal(&format!("monomorphising heterogeneous SIMD type `{}`", ty));
- }
- }
-
- // The element type and number of elements of the SIMD vector
- // are obtained from:
- //
- // * the element type and length of the single array field, if
- // the first field is of array type, or
- //
- // * the homogeneous field type and the number of fields.
- let (e_ty, e_len, is_array) = if let ty::Array(e_ty, _) = f0_ty.kind() {
- // First ADT field is an array:
-
- // SIMD vectors with multiple array fields are not supported:
- // (should be caught by typeck)
- if def.non_enum_variant().fields.len() != 1 {
- tcx.sess.fatal(&format!(
- "monomorphising SIMD type `{}` with more than one array field",
- ty
- ));
- }
-
- // Extract the number of elements from the layout of the array field:
- let FieldsShape::Array { count, .. } = self.layout_of(f0_ty)?.layout.fields() else {
- return Err(LayoutError::Unknown(ty));
- };
-
- (*e_ty, *count, true)
- } else {
- // First ADT field is not an array:
- (f0_ty, def.non_enum_variant().fields.len() as _, false)
- };
-
- // SIMD vectors of zero length are not supported.
- // Additionally, lengths are capped at 2^16 as a fixed maximum backends must
- // support.
- //
- // Can't be caught in typeck if the array length is generic.
- if e_len == 0 {
- tcx.sess.fatal(&format!("monomorphising SIMD type `{}` of zero length", ty));
- } else if e_len > MAX_SIMD_LANES {
- tcx.sess.fatal(&format!(
- "monomorphising SIMD type `{}` of length greater than {}",
- ty, MAX_SIMD_LANES,
- ));
- }
-
- // Compute the ABI of the element type:
- let e_ly = self.layout_of(e_ty)?;
- let Abi::Scalar(e_abi) = e_ly.abi else {
- // This error isn't caught in typeck, e.g., if
- // the element type of the vector is generic.
- tcx.sess.fatal(&format!(
- "monomorphising SIMD type `{}` with a non-primitive-scalar \
- (integer/float/pointer) element type `{}`",
- ty, e_ty
- ))
- };
-
- // Compute the size and alignment of the vector:
- let size = e_ly.size.checked_mul(e_len, dl).ok_or(LayoutError::SizeOverflow(ty))?;
- let align = dl.vector_align(size);
- let size = size.align_to(align.abi);
-
- // Compute the placement of the vector fields:
- let fields = if is_array {
- FieldsShape::Arbitrary { offsets: vec![Size::ZERO], memory_index: vec![0] }
- } else {
- FieldsShape::Array { stride: e_ly.size, count: e_len }
- };
-
- tcx.intern_layout(LayoutS {
- variants: Variants::Single { index: VariantIdx::new(0) },
- fields,
- abi: Abi::Vector { element: e_abi, count: e_len },
- largest_niche: e_ly.largest_niche,
- size,
- align,
- })
- }
-
- // ADTs.
- ty::Adt(def, substs) => {
- // Cache the field layouts.
- let variants = def
- .variants()
- .iter()
- .map(|v| {
- v.fields
- .iter()
- .map(|field| self.layout_of(field.ty(tcx, substs)))
- .collect::<Result<Vec<_>, _>>()
- })
- .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
-
- if def.is_union() {
- if def.repr().pack.is_some() && def.repr().align.is_some() {
- self.tcx.sess.delay_span_bug(
- tcx.def_span(def.did()),
- "union cannot be packed and aligned",
- );
- return Err(LayoutError::Unknown(ty));
- }
-
- let mut align =
- if def.repr().pack.is_some() { dl.i8_align } else { dl.aggregate_align };
-
- if let Some(repr_align) = def.repr().align {
- align = align.max(AbiAndPrefAlign::new(repr_align));
- }
-
- let optimize = !def.repr().inhibit_union_abi_opt();
- let mut size = Size::ZERO;
- let mut abi = Abi::Aggregate { sized: true };
- let index = VariantIdx::new(0);
- for field in &variants[index] {
- assert!(!field.is_unsized());
- align = align.max(field.align);
-
- // If all non-ZST fields have the same ABI, forward this ABI
- if optimize && !field.is_zst() {
- // Discard valid range information and allow undef
- let field_abi = match field.abi {
- Abi::Scalar(x) => Abi::Scalar(x.to_union()),
- Abi::ScalarPair(x, y) => {
- Abi::ScalarPair(x.to_union(), y.to_union())
- }
- Abi::Vector { element: x, count } => {
- Abi::Vector { element: x.to_union(), count }
- }
- Abi::Uninhabited | Abi::Aggregate { .. } => {
- Abi::Aggregate { sized: true }
- }
- };
-
- if size == Size::ZERO {
- // first non ZST: initialize 'abi'
- abi = field_abi;
- } else if abi != field_abi {
- // different fields have different ABI: reset to Aggregate
- abi = Abi::Aggregate { sized: true };
- }
- }
-
- size = cmp::max(size, field.size);
- }
-
- if let Some(pack) = def.repr().pack {
- align = align.min(AbiAndPrefAlign::new(pack));
- }
-
- return Ok(tcx.intern_layout(LayoutS {
- variants: Variants::Single { index },
- fields: FieldsShape::Union(
- NonZeroUsize::new(variants[index].len())
- .ok_or(LayoutError::Unknown(ty))?,
- ),
- abi,
- largest_niche: None,
- align,
- size: size.align_to(align.abi),
- }));
- }
-
- // A variant is absent if it's uninhabited and only has ZST fields.
- // Present uninhabited variants only require space for their fields,
- // but *not* an encoding of the discriminant (e.g., a tag value).
- // See issue #49298 for more details on the need to leave space
- // for non-ZST uninhabited data (mostly partial initialization).
- let absent = |fields: &[TyAndLayout<'_>]| {
- let uninhabited = fields.iter().any(|f| f.abi.is_uninhabited());
- let is_zst = fields.iter().all(|f| f.is_zst());
- uninhabited && is_zst
- };
- let (present_first, present_second) = {
- let mut present_variants = variants
- .iter_enumerated()
- .filter_map(|(i, v)| if absent(v) { None } else { Some(i) });
- (present_variants.next(), present_variants.next())
- };
- let present_first = match present_first {
- Some(present_first) => present_first,
- // Uninhabited because it has no variants, or only absent ones.
- None if def.is_enum() => {
- return Ok(tcx.layout_of(param_env.and(tcx.types.never))?.layout);
- }
- // If it's a struct, still compute a layout so that we can still compute the
- // field offsets.
- None => VariantIdx::new(0),
- };
-
- let is_struct = !def.is_enum() ||
- // Only one variant is present.
- (present_second.is_none() &&
- // Representation optimizations are allowed.
- !def.repr().inhibit_enum_layout_opt());
- if is_struct {
- // Struct, or univariant enum equivalent to a struct.
- // (Typechecking will reject discriminant-sizing attrs.)
-
- let v = present_first;
- let kind = if def.is_enum() || variants[v].is_empty() {
- StructKind::AlwaysSized
- } else {
- let param_env = tcx.param_env(def.did());
- let last_field = def.variant(v).fields.last().unwrap();
- let always_sized =
- tcx.type_of(last_field.did).is_sized(tcx.at(DUMMY_SP), param_env);
- if !always_sized {
- StructKind::MaybeUnsized
- } else {
- StructKind::AlwaysSized
- }
- };
-
- let mut st = self.univariant_uninterned(ty, &variants[v], &def.repr(), kind)?;
- st.variants = Variants::Single { index: v };
-
- if def.is_unsafe_cell() {
- let hide_niches = |scalar: &mut _| match scalar {
- Scalar::Initialized { value, valid_range } => {
- *valid_range = WrappingRange::full(value.size(dl))
- }
- // Already doesn't have any niches
- Scalar::Union { .. } => {}
- };
- match &mut st.abi {
- Abi::Uninhabited => {}
- Abi::Scalar(scalar) => hide_niches(scalar),
- Abi::ScalarPair(a, b) => {
- hide_niches(a);
- hide_niches(b);
- }
- Abi::Vector { element, count: _ } => hide_niches(element),
- Abi::Aggregate { sized: _ } => {}
- }
- st.largest_niche = None;
- return Ok(tcx.intern_layout(st));
- }
-
- let (start, end) = self.tcx.layout_scalar_valid_range(def.did());
- match st.abi {
- Abi::Scalar(ref mut scalar) | Abi::ScalarPair(ref mut scalar, _) => {
- // the asserts ensure that we are not using the
- // `#[rustc_layout_scalar_valid_range(n)]`
- // attribute to widen the range of anything as that would probably
- // result in UB somewhere
- // FIXME(eddyb) the asserts are probably not needed,
- // as larger validity ranges would result in missed
- // optimizations, *not* wrongly assuming the inner
- // value is valid. e.g. unions enlarge validity ranges,
- // because the values may be uninitialized.
- if let Bound::Included(start) = start {
- // FIXME(eddyb) this might be incorrect - it doesn't
- // account for wrap-around (end < start) ranges.
- let valid_range = scalar.valid_range_mut();
- assert!(valid_range.start <= start);
- valid_range.start = start;
- }
- if let Bound::Included(end) = end {
- // FIXME(eddyb) this might be incorrect - it doesn't
- // account for wrap-around (end < start) ranges.
- let valid_range = scalar.valid_range_mut();
- assert!(valid_range.end >= end);
- valid_range.end = end;
- }
-
- // Update `largest_niche` if we have introduced a larger niche.
- let niche = Niche::from_scalar(dl, Size::ZERO, *scalar);
- if let Some(niche) = niche {
- match st.largest_niche {
- Some(largest_niche) => {
- // Replace the existing niche even if they're equal,
- // because this one is at a lower offset.
- if largest_niche.available(dl) <= niche.available(dl) {
- st.largest_niche = Some(niche);
- }
- }
- None => st.largest_niche = Some(niche),
- }
- }
- }
- _ => assert!(
- start == Bound::Unbounded && end == Bound::Unbounded,
- "nonscalar layout for layout_scalar_valid_range type {:?}: {:#?}",
- def,
- st,
- ),
- }
-
- return Ok(tcx.intern_layout(st));
- }
-
- // At this point, we have handled all unions and
- // structs. (We have also handled univariant enums
- // that allow representation optimization.)
- assert!(def.is_enum());
-
- // Until we've decided whether to use the tagged or
- // niche filling LayoutS, we don't want to intern the
- // variant layouts, so we can't store them in the
- // overall LayoutS. Store the overall LayoutS
- // and the variant LayoutSs here until then.
- struct TmpLayout<'tcx> {
- layout: LayoutS<'tcx>,
- variants: IndexVec<VariantIdx, LayoutS<'tcx>>,
- }
-
- let calculate_niche_filling_layout =
- || -> Result<Option<TmpLayout<'tcx>>, LayoutError<'tcx>> {
- // The current code for niche-filling relies on variant indices
- // instead of actual discriminants, so enums with
- // explicit discriminants (RFC #2363) would misbehave.
- if def.repr().inhibit_enum_layout_opt()
- || def
- .variants()
- .iter_enumerated()
- .any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32()))
- {
- return Ok(None);
- }
-
- if variants.len() < 2 {
- return Ok(None);
- }
-
- let mut align = dl.aggregate_align;
- let mut variant_layouts = variants
- .iter_enumerated()
- .map(|(j, v)| {
- let mut st = self.univariant_uninterned(
- ty,
- v,
- &def.repr(),
- StructKind::AlwaysSized,
- )?;
- st.variants = Variants::Single { index: j };
-
- align = align.max(st.align);
-
- Ok(st)
- })
- .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
-
- let largest_variant_index = match variant_layouts
- .iter_enumerated()
- .max_by_key(|(_i, layout)| layout.size.bytes())
- .map(|(i, _layout)| i)
- {
- None => return Ok(None),
- Some(i) => i,
- };
-
- let all_indices = VariantIdx::new(0)..=VariantIdx::new(variants.len() - 1);
- let needs_disc = |index: VariantIdx| {
- index != largest_variant_index && !absent(&variants[index])
- };
- let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap()
- ..=all_indices.rev().find(|v| needs_disc(*v)).unwrap();
-
- let count = niche_variants.size_hint().1.unwrap() as u128;
-
- // Find the field with the largest niche
- let (field_index, niche, (niche_start, niche_scalar)) = match variants
- [largest_variant_index]
- .iter()
- .enumerate()
- .filter_map(|(j, field)| Some((j, field.largest_niche?)))
- .max_by_key(|(_, niche)| niche.available(dl))
- .and_then(|(j, niche)| Some((j, niche, niche.reserve(self, count)?)))
- {
- None => return Ok(None),
- Some(x) => x,
- };
-
- let niche_offset = niche.offset
- + variant_layouts[largest_variant_index].fields.offset(field_index);
- let niche_size = niche.value.size(dl);
- let size = variant_layouts[largest_variant_index].size.align_to(align.abi);
-
- let all_variants_fit =
- variant_layouts.iter_enumerated_mut().all(|(i, layout)| {
- if i == largest_variant_index {
- return true;
- }
-
- layout.largest_niche = None;
-
- if layout.size <= niche_offset {
- // This variant will fit before the niche.
- return true;
- }
-
- // Determine if it'll fit after the niche.
- let this_align = layout.align.abi;
- let this_offset = (niche_offset + niche_size).align_to(this_align);
-
- if this_offset + layout.size > size {
- return false;
- }
-
- // It'll fit, but we need to make some adjustments.
- match layout.fields {
- FieldsShape::Arbitrary { ref mut offsets, .. } => {
- for (j, offset) in offsets.iter_mut().enumerate() {
- if !variants[i][j].is_zst() {
- *offset += this_offset;
- }
- }
- }
- _ => {
- panic!("Layout of fields should be Arbitrary for variants")
- }
- }
-
- // It can't be a Scalar or ScalarPair because the offset isn't 0.
- if !layout.abi.is_uninhabited() {
- layout.abi = Abi::Aggregate { sized: true };
- }
- layout.size += this_offset;
-
- true
- });
-
- if !all_variants_fit {
- return Ok(None);
- }
-
- let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar);
-
- let others_zst = variant_layouts.iter_enumerated().all(|(i, layout)| {
- i == largest_variant_index || layout.size == Size::ZERO
- });
- let same_size = size == variant_layouts[largest_variant_index].size;
- let same_align = align == variant_layouts[largest_variant_index].align;
-
- let abi = if variant_layouts.iter().all(|v| v.abi.is_uninhabited()) {
- Abi::Uninhabited
- } else if same_size && same_align && others_zst {
- match variant_layouts[largest_variant_index].abi {
- // When the total alignment and size match, we can use the
- // same ABI as the scalar variant with the reserved niche.
- Abi::Scalar(_) => Abi::Scalar(niche_scalar),
- Abi::ScalarPair(first, second) => {
- // Only the niche is guaranteed to be initialised,
- // so use union layouts for the other primitive.
- if niche_offset == Size::ZERO {
- Abi::ScalarPair(niche_scalar, second.to_union())
- } else {
- Abi::ScalarPair(first.to_union(), niche_scalar)
- }
- }
- _ => Abi::Aggregate { sized: true },
- }
- } else {
- Abi::Aggregate { sized: true }
- };
-
- let layout = LayoutS {
- variants: Variants::Multiple {
- tag: niche_scalar,
- tag_encoding: TagEncoding::Niche {
- untagged_variant: largest_variant_index,
- niche_variants,
- niche_start,
- },
- tag_field: 0,
- variants: IndexVec::new(),
- },
- fields: FieldsShape::Arbitrary {
- offsets: vec![niche_offset],
- memory_index: vec![0],
- },
- abi,
- largest_niche,
- size,
- align,
- };
-
- Ok(Some(TmpLayout { layout, variants: variant_layouts }))
- };
-
- let niche_filling_layout = calculate_niche_filling_layout()?;
-
- let (mut min, mut max) = (i128::MAX, i128::MIN);
- let discr_type = def.repr().discr_type();
- let bits = Integer::from_attr(self, discr_type).size().bits();
- for (i, discr) in def.discriminants(tcx) {
- if variants[i].iter().any(|f| f.abi.is_uninhabited()) {
- continue;
- }
- let mut x = discr.val as i128;
- if discr_type.is_signed() {
- // sign extend the raw representation to be an i128
- x = (x << (128 - bits)) >> (128 - bits);
- }
- if x < min {
- min = x;
- }
- if x > max {
- max = x;
- }
- }
- // We might have no inhabited variants, so pretend there's at least one.
- if (min, max) == (i128::MAX, i128::MIN) {
- min = 0;
- max = 0;
- }
- assert!(min <= max, "discriminant range is {}...{}", min, max);
- let (min_ity, signed) = Integer::repr_discr(tcx, ty, &def.repr(), min, max);
-
- let mut align = dl.aggregate_align;
- let mut size = Size::ZERO;
-
- // We're interested in the smallest alignment, so start large.
- let mut start_align = Align::from_bytes(256).unwrap();
- assert_eq!(Integer::for_align(dl, start_align), None);
-
- // repr(C) on an enum tells us to make a (tag, union) layout,
- // so we need to grow the prefix alignment to be at least
- // the alignment of the union. (This value is used both for
- // determining the alignment of the overall enum, and the
- // determining the alignment of the payload after the tag.)
- let mut prefix_align = min_ity.align(dl).abi;
- if def.repr().c() {
- for fields in &variants {
- for field in fields {
- prefix_align = prefix_align.max(field.align.abi);
- }
- }
- }
-
- // Create the set of structs that represent each variant.
- let mut layout_variants = variants
- .iter_enumerated()
- .map(|(i, field_layouts)| {
- let mut st = self.univariant_uninterned(
- ty,
- &field_layouts,
- &def.repr(),
- StructKind::Prefixed(min_ity.size(), prefix_align),
- )?;
- st.variants = Variants::Single { index: i };
- // Find the first field we can't move later
- // to make room for a larger discriminant.
- for field in
- st.fields.index_by_increasing_offset().map(|j| field_layouts[j])
- {
- if !field.is_zst() || field.align.abi.bytes() != 1 {
- start_align = start_align.min(field.align.abi);
- break;
- }
- }
- size = cmp::max(size, st.size);
- align = align.max(st.align);
- Ok(st)
- })
- .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
-
- // Align the maximum variant size to the largest alignment.
- size = size.align_to(align.abi);
-
- if size.bytes() >= dl.obj_size_bound() {
- return Err(LayoutError::SizeOverflow(ty));
- }
-
- let typeck_ity = Integer::from_attr(dl, def.repr().discr_type());
- if typeck_ity < min_ity {
- // It is a bug if Layout decided on a greater discriminant size than typeck for
- // some reason at this point (based on values discriminant can take on). Mostly
- // because this discriminant will be loaded, and then stored into variable of
- // type calculated by typeck. Consider such case (a bug): typeck decided on
- // byte-sized discriminant, but layout thinks we need a 16-bit to store all
- // discriminant values. That would be a bug, because then, in codegen, in order
- // to store this 16-bit discriminant into 8-bit sized temporary some of the
- // space necessary to represent would have to be discarded (or layout is wrong
- // on thinking it needs 16 bits)
- bug!(
- "layout decided on a larger discriminant type ({:?}) than typeck ({:?})",
- min_ity,
- typeck_ity
- );
- // However, it is fine to make discr type however large (as an optimisation)
- // after this point – we’ll just truncate the value we load in codegen.
- }
-
- // Check to see if we should use a different type for the
- // discriminant. We can safely use a type with the same size
- // as the alignment of the first field of each variant.
- // We increase the size of the discriminant to avoid LLVM copying
- // padding when it doesn't need to. This normally causes unaligned
- // load/stores and excessive memcpy/memset operations. By using a
- // bigger integer size, LLVM can be sure about its contents and
- // won't be so conservative.
-
- // Use the initial field alignment
- let mut ity = if def.repr().c() || def.repr().int.is_some() {
- min_ity
- } else {
- Integer::for_align(dl, start_align).unwrap_or(min_ity)
- };
-
- // If the alignment is not larger than the chosen discriminant size,
- // don't use the alignment as the final size.
- if ity <= min_ity {
- ity = min_ity;
- } else {
- // Patch up the variants' first few fields.
- let old_ity_size = min_ity.size();
- let new_ity_size = ity.size();
- for variant in &mut layout_variants {
- match variant.fields {
- FieldsShape::Arbitrary { ref mut offsets, .. } => {
- for i in offsets {
- if *i <= old_ity_size {
- assert_eq!(*i, old_ity_size);
- *i = new_ity_size;
- }
- }
- // We might be making the struct larger.
- if variant.size <= old_ity_size {
- variant.size = new_ity_size;
- }
- }
- _ => bug!(),
- }
- }
- }
-
- let tag_mask = ity.size().unsigned_int_max();
- let tag = Scalar::Initialized {
- value: Int(ity, signed),
- valid_range: WrappingRange {
- start: (min as u128 & tag_mask),
- end: (max as u128 & tag_mask),
- },
- };
- let mut abi = Abi::Aggregate { sized: true };
-
- if layout_variants.iter().all(|v| v.abi.is_uninhabited()) {
- abi = Abi::Uninhabited;
- } else if tag.size(dl) == size {
- // Make sure we only use scalar layout when the enum is entirely its
- // own tag (i.e. it has no padding nor any non-ZST variant fields).
- abi = Abi::Scalar(tag);
- } else {
- // Try to use a ScalarPair for all tagged enums.
- let mut common_prim = None;
- let mut common_prim_initialized_in_all_variants = true;
- for (field_layouts, layout_variant) in iter::zip(&variants, &layout_variants) {
- let FieldsShape::Arbitrary { ref offsets, .. } = layout_variant.fields else {
- bug!();
- };
- let mut fields =
- iter::zip(field_layouts, offsets).filter(|p| !p.0.is_zst());
- let (field, offset) = match (fields.next(), fields.next()) {
- (None, None) => {
- common_prim_initialized_in_all_variants = false;
- continue;
- }
- (Some(pair), None) => pair,
- _ => {
- common_prim = None;
- break;
- }
- };
- let prim = match field.abi {
- Abi::Scalar(scalar) => {
- common_prim_initialized_in_all_variants &=
- matches!(scalar, Scalar::Initialized { .. });
- scalar.primitive()
- }
- _ => {
- common_prim = None;
- break;
- }
- };
- if let Some(pair) = common_prim {
- // This is pretty conservative. We could go fancier
- // by conflating things like i32 and u32, or even
- // realising that (u8, u8) could just cohabit with
- // u16 or even u32.
- if pair != (prim, offset) {
- common_prim = None;
- break;
- }
- } else {
- common_prim = Some((prim, offset));
- }
- }
- if let Some((prim, offset)) = common_prim {
- let prim_scalar = if common_prim_initialized_in_all_variants {
- scalar_unit(prim)
- } else {
- // Common prim might be uninit.
- Scalar::Union { value: prim }
- };
- let pair = self.scalar_pair(tag, prim_scalar);
- let pair_offsets = match pair.fields {
- FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
- assert_eq!(memory_index, &[0, 1]);
- offsets
- }
- _ => bug!(),
- };
- if pair_offsets[0] == Size::ZERO
- && pair_offsets[1] == *offset
- && align == pair.align
- && size == pair.size
- {
- // We can use `ScalarPair` only when it matches our
- // already computed layout (including `#[repr(C)]`).
- abi = pair.abi;
- }
- }
- }
-
- // If we pick a "clever" (by-value) ABI, we might have to adjust the ABI of the
- // variants to ensure they are consistent. This is because a downcast is
- // semantically a NOP, and thus should not affect layout.
- if matches!(abi, Abi::Scalar(..) | Abi::ScalarPair(..)) {
- for variant in &mut layout_variants {
- // We only do this for variants with fields; the others are not accessed anyway.
- // Also do not overwrite any already existing "clever" ABIs.
- if variant.fields.count() > 0
- && matches!(variant.abi, Abi::Aggregate { .. })
- {
- variant.abi = abi;
- // Also need to bump up the size and alignment, so that the entire value fits in here.
- variant.size = cmp::max(variant.size, size);
- variant.align.abi = cmp::max(variant.align.abi, align.abi);
- }
- }
- }
-
- let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag);
-
- let tagged_layout = LayoutS {
- variants: Variants::Multiple {
- tag,
- tag_encoding: TagEncoding::Direct,
- tag_field: 0,
- variants: IndexVec::new(),
- },
- fields: FieldsShape::Arbitrary {
- offsets: vec![Size::ZERO],
- memory_index: vec![0],
- },
- largest_niche,
- abi,
- align,
- size,
- };
-
- let tagged_layout = TmpLayout { layout: tagged_layout, variants: layout_variants };
-
- let mut best_layout = match (tagged_layout, niche_filling_layout) {
- (tl, Some(nl)) => {
- // Pick the smaller layout; otherwise,
- // pick the layout with the larger niche; otherwise,
- // pick tagged as it has simpler codegen.
- use Ordering::*;
- let niche_size = |tmp_l: &TmpLayout<'_>| {
- tmp_l.layout.largest_niche.map_or(0, |n| n.available(dl))
- };
- match (
- tl.layout.size.cmp(&nl.layout.size),
- niche_size(&tl).cmp(&niche_size(&nl)),
- ) {
- (Greater, _) => nl,
- (Equal, Less) => nl,
- _ => tl,
- }
- }
- (tl, None) => tl,
- };
-
- // Now we can intern the variant layouts and store them in the enum layout.
- best_layout.layout.variants = match best_layout.layout.variants {
- Variants::Multiple { tag, tag_encoding, tag_field, .. } => Variants::Multiple {
- tag,
- tag_encoding,
- tag_field,
- variants: best_layout
- .variants
- .into_iter()
- .map(|layout| tcx.intern_layout(layout))
- .collect(),
- },
- _ => bug!(),
- };
-
- tcx.intern_layout(best_layout.layout)
- }
-
- // Types with no meaningful known layout.
- ty::Projection(_) | ty::Opaque(..) => {
- // NOTE(eddyb) `layout_of` query should've normalized these away,
- // if that was possible, so there's no reason to try again here.
- return Err(LayoutError::Unknown(ty));
- }
-
- ty::Placeholder(..) | ty::GeneratorWitness(..) | ty::Infer(_) => {
- bug!("Layout::compute: unexpected type `{}`", ty)
- }
-
- ty::Bound(..) | ty::Param(_) | ty::Error(_) => {
- return Err(LayoutError::Unknown(ty));
- }
- })
- }
-}
-
-/// Overlap eligibility and variant assignment for each GeneratorSavedLocal.
-#[derive(Clone, Debug, PartialEq)]
-enum SavedLocalEligibility {
- Unassigned,
- Assigned(VariantIdx),
- // FIXME: Use newtype_index so we aren't wasting bytes
- Ineligible(Option<u32>),
-}
-
-// When laying out generators, we divide our saved local fields into two
-// categories: overlap-eligible and overlap-ineligible.
-//
-// Those fields which are ineligible for overlap go in a "prefix" at the
-// beginning of the layout, and always have space reserved for them.
-//
-// Overlap-eligible fields are only assigned to one variant, so we lay
-// those fields out for each variant and put them right after the
-// prefix.
-//
-// Finally, in the layout details, we point to the fields from the
-// variants they are assigned to. It is possible for some fields to be
-// included in multiple variants. No field ever "moves around" in the
-// layout; its offset is always the same.
-//
-// Also included in the layout are the upvars and the discriminant.
-// These are included as fields on the "outer" layout; they are not part
-// of any variant.
-impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
- /// Compute the eligibility and assignment of each local.
- fn generator_saved_local_eligibility(
- &self,
- info: &GeneratorLayout<'tcx>,
- ) -> (BitSet<GeneratorSavedLocal>, IndexVec<GeneratorSavedLocal, SavedLocalEligibility>) {
- use SavedLocalEligibility::*;
-
- let mut assignments: IndexVec<GeneratorSavedLocal, SavedLocalEligibility> =
- IndexVec::from_elem_n(Unassigned, info.field_tys.len());
-
- // The saved locals not eligible for overlap. These will get
- // "promoted" to the prefix of our generator.
- let mut ineligible_locals = BitSet::new_empty(info.field_tys.len());
-
- // Figure out which of our saved locals are fields in only
- // one variant. The rest are deemed ineligible for overlap.
- for (variant_index, fields) in info.variant_fields.iter_enumerated() {
- for local in fields {
- match assignments[*local] {
- Unassigned => {
- assignments[*local] = Assigned(variant_index);
- }
- Assigned(idx) => {
- // We've already seen this local at another suspension
- // point, so it is no longer a candidate.
- trace!(
- "removing local {:?} in >1 variant ({:?}, {:?})",
- local,
- variant_index,
- idx
- );
- ineligible_locals.insert(*local);
- assignments[*local] = Ineligible(None);
- }
- Ineligible(_) => {}
- }
- }
- }
-
- // Next, check every pair of eligible locals to see if they
- // conflict.
- for local_a in info.storage_conflicts.rows() {
- let conflicts_a = info.storage_conflicts.count(local_a);
- if ineligible_locals.contains(local_a) {
- continue;
- }
-
- for local_b in info.storage_conflicts.iter(local_a) {
- // local_a and local_b are storage live at the same time, therefore they
- // cannot overlap in the generator layout. The only way to guarantee
- // this is if they are in the same variant, or one is ineligible
- // (which means it is stored in every variant).
- if ineligible_locals.contains(local_b)
- || assignments[local_a] == assignments[local_b]
- {
- continue;
- }
-
- // If they conflict, we will choose one to make ineligible.
- // This is not always optimal; it's just a greedy heuristic that
- // seems to produce good results most of the time.
- let conflicts_b = info.storage_conflicts.count(local_b);
- let (remove, other) =
- if conflicts_a > conflicts_b { (local_a, local_b) } else { (local_b, local_a) };
- ineligible_locals.insert(remove);
- assignments[remove] = Ineligible(None);
- trace!("removing local {:?} due to conflict with {:?}", remove, other);
- }
- }
-
- // Count the number of variants in use. If only one of them, then it is
- // impossible to overlap any locals in our layout. In this case it's
- // always better to make the remaining locals ineligible, so we can
- // lay them out with the other locals in the prefix and eliminate
- // unnecessary padding bytes.
- {
- let mut used_variants = BitSet::new_empty(info.variant_fields.len());
- for assignment in &assignments {
- if let Assigned(idx) = assignment {
- used_variants.insert(*idx);
- }
- }
- if used_variants.count() < 2 {
- for assignment in assignments.iter_mut() {
- *assignment = Ineligible(None);
- }
- ineligible_locals.insert_all();
- }
- }
-
- // Write down the order of our locals that will be promoted to the prefix.
- {
- for (idx, local) in ineligible_locals.iter().enumerate() {
- assignments[local] = Ineligible(Some(idx as u32));
- }
- }
- debug!("generator saved local assignments: {:?}", assignments);
-
- (ineligible_locals, assignments)
- }
-
- /// Compute the full generator layout.
- fn generator_layout(
- &self,
- ty: Ty<'tcx>,
- def_id: hir::def_id::DefId,
- substs: SubstsRef<'tcx>,
- ) -> Result<Layout<'tcx>, LayoutError<'tcx>> {
- use SavedLocalEligibility::*;
- let tcx = self.tcx;
- let subst_field = |ty: Ty<'tcx>| EarlyBinder(ty).subst(tcx, substs);
-
- let Some(info) = tcx.generator_layout(def_id) else {
- return Err(LayoutError::Unknown(ty));
- };
- let (ineligible_locals, assignments) = self.generator_saved_local_eligibility(&info);
-
- // Build a prefix layout, including "promoting" all ineligible
- // locals as part of the prefix. We compute the layout of all of
- // these fields at once to get optimal packing.
- let tag_index = substs.as_generator().prefix_tys().count();
-
- // `info.variant_fields` already accounts for the reserved variants, so no need to add them.
- let max_discr = (info.variant_fields.len() - 1) as u128;
- let discr_int = Integer::fit_unsigned(max_discr);
- let discr_int_ty = discr_int.to_ty(tcx, false);
- let tag = Scalar::Initialized {
- value: Primitive::Int(discr_int, false),
- valid_range: WrappingRange { start: 0, end: max_discr },
- };
- let tag_layout = self.tcx.intern_layout(LayoutS::scalar(self, tag));
- let tag_layout = TyAndLayout { ty: discr_int_ty, layout: tag_layout };
-
- let promoted_layouts = ineligible_locals
- .iter()
- .map(|local| subst_field(info.field_tys[local]))
- .map(|ty| tcx.mk_maybe_uninit(ty))
- .map(|ty| self.layout_of(ty));
- let prefix_layouts = substs
- .as_generator()
- .prefix_tys()
- .map(|ty| self.layout_of(ty))
- .chain(iter::once(Ok(tag_layout)))
- .chain(promoted_layouts)
- .collect::<Result<Vec<_>, _>>()?;
- let prefix = self.univariant_uninterned(
- ty,
- &prefix_layouts,
- &ReprOptions::default(),
- StructKind::AlwaysSized,
- )?;
-
- let (prefix_size, prefix_align) = (prefix.size, prefix.align);
-
- // Split the prefix layout into the "outer" fields (upvars and
- // discriminant) and the "promoted" fields. Promoted fields will
- // get included in each variant that requested them in
- // GeneratorLayout.
- debug!("prefix = {:#?}", prefix);
- let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields {
- FieldsShape::Arbitrary { mut offsets, memory_index } => {
- let mut inverse_memory_index = invert_mapping(&memory_index);
-
- // "a" (`0..b_start`) and "b" (`b_start..`) correspond to
- // "outer" and "promoted" fields respectively.
- let b_start = (tag_index + 1) as u32;
- let offsets_b = offsets.split_off(b_start as usize);
- let offsets_a = offsets;
-
- // Disentangle the "a" and "b" components of `inverse_memory_index`
- // by preserving the order but keeping only one disjoint "half" each.
- // FIXME(eddyb) build a better abstraction for permutations, if possible.
- let inverse_memory_index_b: Vec<_> =
- inverse_memory_index.iter().filter_map(|&i| i.checked_sub(b_start)).collect();
- inverse_memory_index.retain(|&i| i < b_start);
- let inverse_memory_index_a = inverse_memory_index;
-
- // Since `inverse_memory_index_{a,b}` each only refer to their
- // respective fields, they can be safely inverted
- let memory_index_a = invert_mapping(&inverse_memory_index_a);
- let memory_index_b = invert_mapping(&inverse_memory_index_b);
-
- let outer_fields =
- FieldsShape::Arbitrary { offsets: offsets_a, memory_index: memory_index_a };
- (outer_fields, offsets_b, memory_index_b)
- }
- _ => bug!(),
- };
-
- let mut size = prefix.size;
- let mut align = prefix.align;
- let variants = info
- .variant_fields
- .iter_enumerated()
- .map(|(index, variant_fields)| {
- // Only include overlap-eligible fields when we compute our variant layout.
- let variant_only_tys = variant_fields
- .iter()
- .filter(|local| match assignments[**local] {
- Unassigned => bug!(),
- Assigned(v) if v == index => true,
- Assigned(_) => bug!("assignment does not match variant"),
- Ineligible(_) => false,
- })
- .map(|local| subst_field(info.field_tys[*local]));
-
- let mut variant = self.univariant_uninterned(
- ty,
- &variant_only_tys
- .map(|ty| self.layout_of(ty))
- .collect::<Result<Vec<_>, _>>()?,
- &ReprOptions::default(),
- StructKind::Prefixed(prefix_size, prefix_align.abi),
- )?;
- variant.variants = Variants::Single { index };
-
- let FieldsShape::Arbitrary { offsets, memory_index } = variant.fields else {
- bug!();
- };
-
- // Now, stitch the promoted and variant-only fields back together in
- // the order they are mentioned by our GeneratorLayout.
- // Because we only use some subset (that can differ between variants)
- // of the promoted fields, we can't just pick those elements of the
- // `promoted_memory_index` (as we'd end up with gaps).
- // So instead, we build an "inverse memory_index", as if all of the
- // promoted fields were being used, but leave the elements not in the
- // subset as `INVALID_FIELD_IDX`, which we can filter out later to
- // obtain a valid (bijective) mapping.
- const INVALID_FIELD_IDX: u32 = !0;
- let mut combined_inverse_memory_index =
- vec![INVALID_FIELD_IDX; promoted_memory_index.len() + memory_index.len()];
- let mut offsets_and_memory_index = iter::zip(offsets, memory_index);
- let combined_offsets = variant_fields
- .iter()
- .enumerate()
- .map(|(i, local)| {
- let (offset, memory_index) = match assignments[*local] {
- Unassigned => bug!(),
- Assigned(_) => {
- let (offset, memory_index) =
- offsets_and_memory_index.next().unwrap();
- (offset, promoted_memory_index.len() as u32 + memory_index)
- }
- Ineligible(field_idx) => {
- let field_idx = field_idx.unwrap() as usize;
- (promoted_offsets[field_idx], promoted_memory_index[field_idx])
- }
- };
- combined_inverse_memory_index[memory_index as usize] = i as u32;
- offset
- })
- .collect();
-
- // Remove the unused slots and invert the mapping to obtain the
- // combined `memory_index` (also see previous comment).
- combined_inverse_memory_index.retain(|&i| i != INVALID_FIELD_IDX);
- let combined_memory_index = invert_mapping(&combined_inverse_memory_index);
-
- variant.fields = FieldsShape::Arbitrary {
- offsets: combined_offsets,
- memory_index: combined_memory_index,
- };
-
- size = size.max(variant.size);
- align = align.max(variant.align);
- Ok(tcx.intern_layout(variant))
- })
- .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
-
- size = size.align_to(align.abi);
-
- let abi =
- if prefix.abi.is_uninhabited() || variants.iter().all(|v| v.abi().is_uninhabited()) {
- Abi::Uninhabited
- } else {
- Abi::Aggregate { sized: true }
- };
-
- let layout = tcx.intern_layout(LayoutS {
- variants: Variants::Multiple {
- tag,
- tag_encoding: TagEncoding::Direct,
- tag_field: tag_index,
- variants,
- },
- fields: outer_fields,
- abi,
- largest_niche: prefix.largest_niche,
- size,
- align,
- });
- debug!("generator layout ({:?}): {:#?}", ty, layout);
- Ok(layout)
- }
-
- /// This is invoked by the `layout_of` query to record the final
- /// layout of each type.
- #[inline(always)]
- fn record_layout_for_printing(&self, layout: TyAndLayout<'tcx>) {
- // If we are running with `-Zprint-type-sizes`, maybe record layouts
- // for dumping later.
- if self.tcx.sess.opts.unstable_opts.print_type_sizes {
- self.record_layout_for_printing_outlined(layout)
- }
- }
-
- fn record_layout_for_printing_outlined(&self, layout: TyAndLayout<'tcx>) {
- // Ignore layouts that are done with non-empty environments or
- // non-monomorphic layouts, as the user only wants to see the stuff
- // resulting from the final codegen session.
- if layout.ty.has_param_types_or_consts() || !self.param_env.caller_bounds().is_empty() {
- return;
- }
-
- // (delay format until we actually need it)
- let record = |kind, packed, opt_discr_size, variants| {
- let type_desc = format!("{:?}", layout.ty);
- self.tcx.sess.code_stats.record_type_size(
- kind,
- type_desc,
- layout.align.abi,
- layout.size,
- packed,
- opt_discr_size,
- variants,
- );
- };
-
- let adt_def = match *layout.ty.kind() {
- ty::Adt(ref adt_def, _) => {
- debug!("print-type-size t: `{:?}` process adt", layout.ty);
- adt_def
- }
-
- ty::Closure(..) => {
- debug!("print-type-size t: `{:?}` record closure", layout.ty);
- record(DataTypeKind::Closure, false, None, vec![]);
- return;
- }
-
- _ => {
- debug!("print-type-size t: `{:?}` skip non-nominal", layout.ty);
- return;
- }
- };
-
- let adt_kind = adt_def.adt_kind();
- let adt_packed = adt_def.repr().pack.is_some();
-
- let build_variant_info = |n: Option<Symbol>, flds: &[Symbol], layout: TyAndLayout<'tcx>| {
- let mut min_size = Size::ZERO;
- let field_info: Vec<_> = flds
- .iter()
- .enumerate()
- .map(|(i, &name)| {
- let field_layout = layout.field(self, i);
- let offset = layout.fields.offset(i);
- let field_end = offset + field_layout.size;
- if min_size < field_end {
- min_size = field_end;
- }
- FieldInfo {
- name,
- offset: offset.bytes(),
- size: field_layout.size.bytes(),
- align: field_layout.align.abi.bytes(),
- }
- })
- .collect();
-
- VariantInfo {
- name: n,
- kind: if layout.is_unsized() { SizeKind::Min } else { SizeKind::Exact },
- align: layout.align.abi.bytes(),
- size: if min_size.bytes() == 0 { layout.size.bytes() } else { min_size.bytes() },
- fields: field_info,
- }
- };
-
- match layout.variants {
- Variants::Single { index } => {
- if !adt_def.variants().is_empty() && layout.fields != FieldsShape::Primitive {
- debug!(
- "print-type-size `{:#?}` variant {}",
- layout,
- adt_def.variant(index).name
- );
- let variant_def = &adt_def.variant(index);
- let fields: Vec<_> = variant_def.fields.iter().map(|f| f.name).collect();
- record(
- adt_kind.into(),
- adt_packed,
- None,
- vec![build_variant_info(Some(variant_def.name), &fields, layout)],
- );
- } else {
- // (This case arises for *empty* enums; so give it
- // zero variants.)
- record(adt_kind.into(), adt_packed, None, vec![]);
- }
- }
-
- Variants::Multiple { tag, ref tag_encoding, .. } => {
- debug!(
- "print-type-size `{:#?}` adt general variants def {}",
- layout.ty,
- adt_def.variants().len()
- );
- let variant_infos: Vec<_> = adt_def
- .variants()
- .iter_enumerated()
- .map(|(i, variant_def)| {
- let fields: Vec<_> = variant_def.fields.iter().map(|f| f.name).collect();
- build_variant_info(
- Some(variant_def.name),
- &fields,
- layout.for_variant(self, i),
- )
- })
- .collect();
- record(
- adt_kind.into(),
- adt_packed,
- match tag_encoding {
- TagEncoding::Direct => Some(tag.size(self)),
- _ => None,
- },
- variant_infos,
- );
- }
- }
- }
-}
-
/// Type size "skeleton", i.e., the only information determining a type's size.
/// While this is conservative, (aside from constant sizes, only pointers,
/// newtypes thereof and null pointer optimized enums are allowed), it is
tcx: TyCtxt<'tcx>,
param_env: ty::ParamEnv<'tcx>,
) -> Result<SizeSkeleton<'tcx>, LayoutError<'tcx>> {
- debug_assert!(!ty.has_infer_types_or_consts());
+ debug_assert!(!ty.has_non_region_infer());
// First try computing a static layout.
let err = match tcx.layout_of(param_env.and(ty)) {
let tail = tcx.struct_tail_erasing_lifetimes(pointee, param_env);
match tail.kind() {
ty::Param(_) | ty::Projection(_) => {
- debug_assert!(tail.has_param_types_or_consts());
+ debug_assert!(tail.has_non_region_param());
Ok(SizeSkeleton::Pointer { non_zero, tail: tcx.erase_regions(tail) })
}
_ => bug!(
}
}
-impl<'tcx> ty::Instance<'tcx> {
- // NOTE(eddyb) this is private to avoid using it from outside of
- // `fn_abi_of_instance` - any other uses are either too high-level
- // for `Instance` (e.g. typeck would use `Ty::fn_sig` instead),
- // or should go through `FnAbi` instead, to avoid losing any
- // adjustments `fn_abi_of_instance` might be performing.
- #[tracing::instrument(level = "debug", skip(tcx, param_env))]
- fn fn_sig_for_fn_abi(
- &self,
- tcx: TyCtxt<'tcx>,
- param_env: ty::ParamEnv<'tcx>,
- ) -> ty::PolyFnSig<'tcx> {
- let ty = self.ty(tcx, param_env);
- match *ty.kind() {
- ty::FnDef(..) => {
- // HACK(davidtwco,eddyb): This is a workaround for polymorphization considering
- // parameters unused if they show up in the signature, but not in the `mir::Body`
- // (i.e. due to being inside a projection that got normalized, see
- // `src/test/ui/polymorphization/normalized_sig_types.rs`), and codegen not keeping
- // track of a polymorphization `ParamEnv` to allow normalizing later.
- //
- // We normalize the `fn_sig` again after substituting at a later point.
- let mut sig = match *ty.kind() {
- ty::FnDef(def_id, substs) => tcx
- .bound_fn_sig(def_id)
- .map_bound(|fn_sig| {
- tcx.normalize_erasing_regions(tcx.param_env(def_id), fn_sig)
- })
- .subst(tcx, substs),
- _ => unreachable!(),
- };
-
- if let ty::InstanceDef::VTableShim(..) = self.def {
- // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`.
- sig = sig.map_bound(|mut sig| {
- let mut inputs_and_output = sig.inputs_and_output.to_vec();
- inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]);
- sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
- sig
- });
- }
- sig
- }
- ty::Closure(def_id, substs) => {
- let sig = substs.as_closure().sig();
-
- let bound_vars = tcx.mk_bound_variable_kinds(
- sig.bound_vars()
- .iter()
- .chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))),
- );
- let br = ty::BoundRegion {
- var: ty::BoundVar::from_usize(bound_vars.len() - 1),
- kind: ty::BoundRegionKind::BrEnv,
- };
- let env_region = ty::ReLateBound(ty::INNERMOST, br);
- let env_ty = tcx.closure_env_ty(def_id, substs, env_region).unwrap();
-
- let sig = sig.skip_binder();
- ty::Binder::bind_with_vars(
- tcx.mk_fn_sig(
- iter::once(env_ty).chain(sig.inputs().iter().cloned()),
- sig.output(),
- sig.c_variadic,
- sig.unsafety,
- sig.abi,
- ),
- bound_vars,
- )
- }
- ty::Generator(_, substs, _) => {
- let sig = substs.as_generator().poly_sig();
-
- let bound_vars = tcx.mk_bound_variable_kinds(
- sig.bound_vars()
- .iter()
- .chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))),
- );
- let br = ty::BoundRegion {
- var: ty::BoundVar::from_usize(bound_vars.len() - 1),
- kind: ty::BoundRegionKind::BrEnv,
- };
- let env_region = ty::ReLateBound(ty::INNERMOST, br);
- let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty);
-
- let pin_did = tcx.require_lang_item(LangItem::Pin, None);
- let pin_adt_ref = tcx.adt_def(pin_did);
- let pin_substs = tcx.intern_substs(&[env_ty.into()]);
- let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs);
-
- let sig = sig.skip_binder();
- let state_did = tcx.require_lang_item(LangItem::GeneratorState, None);
- let state_adt_ref = tcx.adt_def(state_did);
- let state_substs = tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]);
- let ret_ty = tcx.mk_adt(state_adt_ref, state_substs);
- ty::Binder::bind_with_vars(
- tcx.mk_fn_sig(
- [env_ty, sig.resume_ty].iter(),
- &ret_ty,
- false,
- hir::Unsafety::Normal,
- rustc_target::spec::abi::Abi::Rust,
- ),
- bound_vars,
- )
- }
- _ => bug!("unexpected type {:?} in Instance::fn_sig", ty),
- }
- }
-}
-
/// Calculates whether a function's ABI can unwind or not.
///
/// This takes two primary parameters:
}
}
-#[inline]
-pub fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: SpecAbi) -> Conv {
- use rustc_target::spec::abi::Abi::*;
- match tcx.sess.target.adjust_abi(abi) {
- RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust,
- RustCold => Conv::RustCold,
-
- // It's the ABI's job to select this, not ours.
- System { .. } => bug!("system abi should be selected elsewhere"),
- EfiApi => bug!("eficall abi should be selected elsewhere"),
-
- Stdcall { .. } => Conv::X86Stdcall,
- Fastcall { .. } => Conv::X86Fastcall,
- Vectorcall { .. } => Conv::X86VectorCall,
- Thiscall { .. } => Conv::X86ThisCall,
- C { .. } => Conv::C,
- Unadjusted => Conv::C,
- Win64 { .. } => Conv::X86_64Win64,
- SysV64 { .. } => Conv::X86_64SysV,
- Aapcs { .. } => Conv::ArmAapcs,
- CCmseNonSecureCall => Conv::CCmseNonSecureCall,
- PtxKernel => Conv::PtxKernel,
- Msp430Interrupt => Conv::Msp430Intr,
- X86Interrupt => Conv::X86Intr,
- AmdGpuKernel => Conv::AmdGpuKernel,
- AvrInterrupt => Conv::AvrInterrupt,
- AvrNonBlockingInterrupt => Conv::AvrNonBlockingInterrupt,
- Wasm => Conv::C,
-
- // These API constants ought to be more specific...
- Cdecl { .. } => Conv::C,
- }
-}
-
/// Error produced by attempting to compute or adjust a `FnAbi`.
#[derive(Copy, Clone, Debug, HashStable)]
pub enum FnAbiError<'tcx> {
}
impl<'tcx, C: FnAbiOfHelpers<'tcx>> FnAbiOf<'tcx> for C {}
-
-fn fn_abi_of_fn_ptr<'tcx>(
- tcx: TyCtxt<'tcx>,
- query: ty::ParamEnvAnd<'tcx, (ty::PolyFnSig<'tcx>, &'tcx ty::List<Ty<'tcx>>)>,
-) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> {
- let (param_env, (sig, extra_args)) = query.into_parts();
-
- LayoutCx { tcx, param_env }.fn_abi_new_uncached(sig, extra_args, None, None, false)
-}
-
-fn fn_abi_of_instance<'tcx>(
- tcx: TyCtxt<'tcx>,
- query: ty::ParamEnvAnd<'tcx, (ty::Instance<'tcx>, &'tcx ty::List<Ty<'tcx>>)>,
-) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> {
- let (param_env, (instance, extra_args)) = query.into_parts();
-
- let sig = instance.fn_sig_for_fn_abi(tcx, param_env);
-
- let caller_location = if instance.def.requires_caller_location(tcx) {
- Some(tcx.caller_location_ty())
- } else {
- None
- };
-
- LayoutCx { tcx, param_env }.fn_abi_new_uncached(
- sig,
- extra_args,
- caller_location,
- Some(instance.def_id()),
- matches!(instance.def, ty::InstanceDef::Virtual(..)),
- )
-}
-
-// Handle safe Rust thin and fat pointers.
-pub fn adjust_for_rust_scalar<'tcx>(
- cx: LayoutCx<'tcx, TyCtxt<'tcx>>,
- attrs: &mut ArgAttributes,
- scalar: Scalar,
- layout: TyAndLayout<'tcx>,
- offset: Size,
- is_return: bool,
-) {
- // Booleans are always a noundef i1 that needs to be zero-extended.
- if scalar.is_bool() {
- attrs.ext(ArgExtension::Zext);
- attrs.set(ArgAttribute::NoUndef);
- return;
- }
-
- // Scalars which have invalid values cannot be undef.
- if !scalar.is_always_valid(&cx) {
- attrs.set(ArgAttribute::NoUndef);
- }
-
- // Only pointer types handled below.
- let Scalar::Initialized { value: Pointer, valid_range} = scalar else { return };
-
- if !valid_range.contains(0) {
- attrs.set(ArgAttribute::NonNull);
- }
-
- if let Some(pointee) = layout.pointee_info_at(&cx, offset) {
- if let Some(kind) = pointee.safe {
- attrs.pointee_align = Some(pointee.align);
-
- // `Box` (`UniqueBorrowed`) are not necessarily dereferenceable
- // for the entire duration of the function as they can be deallocated
- // at any time. Same for shared mutable references. If LLVM had a
- // way to say "dereferenceable on entry" we could use it here.
- attrs.pointee_size = match kind {
- PointerKind::UniqueBorrowed
- | PointerKind::UniqueBorrowedPinned
- | PointerKind::Frozen => pointee.size,
- PointerKind::SharedMutable | PointerKind::UniqueOwned => Size::ZERO,
- };
-
- // `Box`, `&T`, and `&mut T` cannot be undef.
- // Note that this only applies to the value of the pointer itself;
- // this attribute doesn't make it UB for the pointed-to data to be undef.
- attrs.set(ArgAttribute::NoUndef);
-
- // The aliasing rules for `Box<T>` are still not decided, but currently we emit
- // `noalias` for it. This can be turned off using an unstable flag.
- // See https://github.com/rust-lang/unsafe-code-guidelines/issues/326
- let noalias_for_box = cx.tcx.sess.opts.unstable_opts.box_noalias.unwrap_or(true);
-
- // `&mut` pointer parameters never alias other parameters,
- // or mutable global data
- //
- // `&T` where `T` contains no `UnsafeCell<U>` is immutable,
- // and can be marked as both `readonly` and `noalias`, as
- // LLVM's definition of `noalias` is based solely on memory
- // dependencies rather than pointer equality
- //
- // Due to past miscompiles in LLVM, we apply a separate NoAliasMutRef attribute
- // for UniqueBorrowed arguments, so that the codegen backend can decide whether
- // or not to actually emit the attribute. It can also be controlled with the
- // `-Zmutable-noalias` debugging option.
- let no_alias = match kind {
- PointerKind::SharedMutable
- | PointerKind::UniqueBorrowed
- | PointerKind::UniqueBorrowedPinned => false,
- PointerKind::UniqueOwned => noalias_for_box,
- PointerKind::Frozen => !is_return,
- };
- if no_alias {
- attrs.set(ArgAttribute::NoAlias);
- }
-
- if kind == PointerKind::Frozen && !is_return {
- attrs.set(ArgAttribute::ReadOnly);
- }
-
- if kind == PointerKind::UniqueBorrowed && !is_return {
- attrs.set(ArgAttribute::NoAliasMutRef);
- }
- }
- }
-}
-
-impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
- // FIXME(eddyb) perhaps group the signature/type-containing (or all of them?)
- // arguments of this method, into a separate `struct`.
- #[tracing::instrument(
- level = "debug",
- skip(self, caller_location, fn_def_id, force_thin_self_ptr)
- )]
- fn fn_abi_new_uncached(
- &self,
- sig: ty::PolyFnSig<'tcx>,
- extra_args: &[Ty<'tcx>],
- caller_location: Option<Ty<'tcx>>,
- fn_def_id: Option<DefId>,
- // FIXME(eddyb) replace this with something typed, like an `enum`.
- force_thin_self_ptr: bool,
- ) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> {
- let sig = self.tcx.normalize_erasing_late_bound_regions(self.param_env, sig);
-
- let conv = conv_from_spec_abi(self.tcx(), sig.abi);
-
- let mut inputs = sig.inputs();
- let extra_args = if sig.abi == RustCall {
- assert!(!sig.c_variadic && extra_args.is_empty());
-
- if let Some(input) = sig.inputs().last() {
- if let ty::Tuple(tupled_arguments) = input.kind() {
- inputs = &sig.inputs()[0..sig.inputs().len() - 1];
- tupled_arguments
- } else {
- bug!(
- "argument to function with \"rust-call\" ABI \
- is not a tuple"
- );
- }
- } else {
- bug!(
- "argument to function with \"rust-call\" ABI \
- is not a tuple"
- );
- }
- } else {
- assert!(sig.c_variadic || extra_args.is_empty());
- extra_args
- };
-
- let target = &self.tcx.sess.target;
- let target_env_gnu_like = matches!(&target.env[..], "gnu" | "musl" | "uclibc");
- let win_x64_gnu = target.os == "windows" && target.arch == "x86_64" && target.env == "gnu";
- let linux_s390x_gnu_like =
- target.os == "linux" && target.arch == "s390x" && target_env_gnu_like;
- let linux_sparc64_gnu_like =
- target.os == "linux" && target.arch == "sparc64" && target_env_gnu_like;
- let linux_powerpc_gnu_like =
- target.os == "linux" && target.arch == "powerpc" && target_env_gnu_like;
- use SpecAbi::*;
- let rust_abi = matches!(sig.abi, RustIntrinsic | PlatformIntrinsic | Rust | RustCall);
-
- let arg_of = |ty: Ty<'tcx>, arg_idx: Option<usize>| -> Result<_, FnAbiError<'tcx>> {
- let span = tracing::debug_span!("arg_of");
- let _entered = span.enter();
- let is_return = arg_idx.is_none();
-
- let layout = self.layout_of(ty)?;
- let layout = if force_thin_self_ptr && arg_idx == Some(0) {
- // Don't pass the vtable, it's not an argument of the virtual fn.
- // Instead, pass just the data pointer, but give it the type `*const/mut dyn Trait`
- // or `&/&mut dyn Trait` because this is special-cased elsewhere in codegen
- make_thin_self_ptr(self, layout)
- } else {
- layout
- };
-
- let mut arg = ArgAbi::new(self, layout, |layout, scalar, offset| {
- let mut attrs = ArgAttributes::new();
- adjust_for_rust_scalar(*self, &mut attrs, scalar, *layout, offset, is_return);
- attrs
- });
-
- if arg.layout.is_zst() {
- // For some forsaken reason, x86_64-pc-windows-gnu
- // doesn't ignore zero-sized struct arguments.
- // The same is true for {s390x,sparc64,powerpc}-unknown-linux-{gnu,musl,uclibc}.
- if is_return
- || rust_abi
- || (!win_x64_gnu
- && !linux_s390x_gnu_like
- && !linux_sparc64_gnu_like
- && !linux_powerpc_gnu_like)
- {
- arg.mode = PassMode::Ignore;
- }
- }
-
- Ok(arg)
- };
-
- let mut fn_abi = FnAbi {
- ret: arg_of(sig.output(), None)?,
- args: inputs
- .iter()
- .copied()
- .chain(extra_args.iter().copied())
- .chain(caller_location)
- .enumerate()
- .map(|(i, ty)| arg_of(ty, Some(i)))
- .collect::<Result<_, _>>()?,
- c_variadic: sig.c_variadic,
- fixed_count: inputs.len() as u32,
- conv,
- can_unwind: fn_can_unwind(self.tcx(), fn_def_id, sig.abi),
- };
- self.fn_abi_adjust_for_abi(&mut fn_abi, sig.abi)?;
- debug!("fn_abi_new_uncached = {:?}", fn_abi);
- Ok(self.tcx.arena.alloc(fn_abi))
- }
-
- #[tracing::instrument(level = "trace", skip(self))]
- fn fn_abi_adjust_for_abi(
- &self,
- fn_abi: &mut FnAbi<'tcx, Ty<'tcx>>,
- abi: SpecAbi,
- ) -> Result<(), FnAbiError<'tcx>> {
- if abi == SpecAbi::Unadjusted {
- return Ok(());
- }
-
- if abi == SpecAbi::Rust
- || abi == SpecAbi::RustCall
- || abi == SpecAbi::RustIntrinsic
- || abi == SpecAbi::PlatformIntrinsic
- {
- let fixup = |arg: &mut ArgAbi<'tcx, Ty<'tcx>>| {
- if arg.is_ignore() {
- return;
- }
-
- match arg.layout.abi {
- Abi::Aggregate { .. } => {}
-
- // This is a fun case! The gist of what this is doing is
- // that we want callers and callees to always agree on the
- // ABI of how they pass SIMD arguments. If we were to *not*
- // make these arguments indirect then they'd be immediates
- // in LLVM, which means that they'd used whatever the
- // appropriate ABI is for the callee and the caller. That
- // means, for example, if the caller doesn't have AVX
- // enabled but the callee does, then passing an AVX argument
- // across this boundary would cause corrupt data to show up.
- //
- // This problem is fixed by unconditionally passing SIMD
- // arguments through memory between callers and callees
- // which should get them all to agree on ABI regardless of
- // target feature sets. Some more information about this
- // issue can be found in #44367.
- //
- // Note that the platform intrinsic ABI is exempt here as
- // that's how we connect up to LLVM and it's unstable
- // anyway, we control all calls to it in libstd.
- Abi::Vector { .. }
- if abi != SpecAbi::PlatformIntrinsic
- && self.tcx.sess.target.simd_types_indirect =>
- {
- arg.make_indirect();
- return;
- }
-
- _ => return,
- }
-
- let size = arg.layout.size;
- if arg.layout.is_unsized() || size > Pointer.size(self) {
- arg.make_indirect();
- } else {
- // We want to pass small aggregates as immediates, but using
- // a LLVM aggregate type for this leads to bad optimizations,
- // so we pick an appropriately sized integer type instead.
- arg.cast_to(Reg { kind: RegKind::Integer, size });
- }
- };
- fixup(&mut fn_abi.ret);
- for arg in fn_abi.args.iter_mut() {
- fixup(arg);
- }
- } else {
- fn_abi.adjust_for_foreign_abi(self, abi)?;
- }
-
- Ok(())
- }
-}
-
-#[tracing::instrument(level = "debug", skip(cx))]
-fn make_thin_self_ptr<'tcx>(
- cx: &(impl HasTyCtxt<'tcx> + HasParamEnv<'tcx>),
- layout: TyAndLayout<'tcx>,
-) -> TyAndLayout<'tcx> {
- let tcx = cx.tcx();
- let fat_pointer_ty = if layout.is_unsized() {
- // unsized `self` is passed as a pointer to `self`
- // FIXME (mikeyhew) change this to use &own if it is ever added to the language
- tcx.mk_mut_ptr(layout.ty)
- } else {
- match layout.abi {
- Abi::ScalarPair(..) | Abi::Scalar(..) => (),
- _ => bug!("receiver type has unsupported layout: {:?}", layout),
- }
-
- // In the case of Rc<Self>, we need to explicitly pass a *mut RcBox<Self>
- // with a Scalar (not ScalarPair) ABI. This is a hack that is understood
- // elsewhere in the compiler as a method on a `dyn Trait`.
- // To get the type `*mut RcBox<Self>`, we just keep unwrapping newtypes until we
- // get a built-in pointer type
- let mut fat_pointer_layout = layout;
- 'descend_newtypes: while !fat_pointer_layout.ty.is_unsafe_ptr()
- && !fat_pointer_layout.ty.is_region_ptr()
- {
- for i in 0..fat_pointer_layout.fields.count() {
- let field_layout = fat_pointer_layout.field(cx, i);
-
- if !field_layout.is_zst() {
- fat_pointer_layout = field_layout;
- continue 'descend_newtypes;
- }
- }
-
- bug!("receiver has no non-zero-sized fields {:?}", fat_pointer_layout);
- }
-
- fat_pointer_layout.ty
- };
-
- // we now have a type like `*mut RcBox<dyn Trait>`
- // change its layout to that of `*mut ()`, a thin pointer, but keep the same type
- // this is understood as a special case elsewhere in the compiler
- let unit_ptr_ty = tcx.mk_mut_ptr(tcx.mk_unit());
-
- TyAndLayout {
- ty: fat_pointer_ty,
-
- // NOTE(eddyb) using an empty `ParamEnv`, and `unwrap`-ing the `Result`
- // should always work because the type is always `*mut ()`.
- ..tcx.layout_of(ty::ParamEnv::reveal_all().and(unit_ptr_ty)).unwrap()
- }
-}
projects / rust.git / blobdiff