src/librustc/ty/layout.rs

   1 use crate::session::{self, DataTypeKind};
   2 use crate::ty::{self, subst::SubstsRef, ReprOptions, Ty, TyCtxt, TypeFoldable};
   3
   4 use rustc_span::DUMMY_SP;
   5 use syntax::ast::{self, Ident, IntTy, UintTy};
   6 use syntax::attr;
   7
   8 use std::cmp;
   9 use std::fmt;
  10 use std::i128;
  11 use std::iter;
  12 use std::mem;
  13 use std::ops::Bound;
  14
  15 use crate::ich::StableHashingContext;
  16 use crate::mir::{GeneratorLayout, GeneratorSavedLocal};
  17 use crate::ty::subst::Subst;
  18 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
  19 use rustc_hir as hir;
  20 use rustc_index::bit_set::BitSet;
  21 use rustc_index::vec::{Idx, IndexVec};
  22
  23 use rustc_target::abi::call::{
  24     ArgAbi, ArgAttribute, ArgAttributes, Conv, FnAbi, PassMode, Reg, RegKind,
  25 };
  26 pub use rustc_target::abi::*;
  27 use rustc_target::spec::{abi::Abi as SpecAbi, HasTargetSpec};
  28
  29 pub trait IntegerExt {
  30     fn to_ty<'tcx>(&self, tcx: TyCtxt<'tcx>, signed: bool) -> Ty<'tcx>;
  31     fn from_attr<C: HasDataLayout>(cx: &C, ity: attr::IntType) -> Integer;
  32     fn repr_discr<'tcx>(
  33         tcx: TyCtxt<'tcx>,
  34         ty: Ty<'tcx>,
  35         repr: &ReprOptions,
  36         min: i128,
  37         max: i128,
  38     ) -> (Integer, bool);
  39 }
  40
  41 impl IntegerExt for Integer {
  42     fn to_ty<'tcx>(&self, tcx: TyCtxt<'tcx>, signed: bool) -> Ty<'tcx> {
  43         match (*self, signed) {
  44             (I8, false) => tcx.types.u8,
  45             (I16, false) => tcx.types.u16,
  46             (I32, false) => tcx.types.u32,
  47             (I64, false) => tcx.types.u64,
  48             (I128, false) => tcx.types.u128,
  49             (I8, true) => tcx.types.i8,
  50             (I16, true) => tcx.types.i16,
  51             (I32, true) => tcx.types.i32,
  52             (I64, true) => tcx.types.i64,
  53             (I128, true) => tcx.types.i128,
  54         }
  55     }
  56
  57     /// Gets the Integer type from an attr::IntType.
  58     fn from_attr<C: HasDataLayout>(cx: &C, ity: attr::IntType) -> Integer {
  59         let dl = cx.data_layout();
  60
  61         match ity {
  62             attr::SignedInt(IntTy::I8) | attr::UnsignedInt(UintTy::U8) => I8,
  63             attr::SignedInt(IntTy::I16) | attr::UnsignedInt(UintTy::U16) => I16,
  64             attr::SignedInt(IntTy::I32) | attr::UnsignedInt(UintTy::U32) => I32,
  65             attr::SignedInt(IntTy::I64) | attr::UnsignedInt(UintTy::U64) => I64,
  66             attr::SignedInt(IntTy::I128) | attr::UnsignedInt(UintTy::U128) => I128,
  67             attr::SignedInt(IntTy::Isize) | attr::UnsignedInt(UintTy::Usize) => {
  68                 dl.ptr_sized_integer()
  69             }
  70         }
  71     }
  72
  73     /// Finds the appropriate Integer type and signedness for the given
  74     /// signed discriminant range and #[repr] attribute.
  75     /// N.B.: u128 values above i128::MAX will be treated as signed, but
  76     /// that shouldn't affect anything, other than maybe debuginfo.
  77     fn repr_discr<'tcx>(
  78         tcx: TyCtxt<'tcx>,
  79         ty: Ty<'tcx>,
  80         repr: &ReprOptions,
  81         min: i128,
  82         max: i128,
  83     ) -> (Integer, bool) {
  84         // Theoretically, negative values could be larger in unsigned representation
  85         // than the unsigned representation of the signed minimum. However, if there
  86         // are any negative values, the only valid unsigned representation is u128
  87         // which can fit all i128 values, so the result remains unaffected.
  88         let unsigned_fit = Integer::fit_unsigned(cmp::max(min as u128, max as u128));
  89         let signed_fit = cmp::max(Integer::fit_signed(min), Integer::fit_signed(max));
  90
  91         let mut min_from_extern = None;
  92         let min_default = I8;
  93
  94         if let Some(ity) = repr.int {
  95             let discr = Integer::from_attr(&tcx, ity);
  96             let fit = if ity.is_signed() { signed_fit } else { unsigned_fit };
  97             if discr < fit {
  98                 bug!(
  99                     "Integer::repr_discr: `#[repr]` hint too small for \
 100                       discriminant range of enum `{}",
 101                     ty
 102                 )
 103             }
 104             return (discr, ity.is_signed());
 105         }
 106
 107         if repr.c() {
 108             match &tcx.sess.target.target.arch[..] {
 109                 // WARNING: the ARM EABI has two variants; the one corresponding
 110                 // to `at_least == I32` appears to be used on Linux and NetBSD,
 111                 // but some systems may use the variant corresponding to no
 112                 // lower bound. However, we don't run on those yet...?
 113                 "arm" => min_from_extern = Some(I32),
 114                 _ => min_from_extern = Some(I32),
 115             }
 116         }
 117
 118         let at_least = min_from_extern.unwrap_or(min_default);
 119
 120         // If there are no negative values, we can use the unsigned fit.
 121         if min >= 0 {
 122             (cmp::max(unsigned_fit, at_least), false)
 123         } else {
 124             (cmp::max(signed_fit, at_least), true)
 125         }
 126     }
 127 }
 128
 129 pub trait PrimitiveExt {
 130     fn to_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx>;
 131     fn to_int_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx>;
 132 }
 133
 134 impl PrimitiveExt for Primitive {
 135     fn to_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
 136         match *self {
 137             Int(i, signed) => i.to_ty(tcx, signed),
 138             F32 => tcx.types.f32,
 139             F64 => tcx.types.f64,
 140             Pointer => tcx.mk_mut_ptr(tcx.mk_unit()),
 141         }
 142     }
 143
 144     /// Return an *integer* type matching this primitive.
 145     /// Useful in particular when dealing with enum discriminants.
 146     fn to_int_ty(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
 147         match *self {
 148             Int(i, signed) => i.to_ty(tcx, signed),
 149             Pointer => tcx.types.usize,
 150             F32 | F64 => bug!("floats do not have an int type"),
 151         }
 152     }
 153 }
 154
 155 /// The first half of a fat pointer.
 156 ///
 157 /// - For a trait object, this is the address of the box.
 158 /// - For a slice, this is the base address.
 159 pub const FAT_PTR_ADDR: usize = 0;
 160
 161 /// The second half of a fat pointer.
 162 ///
 163 /// - For a trait object, this is the address of the vtable.
 164 /// - For a slice, this is the length.
 165 pub const FAT_PTR_EXTRA: usize = 1;
 166
 167 #[derive(Copy, Clone, Debug, RustcEncodable, RustcDecodable)]
 168 pub enum LayoutError<'tcx> {
 169     Unknown(Ty<'tcx>),
 170     SizeOverflow(Ty<'tcx>),
 171 }
 172
 173 impl<'tcx> fmt::Display for LayoutError<'tcx> {
 174     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 175         match *self {
 176             LayoutError::Unknown(ty) => write!(f, "the type `{:?}` has an unknown layout", ty),
 177             LayoutError::SizeOverflow(ty) => {
 178                 write!(f, "the type `{:?}` is too big for the current architecture", ty)
 179             }
 180         }
 181     }
 182 }
 183
 184 fn layout_raw<'tcx>(
 185     tcx: TyCtxt<'tcx>,
 186     query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
 187 ) -> Result<&'tcx LayoutDetails, LayoutError<'tcx>> {
 188     ty::tls::with_related_context(tcx, move |icx| {
 189         let rec_limit = *tcx.sess.recursion_limit.get();
 190         let (param_env, ty) = query.into_parts();
 191
 192         if icx.layout_depth > rec_limit {
 193             tcx.sess.fatal(&format!("overflow representing the type `{}`", ty));
 194         }
 195
 196         // Update the ImplicitCtxt to increase the layout_depth
 197         let icx = ty::tls::ImplicitCtxt { layout_depth: icx.layout_depth + 1, ..icx.clone() };
 198
 199         ty::tls::enter_context(&icx, |_| {
 200             let cx = LayoutCx { tcx, param_env };
 201             let layout = cx.layout_raw_uncached(ty);
 202             // Type-level uninhabitedness should always imply ABI uninhabitedness.
 203             if let Ok(layout) = layout {
 204                 if ty.conservative_is_privately_uninhabited(tcx) {
 205                     assert!(layout.abi.is_uninhabited());
 206                 }
 207             }
 208             layout
 209         })
 210     })
 211 }
 212
 213 pub fn provide(providers: &mut ty::query::Providers<'_>) {
 214     *providers = ty::query::Providers { layout_raw, ..*providers };
 215 }
 216
 217 pub struct LayoutCx<'tcx, C> {
 218     pub tcx: C,
 219     pub param_env: ty::ParamEnv<'tcx>,
 220 }
 221
 222 #[derive(Copy, Clone, Debug)]
 223 enum StructKind {
 224     /// A tuple, closure, or univariant which cannot be coerced to unsized.
 225     AlwaysSized,
 226     /// A univariant, the last field of which may be coerced to unsized.
 227     MaybeUnsized,
 228     /// A univariant, but with a prefix of an arbitrary size & alignment (e.g., enum tag).
 229     Prefixed(Size, Align),
 230 }
 231
 232 // Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`.
 233 // This is used to go between `memory_index` (source field order to memory order)
 234 // and `inverse_memory_index` (memory order to source field order).
 235 // See also `FieldPlacement::Arbitrary::memory_index` for more details.
 236 // FIXME(eddyb) build a better abstraction for permutations, if possible.
 237 fn invert_mapping(map: &[u32]) -> Vec<u32> {
 238     let mut inverse = vec![0; map.len()];
 239     for i in 0..map.len() {
 240         inverse[map[i] as usize] = i as u32;
 241     }
 242     inverse
 243 }
 244
 245 impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
 246     fn scalar_pair(&self, a: Scalar, b: Scalar) -> LayoutDetails {
 247         let dl = self.data_layout();
 248         let b_align = b.value.align(dl);
 249         let align = a.value.align(dl).max(b_align).max(dl.aggregate_align);
 250         let b_offset = a.value.size(dl).align_to(b_align.abi);
 251         let size = (b_offset + b.value.size(dl)).align_to(align.abi);
 252
 253         // HACK(nox): We iter on `b` and then `a` because `max_by_key`
 254         // returns the last maximum.
 255         let largest_niche = Niche::from_scalar(dl, b_offset, b.clone())
 256             .into_iter()
 257             .chain(Niche::from_scalar(dl, Size::ZERO, a.clone()))
 258             .max_by_key(|niche| niche.available(dl));
 259
 260         LayoutDetails {
 261             variants: Variants::Single { index: VariantIdx::new(0) },
 262             fields: FieldPlacement::Arbitrary {
 263                 offsets: vec![Size::ZERO, b_offset],
 264                 memory_index: vec![0, 1],
 265             },
 266             abi: Abi::ScalarPair(a, b),
 267             largest_niche,
 268             align,
 269             size,
 270         }
 271     }
 272
 273     fn univariant_uninterned(
 274         &self,
 275         ty: Ty<'tcx>,
 276         fields: &[TyLayout<'_>],
 277         repr: &ReprOptions,
 278         kind: StructKind,
 279     ) -> Result<LayoutDetails, LayoutError<'tcx>> {
 280         let dl = self.data_layout();
 281         let pack = repr.pack;
 282         if pack.is_some() && repr.align.is_some() {
 283             bug!("struct cannot be packed and aligned");
 284         }
 285
 286         let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
 287
 288         let mut sized = true;
 289         let mut offsets = vec![Size::ZERO; fields.len()];
 290         let mut inverse_memory_index: Vec<u32> = (0..fields.len() as u32).collect();
 291
 292         let mut optimize = !repr.inhibit_struct_field_reordering_opt();
 293         if let StructKind::Prefixed(_, align) = kind {
 294             optimize &= align.bytes() == 1;
 295         }
 296
 297         if optimize {
 298             let end =
 299                 if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
 300             let optimizing = &mut inverse_memory_index[..end];
 301             let field_align = |f: &TyLayout<'_>| {
 302                 if let Some(pack) = pack { f.align.abi.min(pack) } else { f.align.abi }
 303             };
 304             match kind {
 305                 StructKind::AlwaysSized | StructKind::MaybeUnsized => {
 306                     optimizing.sort_by_key(|&x| {
 307                         // Place ZSTs first to avoid "interesting offsets",
 308                         // especially with only one or two non-ZST fields.
 309                         let f = &fields[x as usize];
 310                         (!f.is_zst(), cmp::Reverse(field_align(f)))
 311                     });
 312                 }
 313                 StructKind::Prefixed(..) => {
 314                     optimizing.sort_by_key(|&x| field_align(&fields[x as usize]));
 315                 }
 316             }
 317         }
 318
 319         // inverse_memory_index holds field indices by increasing memory offset.
 320         // That is, if field 5 has offset 0, the first element of inverse_memory_index is 5.
 321         // We now write field offsets to the corresponding offset slot;
 322         // field 5 with offset 0 puts 0 in offsets[5].
 323         // At the bottom of this function, we invert `inverse_memory_index` to
 324         // produce `memory_index` (see `invert_mapping`).
 325
 326         let mut offset = Size::ZERO;
 327         let mut largest_niche = None;
 328         let mut largest_niche_available = 0;
 329
 330         if let StructKind::Prefixed(prefix_size, prefix_align) = kind {
 331             let prefix_align =
 332                 if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align };
 333             align = align.max(AbiAndPrefAlign::new(prefix_align));
 334             offset = prefix_size.align_to(prefix_align);
 335         }
 336
 337         for &i in &inverse_memory_index {
 338             let field = fields[i as usize];
 339             if !sized {
 340                 bug!("univariant: field #{} of `{}` comes after unsized field", offsets.len(), ty);
 341             }
 342
 343             if field.is_unsized() {
 344                 sized = false;
 345             }
 346
 347             // Invariant: offset < dl.obj_size_bound() <= 1<<61
 348             let field_align = if let Some(pack) = pack {
 349                 field.align.min(AbiAndPrefAlign::new(pack))
 350             } else {
 351                 field.align
 352             };
 353             offset = offset.align_to(field_align.abi);
 354             align = align.max(field_align);
 355
 356             debug!("univariant offset: {:?} field: {:#?}", offset, field);
 357             offsets[i as usize] = offset;
 358
 359             if let Some(mut niche) = field.largest_niche.clone() {
 360                 let available = niche.available(dl);
 361                 if available > largest_niche_available {
 362                     largest_niche_available = available;
 363                     niche.offset += offset;
 364                     largest_niche = Some(niche);
 365                 }
 366             }
 367
 368             offset = offset.checked_add(field.size, dl).ok_or(LayoutError::SizeOverflow(ty))?;
 369         }
 370
 371         if let Some(repr_align) = repr.align {
 372             align = align.max(AbiAndPrefAlign::new(repr_align));
 373         }
 374
 375         debug!("univariant min_size: {:?}", offset);
 376         let min_size = offset;
 377
 378         // As stated above, inverse_memory_index holds field indices by increasing offset.
 379         // This makes it an already-sorted view of the offsets vec.
 380         // To invert it, consider:
 381         // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
 382         // Field 5 would be the first element, so memory_index is i:
 383         // Note: if we didn't optimize, it's already right.
 384
 385         let memory_index;
 386         if optimize {
 387             memory_index = invert_mapping(&inverse_memory_index);
 388         } else {
 389             memory_index = inverse_memory_index;
 390         }
 391
 392         let size = min_size.align_to(align.abi);
 393         let mut abi = Abi::Aggregate { sized };
 394
 395         // Unpack newtype ABIs and find scalar pairs.
 396         if sized && size.bytes() > 0 {
 397             // All other fields must be ZSTs, and we need them to all start at 0.
 398             let mut zst_offsets = offsets.iter().enumerate().filter(|&(i, _)| fields[i].is_zst());
 399             if zst_offsets.all(|(_, o)| o.bytes() == 0) {
 400                 let mut non_zst_fields = fields.iter().enumerate().filter(|&(_, f)| !f.is_zst());
 401
 402                 match (non_zst_fields.next(), non_zst_fields.next(), non_zst_fields.next()) {
 403                     // We have exactly one non-ZST field.
 404                     (Some((i, field)), None, None) => {
 405                         // Field fills the struct and it has a scalar or scalar pair ABI.
 406                         if offsets[i].bytes() == 0
 407                             && align.abi == field.align.abi
 408                             && size == field.size
 409                         {
 410                             match field.abi {
 411                                 // For plain scalars, or vectors of them, we can't unpack
 412                                 // newtypes for `#[repr(C)]`, as that affects C ABIs.
 413                                 Abi::Scalar(_) | Abi::Vector { .. } if optimize => {
 414                                     abi = field.abi.clone();
 415                                 }
 416                                 // But scalar pairs are Rust-specific and get
 417                                 // treated as aggregates by C ABIs anyway.
 418                                 Abi::ScalarPair(..) => {
 419                                     abi = field.abi.clone();
 420                                 }
 421                                 _ => {}
 422                             }
 423                         }
 424                     }
 425
 426                     // Two non-ZST fields, and they're both scalars.
 427                     (
 428                         Some((
 429                             i,
 430                             &TyLayout {
 431                                 details: &LayoutDetails { abi: Abi::Scalar(ref a), .. },
 432                                 ..
 433                             },
 434                         )),
 435                         Some((
 436                             j,
 437                             &TyLayout {
 438                                 details: &LayoutDetails { abi: Abi::Scalar(ref b), .. },
 439                                 ..
 440                             },
 441                         )),
 442                         None,
 443                     ) => {
 444                         // Order by the memory placement, not source order.
 445                         let ((i, a), (j, b)) = if offsets[i] < offsets[j] {
 446                             ((i, a), (j, b))
 447                         } else {
 448                             ((j, b), (i, a))
 449                         };
 450                         let pair = self.scalar_pair(a.clone(), b.clone());
 451                         let pair_offsets = match pair.fields {
 452                             FieldPlacement::Arbitrary { ref offsets, ref memory_index } => {
 453                                 assert_eq!(memory_index, &[0, 1]);
 454                                 offsets
 455                             }
 456                             _ => bug!(),
 457                         };
 458                         if offsets[i] == pair_offsets[0]
 459                             && offsets[j] == pair_offsets[1]
 460                             && align == pair.align
 461                             && size == pair.size
 462                         {
 463                             // We can use `ScalarPair` only when it matches our
 464                             // already computed layout (including `#[repr(C)]`).
 465                             abi = pair.abi;
 466                         }
 467                     }
 468
 469                     _ => {}
 470                 }
 471             }
 472         }
 473
 474         if sized && fields.iter().any(|f| f.abi.is_uninhabited()) {
 475             abi = Abi::Uninhabited;
 476         }
 477
 478         Ok(LayoutDetails {
 479             variants: Variants::Single { index: VariantIdx::new(0) },
 480             fields: FieldPlacement::Arbitrary { offsets, memory_index },
 481             abi,
 482             largest_niche,
 483             align,
 484             size,
 485         })
 486     }
 487
 488     fn layout_raw_uncached(&self, ty: Ty<'tcx>) -> Result<&'tcx LayoutDetails, LayoutError<'tcx>> {
 489         let tcx = self.tcx;
 490         let param_env = self.param_env;
 491         let dl = self.data_layout();
 492         let scalar_unit = |value: Primitive| {
 493             let bits = value.size(dl).bits();
 494             assert!(bits <= 128);
 495             Scalar { value, valid_range: 0..=(!0 >> (128 - bits)) }
 496         };
 497         let scalar =
 498             |value: Primitive| tcx.intern_layout(LayoutDetails::scalar(self, scalar_unit(value)));
 499
 500         let univariant = |fields: &[TyLayout<'_>], repr: &ReprOptions, kind| {
 501             Ok(tcx.intern_layout(self.univariant_uninterned(ty, fields, repr, kind)?))
 502         };
 503         debug_assert!(!ty.has_infer_types());
 504
 505         Ok(match ty.kind {
 506             // Basic scalars.
 507             ty::Bool => tcx.intern_layout(LayoutDetails::scalar(
 508                 self,
 509                 Scalar { value: Int(I8, false), valid_range: 0..=1 },
 510             )),
 511             ty::Char => tcx.intern_layout(LayoutDetails::scalar(
 512                 self,
 513                 Scalar { value: Int(I32, false), valid_range: 0..=0x10FFFF },
 514             )),
 515             ty::Int(ity) => scalar(Int(Integer::from_attr(dl, attr::SignedInt(ity)), true)),
 516             ty::Uint(ity) => scalar(Int(Integer::from_attr(dl, attr::UnsignedInt(ity)), false)),
 517             ty::Float(fty) => scalar(match fty {
 518                 ast::FloatTy::F32 => F32,
 519                 ast::FloatTy::F64 => F64,
 520             }),
 521             ty::FnPtr(_) => {
 522                 let mut ptr = scalar_unit(Pointer);
 523                 ptr.valid_range = 1..=*ptr.valid_range.end();
 524                 tcx.intern_layout(LayoutDetails::scalar(self, ptr))
 525             }
 526
 527             // The never type.
 528             ty::Never => tcx.intern_layout(LayoutDetails {
 529                 variants: Variants::Single { index: VariantIdx::new(0) },
 530                 fields: FieldPlacement::Union(0),
 531                 abi: Abi::Uninhabited,
 532                 largest_niche: None,
 533                 align: dl.i8_align,
 534                 size: Size::ZERO,
 535             }),
 536
 537             // Potentially-fat pointers.
 538             ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => {
 539                 let mut data_ptr = scalar_unit(Pointer);
 540                 if !ty.is_unsafe_ptr() {
 541                     data_ptr.valid_range = 1..=*data_ptr.valid_range.end();
 542                 }
 543
 544                 let pointee = tcx.normalize_erasing_regions(param_env, pointee);
 545                 if pointee.is_sized(tcx.at(DUMMY_SP), param_env) {
 546                     return Ok(tcx.intern_layout(LayoutDetails::scalar(self, data_ptr)));
 547                 }
 548
 549                 let unsized_part = tcx.struct_tail_erasing_lifetimes(pointee, param_env);
 550                 let metadata = match unsized_part.kind {
 551                     ty::Foreign(..) => {
 552                         return Ok(tcx.intern_layout(LayoutDetails::scalar(self, data_ptr)));
 553                     }
 554                     ty::Slice(_) | ty::Str => scalar_unit(Int(dl.ptr_sized_integer(), false)),
 555                     ty::Dynamic(..) => {
 556                         let mut vtable = scalar_unit(Pointer);
 557                         vtable.valid_range = 1..=*vtable.valid_range.end();
 558                         vtable
 559                     }
 560                     _ => return Err(LayoutError::Unknown(unsized_part)),
 561                 };
 562
 563                 // Effectively a (ptr, meta) tuple.
 564                 tcx.intern_layout(self.scalar_pair(data_ptr, metadata))
 565             }
 566
 567             // Arrays and slices.
 568             ty::Array(element, mut count) => {
 569                 if count.has_projections() {
 570                     count = tcx.normalize_erasing_regions(param_env, count);
 571                     if count.has_projections() {
 572                         return Err(LayoutError::Unknown(ty));
 573                     }
 574                 }
 575
 576                 let count = count.try_eval_usize(tcx, param_env).ok_or(LayoutError::Unknown(ty))?;
 577                 let element = self.layout_of(element)?;
 578                 let size =
 579                     element.size.checked_mul(count, dl).ok_or(LayoutError::SizeOverflow(ty))?;
 580
 581                 let abi = if count != 0 && ty.conservative_is_privately_uninhabited(tcx) {
 582                     Abi::Uninhabited
 583                 } else {
 584                     Abi::Aggregate { sized: true }
 585                 };
 586
 587                 let largest_niche = if count != 0 { element.largest_niche.clone() } else { None };
 588
 589                 tcx.intern_layout(LayoutDetails {
 590                     variants: Variants::Single { index: VariantIdx::new(0) },
 591                     fields: FieldPlacement::Array { stride: element.size, count },
 592                     abi,
 593                     largest_niche,
 594                     align: element.align,
 595                     size,
 596                 })
 597             }
 598             ty::Slice(element) => {
 599                 let element = self.layout_of(element)?;
 600                 tcx.intern_layout(LayoutDetails {
 601                     variants: Variants::Single { index: VariantIdx::new(0) },
 602                     fields: FieldPlacement::Array { stride: element.size, count: 0 },
 603                     abi: Abi::Aggregate { sized: false },
 604                     largest_niche: None,
 605                     align: element.align,
 606                     size: Size::ZERO,
 607                 })
 608             }
 609             ty::Str => tcx.intern_layout(LayoutDetails {
 610                 variants: Variants::Single { index: VariantIdx::new(0) },
 611                 fields: FieldPlacement::Array { stride: Size::from_bytes(1), count: 0 },
 612                 abi: Abi::Aggregate { sized: false },
 613                 largest_niche: None,
 614                 align: dl.i8_align,
 615                 size: Size::ZERO,
 616             }),
 617
 618             // Odd unit types.
 619             ty::FnDef(..) => univariant(&[], &ReprOptions::default(), StructKind::AlwaysSized)?,
 620             ty::Dynamic(..) | ty::Foreign(..) => {
 621                 let mut unit = self.univariant_uninterned(
 622                     ty,
 623                     &[],
 624                     &ReprOptions::default(),
 625                     StructKind::AlwaysSized,
 626                 )?;
 627                 match unit.abi {
 628                     Abi::Aggregate { ref mut sized } => *sized = false,
 629                     _ => bug!(),
 630                 }
 631                 tcx.intern_layout(unit)
 632             }
 633
 634             ty::Generator(def_id, substs, _) => self.generator_layout(ty, def_id, substs)?,
 635
 636             ty::Closure(def_id, ref substs) => {
 637                 let tys = substs.as_closure().upvar_tys(def_id, tcx);
 638                 univariant(
 639                     &tys.map(|ty| self.layout_of(ty)).collect::<Result<Vec<_>, _>>()?,
 640                     &ReprOptions::default(),
 641                     StructKind::AlwaysSized,
 642                 )?
 643             }
 644
 645             ty::Tuple(tys) => {
 646                 let kind =
 647                     if tys.len() == 0 { StructKind::AlwaysSized } else { StructKind::MaybeUnsized };
 648
 649                 univariant(
 650                     &tys.iter()
 651                         .map(|k| self.layout_of(k.expect_ty()))
 652                         .collect::<Result<Vec<_>, _>>()?,
 653                     &ReprOptions::default(),
 654                     kind,
 655                 )?
 656             }
 657
 658             // SIMD vector types.
 659             ty::Adt(def, ..) if def.repr.simd() => {
 660                 let element = self.layout_of(ty.simd_type(tcx))?;
 661                 let count = ty.simd_size(tcx);
 662                 assert!(count > 0);
 663                 let scalar = match element.abi {
 664                     Abi::Scalar(ref scalar) => scalar.clone(),
 665                     _ => {
 666                         tcx.sess.fatal(&format!(
 667                             "monomorphising SIMD type `{}` with \
 668                                                  a non-machine element type `{}`",
 669                             ty, element.ty
 670                         ));
 671                     }
 672                 };
 673                 let size =
 674                     element.size.checked_mul(count, dl).ok_or(LayoutError::SizeOverflow(ty))?;
 675                 let align = dl.vector_align(size);
 676                 let size = size.align_to(align.abi);
 677
 678                 tcx.intern_layout(LayoutDetails {
 679                     variants: Variants::Single { index: VariantIdx::new(0) },
 680                     fields: FieldPlacement::Array { stride: element.size, count },
 681                     abi: Abi::Vector { element: scalar, count },
 682                     largest_niche: element.largest_niche.clone(),
 683                     size,
 684                     align,
 685                 })
 686             }
 687
 688             // ADTs.
 689             ty::Adt(def, substs) => {
 690                 // Cache the field layouts.
 691                 let variants = def
 692                     .variants
 693                     .iter()
 694                     .map(|v| {
 695                         v.fields
 696                             .iter()
 697                             .map(|field| self.layout_of(field.ty(tcx, substs)))
 698                             .collect::<Result<Vec<_>, _>>()
 699                     })
 700                     .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
 701
 702                 if def.is_union() {
 703                     if def.repr.pack.is_some() && def.repr.align.is_some() {
 704                         bug!("union cannot be packed and aligned");
 705                     }
 706
 707                     let mut align =
 708                         if def.repr.pack.is_some() { dl.i8_align } else { dl.aggregate_align };
 709
 710                     if let Some(repr_align) = def.repr.align {
 711                         align = align.max(AbiAndPrefAlign::new(repr_align));
 712                     }
 713
 714                     let optimize = !def.repr.inhibit_union_abi_opt();
 715                     let mut size = Size::ZERO;
 716                     let mut abi = Abi::Aggregate { sized: true };
 717                     let index = VariantIdx::new(0);
 718                     for field in &variants[index] {
 719                         assert!(!field.is_unsized());
 720                         align = align.max(field.align);
 721
 722                         // If all non-ZST fields have the same ABI, forward this ABI
 723                         if optimize && !field.is_zst() {
 724                             // Normalize scalar_unit to the maximal valid range
 725                             let field_abi = match &field.abi {
 726                                 Abi::Scalar(x) => Abi::Scalar(scalar_unit(x.value)),
 727                                 Abi::ScalarPair(x, y) => {
 728                                     Abi::ScalarPair(scalar_unit(x.value), scalar_unit(y.value))
 729                                 }
 730                                 Abi::Vector { element: x, count } => {
 731                                     Abi::Vector { element: scalar_unit(x.value), count: *count }
 732                                 }
 733                                 Abi::Uninhabited | Abi::Aggregate { .. } => {
 734                                     Abi::Aggregate { sized: true }
 735                                 }
 736                             };
 737
 738                             if size == Size::ZERO {
 739                                 // first non ZST: initialize 'abi'
 740                                 abi = field_abi;
 741                             } else if abi != field_abi {
 742                                 // different fields have different ABI: reset to Aggregate
 743                                 abi = Abi::Aggregate { sized: true };
 744                             }
 745                         }
 746
 747                         size = cmp::max(size, field.size);
 748                     }
 749
 750                     if let Some(pack) = def.repr.pack {
 751                         align = align.min(AbiAndPrefAlign::new(pack));
 752                     }
 753
 754                     return Ok(tcx.intern_layout(LayoutDetails {
 755                         variants: Variants::Single { index },
 756                         fields: FieldPlacement::Union(variants[index].len()),
 757                         abi,
 758                         largest_niche: None,
 759                         align,
 760                         size: size.align_to(align.abi),
 761                     }));
 762                 }
 763
 764                 // A variant is absent if it's uninhabited and only has ZST fields.
 765                 // Present uninhabited variants only require space for their fields,
 766                 // but *not* an encoding of the discriminant (e.g., a tag value).
 767                 // See issue #49298 for more details on the need to leave space
 768                 // for non-ZST uninhabited data (mostly partial initialization).
 769                 let absent = |fields: &[TyLayout<'_>]| {
 770                     let uninhabited = fields.iter().any(|f| f.abi.is_uninhabited());
 771                     let is_zst = fields.iter().all(|f| f.is_zst());
 772                     uninhabited && is_zst
 773                 };
 774                 let (present_first, present_second) = {
 775                     let mut present_variants = variants
 776                         .iter_enumerated()
 777                         .filter_map(|(i, v)| if absent(v) { None } else { Some(i) });
 778                     (present_variants.next(), present_variants.next())
 779                 };
 780                 let present_first = match present_first {
 781                     present_first @ Some(_) => present_first,
 782                     // Uninhabited because it has no variants, or only absent ones.
 783                     None if def.is_enum() => return tcx.layout_raw(param_env.and(tcx.types.never)),
 784                     // if it's a struct, still compute a layout so that we can still compute the
 785                     // field offsets
 786                     None => Some(VariantIdx::new(0)),
 787                 };
 788
 789                 let is_struct = !def.is_enum() ||
 790                     // Only one variant is present.
 791                     (present_second.is_none() &&
 792                     // Representation optimizations are allowed.
 793                     !def.repr.inhibit_enum_layout_opt());
 794                 if is_struct {
 795                     // Struct, or univariant enum equivalent to a struct.
 796                     // (Typechecking will reject discriminant-sizing attrs.)
 797
 798                     let v = present_first.unwrap();
 799                     let kind = if def.is_enum() || variants[v].len() == 0 {
 800                         StructKind::AlwaysSized
 801                     } else {
 802                         let param_env = tcx.param_env(def.did);
 803                         let last_field = def.variants[v].fields.last().unwrap();
 804                         let always_sized =
 805                             tcx.type_of(last_field.did).is_sized(tcx.at(DUMMY_SP), param_env);
 806                         if !always_sized {
 807                             StructKind::MaybeUnsized
 808                         } else {
 809                             StructKind::AlwaysSized
 810                         }
 811                     };
 812
 813                     let mut st = self.univariant_uninterned(ty, &variants[v], &def.repr, kind)?;
 814                     st.variants = Variants::Single { index: v };
 815                     let (start, end) = self.tcx.layout_scalar_valid_range(def.did);
 816                     match st.abi {
 817                         Abi::Scalar(ref mut scalar) | Abi::ScalarPair(ref mut scalar, _) => {
 818                             // the asserts ensure that we are not using the
 819                             // `#[rustc_layout_scalar_valid_range(n)]`
 820                             // attribute to widen the range of anything as that would probably
 821                             // result in UB somewhere
 822                             // FIXME(eddyb) the asserts are probably not needed,
 823                             // as larger validity ranges would result in missed
 824                             // optimizations, *not* wrongly assuming the inner
 825                             // value is valid. e.g. unions enlarge validity ranges,
 826                             // because the values may be uninitialized.
 827                             if let Bound::Included(start) = start {
 828                                 // FIXME(eddyb) this might be incorrect - it doesn't
 829                                 // account for wrap-around (end < start) ranges.
 830                                 assert!(*scalar.valid_range.start() <= start);
 831                                 scalar.valid_range = start..=*scalar.valid_range.end();
 832                             }
 833                             if let Bound::Included(end) = end {
 834                                 // FIXME(eddyb) this might be incorrect - it doesn't
 835                                 // account for wrap-around (end < start) ranges.
 836                                 assert!(*scalar.valid_range.end() >= end);
 837                                 scalar.valid_range = *scalar.valid_range.start()..=end;
 838                             }
 839
 840                             // Update `largest_niche` if we have introduced a larger niche.
 841                             let niche = Niche::from_scalar(dl, Size::ZERO, scalar.clone());
 842                             if let Some(niche) = niche {
 843                                 match &st.largest_niche {
 844                                     Some(largest_niche) => {
 845                                         // Replace the existing niche even if they're equal,
 846                                         // because this one is at a lower offset.
 847                                         if largest_niche.available(dl) <= niche.available(dl) {
 848                                             st.largest_niche = Some(niche);
 849                                         }
 850                                     }
 851                                     None => st.largest_niche = Some(niche),
 852                                 }
 853                             }
 854                         }
 855                         _ => assert!(
 856                             start == Bound::Unbounded && end == Bound::Unbounded,
 857                             "nonscalar layout for layout_scalar_valid_range type {:?}: {:#?}",
 858                             def,
 859                             st,
 860                         ),
 861                     }
 862
 863                     return Ok(tcx.intern_layout(st));
 864                 }
 865
 866                 // The current code for niche-filling relies on variant indices
 867                 // instead of actual discriminants, so dataful enums with
 868                 // explicit discriminants (RFC #2363) would misbehave.
 869                 let no_explicit_discriminants = def
 870                     .variants
 871                     .iter_enumerated()
 872                     .all(|(i, v)| v.discr == ty::VariantDiscr::Relative(i.as_u32()));
 873
 874                 // Niche-filling enum optimization.
 875                 if !def.repr.inhibit_enum_layout_opt() && no_explicit_discriminants {
 876                     let mut dataful_variant = None;
 877                     let mut niche_variants = VariantIdx::MAX..=VariantIdx::new(0);
 878
 879                     // Find one non-ZST variant.
 880                     'variants: for (v, fields) in variants.iter_enumerated() {
 881                         if absent(fields) {
 882                             continue 'variants;
 883                         }
 884                         for f in fields {
 885                             if !f.is_zst() {
 886                                 if dataful_variant.is_none() {
 887                                     dataful_variant = Some(v);
 888                                     continue 'variants;
 889                                 } else {
 890                                     dataful_variant = None;
 891                                     break 'variants;
 892                                 }
 893                             }
 894                         }
 895                         niche_variants = *niche_variants.start().min(&v)..=v;
 896                     }
 897
 898                     if niche_variants.start() > niche_variants.end() {
 899                         dataful_variant = None;
 900                     }
 901
 902                     if let Some(i) = dataful_variant {
 903                         let count = (niche_variants.end().as_u32()
 904                             - niche_variants.start().as_u32()
 905                             + 1) as u128;
 906                         // FIXME(#62691) use the largest niche across all fields,
 907                         // not just the first one.
 908                         for (field_index, &field) in variants[i].iter().enumerate() {
 909                             let niche = match &field.largest_niche {
 910                                 Some(niche) => niche,
 911                                 _ => continue,
 912                             };
 913                             let (niche_start, niche_scalar) = match niche.reserve(self, count) {
 914                                 Some(pair) => pair,
 915                                 None => continue,
 916                             };
 917
 918                             let mut align = dl.aggregate_align;
 919                             let st = variants
 920                                 .iter_enumerated()
 921                                 .map(|(j, v)| {
 922                                     let mut st = self.univariant_uninterned(
 923                                         ty,
 924                                         v,
 925                                         &def.repr,
 926                                         StructKind::AlwaysSized,
 927                                     )?;
 928                                     st.variants = Variants::Single { index: j };
 929
 930                                     align = align.max(st.align);
 931
 932                                     Ok(st)
 933                                 })
 934                                 .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
 935
 936                             let offset = st[i].fields.offset(field_index) + niche.offset;
 937                             let size = st[i].size;
 938
 939                             let mut abi = match st[i].abi {
 940                                 Abi::Scalar(_) => Abi::Scalar(niche_scalar.clone()),
 941                                 Abi::ScalarPair(ref first, ref second) => {
 942                                     // We need to use scalar_unit to reset the
 943                                     // valid range to the maximal one for that
 944                                     // primitive, because only the niche is
 945                                     // guaranteed to be initialised, not the
 946                                     // other primitive.
 947                                     if offset.bytes() == 0 {
 948                                         Abi::ScalarPair(
 949                                             niche_scalar.clone(),
 950                                             scalar_unit(second.value),
 951                                         )
 952                                     } else {
 953                                         Abi::ScalarPair(
 954                                             scalar_unit(first.value),
 955                                             niche_scalar.clone(),
 956                                         )
 957                                     }
 958                                 }
 959                                 _ => Abi::Aggregate { sized: true },
 960                             };
 961
 962                             if st.iter().all(|v| v.abi.is_uninhabited()) {
 963                                 abi = Abi::Uninhabited;
 964                             }
 965
 966                             let largest_niche =
 967                                 Niche::from_scalar(dl, offset, niche_scalar.clone());
 968
 969                             return Ok(tcx.intern_layout(LayoutDetails {
 970                                 variants: Variants::Multiple {
 971                                     discr: niche_scalar,
 972                                     discr_kind: DiscriminantKind::Niche {
 973                                         dataful_variant: i,
 974                                         niche_variants,
 975                                         niche_start,
 976                                     },
 977                                     discr_index: 0,
 978                                     variants: st,
 979                                 },
 980                                 fields: FieldPlacement::Arbitrary {
 981                                     offsets: vec![offset],
 982                                     memory_index: vec![0],
 983                                 },
 984                                 abi,
 985                                 largest_niche,
 986                                 size,
 987                                 align,
 988                             }));
 989                         }
 990                     }
 991                 }
 992
 993                 let (mut min, mut max) = (i128::max_value(), i128::min_value());
 994                 let discr_type = def.repr.discr_type();
 995                 let bits = Integer::from_attr(self, discr_type).size().bits();
 996                 for (i, discr) in def.discriminants(tcx) {
 997                     if variants[i].iter().any(|f| f.abi.is_uninhabited()) {
 998                         continue;
 999                     }
1000                     let mut x = discr.val as i128;
1001                     if discr_type.is_signed() {
1002                         // sign extend the raw representation to be an i128
1003                         x = (x << (128 - bits)) >> (128 - bits);
1004                     }
1005                     if x < min {
1006                         min = x;
1007                     }
1008                     if x > max {
1009                         max = x;
1010                     }
1011                 }
1012                 // We might have no inhabited variants, so pretend there's at least one.
1013                 if (min, max) == (i128::max_value(), i128::min_value()) {
1014                     min = 0;
1015                     max = 0;
1016                 }
1017                 assert!(min <= max, "discriminant range is {}...{}", min, max);
1018                 let (min_ity, signed) = Integer::repr_discr(tcx, ty, &def.repr, min, max);
1019
1020                 let mut align = dl.aggregate_align;
1021                 let mut size = Size::ZERO;
1022
1023                 // We're interested in the smallest alignment, so start large.
1024                 let mut start_align = Align::from_bytes(256).unwrap();
1025                 assert_eq!(Integer::for_align(dl, start_align), None);
1026
1027                 // repr(C) on an enum tells us to make a (tag, union) layout,
1028                 // so we need to grow the prefix alignment to be at least
1029                 // the alignment of the union. (This value is used both for
1030                 // determining the alignment of the overall enum, and the
1031                 // determining the alignment of the payload after the tag.)
1032                 let mut prefix_align = min_ity.align(dl).abi;
1033                 if def.repr.c() {
1034                     for fields in &variants {
1035                         for field in fields {
1036                             prefix_align = prefix_align.max(field.align.abi);
1037                         }
1038                     }
1039                 }
1040
1041                 // Create the set of structs that represent each variant.
1042                 let mut layout_variants = variants
1043                     .iter_enumerated()
1044                     .map(|(i, field_layouts)| {
1045                         let mut st = self.univariant_uninterned(
1046                             ty,
1047                             &field_layouts,
1048                             &def.repr,
1049                             StructKind::Prefixed(min_ity.size(), prefix_align),
1050                         )?;
1051                         st.variants = Variants::Single { index: i };
1052                         // Find the first field we can't move later
1053                         // to make room for a larger discriminant.
1054                         for field in
1055                             st.fields.index_by_increasing_offset().map(|j| field_layouts[j])
1056                         {
1057                             if !field.is_zst() || field.align.abi.bytes() != 1 {
1058                                 start_align = start_align.min(field.align.abi);
1059                                 break;
1060                             }
1061                         }
1062                         size = cmp::max(size, st.size);
1063                         align = align.max(st.align);
1064                         Ok(st)
1065                     })
1066                     .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
1067
1068                 // Align the maximum variant size to the largest alignment.
1069                 size = size.align_to(align.abi);
1070
1071                 if size.bytes() >= dl.obj_size_bound() {
1072                     return Err(LayoutError::SizeOverflow(ty));
1073                 }
1074
1075                 let typeck_ity = Integer::from_attr(dl, def.repr.discr_type());
1076                 if typeck_ity < min_ity {
1077                     // It is a bug if Layout decided on a greater discriminant size than typeck for
1078                     // some reason at this point (based on values discriminant can take on). Mostly
1079                     // because this discriminant will be loaded, and then stored into variable of
1080                     // type calculated by typeck. Consider such case (a bug): typeck decided on
1081                     // byte-sized discriminant, but layout thinks we need a 16-bit to store all
1082                     // discriminant values. That would be a bug, because then, in codegen, in order
1083                     // to store this 16-bit discriminant into 8-bit sized temporary some of the
1084                     // space necessary to represent would have to be discarded (or layout is wrong
1085                     // on thinking it needs 16 bits)
1086                     bug!(
1087                         "layout decided on a larger discriminant type ({:?}) than typeck ({:?})",
1088                         min_ity,
1089                         typeck_ity
1090                     );
1091                     // However, it is fine to make discr type however large (as an optimisation)
1092                     // after this point – we’ll just truncate the value we load in codegen.
1093                 }
1094
1095                 // Check to see if we should use a different type for the
1096                 // discriminant. We can safely use a type with the same size
1097                 // as the alignment of the first field of each variant.
1098                 // We increase the size of the discriminant to avoid LLVM copying
1099                 // padding when it doesn't need to. This normally causes unaligned
1100                 // load/stores and excessive memcpy/memset operations. By using a
1101                 // bigger integer size, LLVM can be sure about its contents and
1102                 // won't be so conservative.
1103
1104                 // Use the initial field alignment
1105                 let mut ity = if def.repr.c() || def.repr.int.is_some() {
1106                     min_ity
1107                 } else {
1108                     Integer::for_align(dl, start_align).unwrap_or(min_ity)
1109                 };
1110
1111                 // If the alignment is not larger than the chosen discriminant size,
1112                 // don't use the alignment as the final size.
1113                 if ity <= min_ity {
1114                     ity = min_ity;
1115                 } else {
1116                     // Patch up the variants' first few fields.
1117                     let old_ity_size = min_ity.size();
1118                     let new_ity_size = ity.size();
1119                     for variant in &mut layout_variants {
1120                         match variant.fields {
1121                             FieldPlacement::Arbitrary { ref mut offsets, .. } => {
1122                                 for i in offsets {
1123                                     if *i <= old_ity_size {
1124                                         assert_eq!(*i, old_ity_size);
1125                                         *i = new_ity_size;
1126                                     }
1127                                 }
1128                                 // We might be making the struct larger.
1129                                 if variant.size <= old_ity_size {
1130                                     variant.size = new_ity_size;
1131                                 }
1132                             }
1133                             _ => bug!(),
1134                         }
1135                     }
1136                 }
1137
1138                 let tag_mask = !0u128 >> (128 - ity.size().bits());
1139                 let tag = Scalar {
1140                     value: Int(ity, signed),
1141                     valid_range: (min as u128 & tag_mask)..=(max as u128 & tag_mask),
1142                 };
1143                 let mut abi = Abi::Aggregate { sized: true };
1144                 if tag.value.size(dl) == size {
1145                     abi = Abi::Scalar(tag.clone());
1146                 } else {
1147                     // Try to use a ScalarPair for all tagged enums.
1148                     let mut common_prim = None;
1149                     for (field_layouts, layout_variant) in variants.iter().zip(&layout_variants) {
1150                         let offsets = match layout_variant.fields {
1151                             FieldPlacement::Arbitrary { ref offsets, .. } => offsets,
1152                             _ => bug!(),
1153                         };
1154                         let mut fields =
1155                             field_layouts.iter().zip(offsets).filter(|p| !p.0.is_zst());
1156                         let (field, offset) = match (fields.next(), fields.next()) {
1157                             (None, None) => continue,
1158                             (Some(pair), None) => pair,
1159                             _ => {
1160                                 common_prim = None;
1161                                 break;
1162                             }
1163                         };
1164                         let prim = match field.details.abi {
1165                             Abi::Scalar(ref scalar) => scalar.value,
1166                             _ => {
1167                                 common_prim = None;
1168                                 break;
1169                             }
1170                         };
1171                         if let Some(pair) = common_prim {
1172                             // This is pretty conservative. We could go fancier
1173                             // by conflating things like i32 and u32, or even
1174                             // realising that (u8, u8) could just cohabit with
1175                             // u16 or even u32.
1176                             if pair != (prim, offset) {
1177                                 common_prim = None;
1178                                 break;
1179                             }
1180                         } else {
1181                             common_prim = Some((prim, offset));
1182                         }
1183                     }
1184                     if let Some((prim, offset)) = common_prim {
1185                         let pair = self.scalar_pair(tag.clone(), scalar_unit(prim));
1186                         let pair_offsets = match pair.fields {
1187                             FieldPlacement::Arbitrary { ref offsets, ref memory_index } => {
1188                                 assert_eq!(memory_index, &[0, 1]);
1189                                 offsets
1190                             }
1191                             _ => bug!(),
1192                         };
1193                         if pair_offsets[0] == Size::ZERO
1194                             && pair_offsets[1] == *offset
1195                             && align == pair.align
1196                             && size == pair.size
1197                         {
1198                             // We can use `ScalarPair` only when it matches our
1199                             // already computed layout (including `#[repr(C)]`).
1200                             abi = pair.abi;
1201                         }
1202                     }
1203                 }
1204
1205                 if layout_variants.iter().all(|v| v.abi.is_uninhabited()) {
1206                     abi = Abi::Uninhabited;
1207                 }
1208
1209                 let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag.clone());
1210
1211                 tcx.intern_layout(LayoutDetails {
1212                     variants: Variants::Multiple {
1213                         discr: tag,
1214                         discr_kind: DiscriminantKind::Tag,
1215                         discr_index: 0,
1216                         variants: layout_variants,
1217                     },
1218                     fields: FieldPlacement::Arbitrary {
1219                         offsets: vec![Size::ZERO],
1220                         memory_index: vec![0],
1221                     },
1222                     largest_niche,
1223                     abi,
1224                     align,
1225                     size,
1226                 })
1227             }
1228
1229             // Types with no meaningful known layout.
1230             ty::Projection(_) | ty::Opaque(..) => {
1231                 let normalized = tcx.normalize_erasing_regions(param_env, ty);
1232                 if ty == normalized {
1233                     return Err(LayoutError::Unknown(ty));
1234                 }
1235                 tcx.layout_raw(param_env.and(normalized))?
1236             }
1237
1238             ty::Bound(..)
1239             | ty::Placeholder(..)
1240             | ty::UnnormalizedProjection(..)
1241             | ty::GeneratorWitness(..)
1242             | ty::Infer(_) => bug!("LayoutDetails::compute: unexpected type `{}`", ty),
1243
1244             ty::Param(_) | ty::Error => {
1245                 return Err(LayoutError::Unknown(ty));
1246             }
1247         })
1248     }
1249 }
1250
1251 /// Overlap eligibility and variant assignment for each GeneratorSavedLocal.
1252 #[derive(Clone, Debug, PartialEq)]
1253 enum SavedLocalEligibility {
1254     Unassigned,
1255     Assigned(VariantIdx),
1256     // FIXME: Use newtype_index so we aren't wasting bytes
1257     Ineligible(Option<u32>),
1258 }
1259
1260 // When laying out generators, we divide our saved local fields into two
1261 // categories: overlap-eligible and overlap-ineligible.
1262 //
1263 // Those fields which are ineligible for overlap go in a "prefix" at the
1264 // beginning of the layout, and always have space reserved for them.
1265 //
1266 // Overlap-eligible fields are only assigned to one variant, so we lay
1267 // those fields out for each variant and put them right after the
1268 // prefix.
1269 //
1270 // Finally, in the layout details, we point to the fields from the
1271 // variants they are assigned to. It is possible for some fields to be
1272 // included in multiple variants. No field ever "moves around" in the
1273 // layout; its offset is always the same.
1274 //
1275 // Also included in the layout are the upvars and the discriminant.
1276 // These are included as fields on the "outer" layout; they are not part
1277 // of any variant.
1278 impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
1279     /// Compute the eligibility and assignment of each local.
1280     fn generator_saved_local_eligibility(
1281         &self,
1282         info: &GeneratorLayout<'tcx>,
1283     ) -> (BitSet<GeneratorSavedLocal>, IndexVec<GeneratorSavedLocal, SavedLocalEligibility>) {
1284         use SavedLocalEligibility::*;
1285
1286         let mut assignments: IndexVec<GeneratorSavedLocal, SavedLocalEligibility> =
1287             IndexVec::from_elem_n(Unassigned, info.field_tys.len());
1288
1289         // The saved locals not eligible for overlap. These will get
1290         // "promoted" to the prefix of our generator.
1291         let mut ineligible_locals = BitSet::new_empty(info.field_tys.len());
1292
1293         // Figure out which of our saved locals are fields in only
1294         // one variant. The rest are deemed ineligible for overlap.
1295         for (variant_index, fields) in info.variant_fields.iter_enumerated() {
1296             for local in fields {
1297                 match assignments[*local] {
1298                     Unassigned => {
1299                         assignments[*local] = Assigned(variant_index);
1300                     }
1301                     Assigned(idx) => {
1302                         // We've already seen this local at another suspension
1303                         // point, so it is no longer a candidate.
1304                         trace!(
1305                             "removing local {:?} in >1 variant ({:?}, {:?})",
1306                             local,
1307                             variant_index,
1308                             idx
1309                         );
1310                         ineligible_locals.insert(*local);
1311                         assignments[*local] = Ineligible(None);
1312                     }
1313                     Ineligible(_) => {}
1314                 }
1315             }
1316         }
1317
1318         // Next, check every pair of eligible locals to see if they
1319         // conflict.
1320         for local_a in info.storage_conflicts.rows() {
1321             let conflicts_a = info.storage_conflicts.count(local_a);
1322             if ineligible_locals.contains(local_a) {
1323                 continue;
1324             }
1325
1326             for local_b in info.storage_conflicts.iter(local_a) {
1327                 // local_a and local_b are storage live at the same time, therefore they
1328                 // cannot overlap in the generator layout. The only way to guarantee
1329                 // this is if they are in the same variant, or one is ineligible
1330                 // (which means it is stored in every variant).
1331                 if ineligible_locals.contains(local_b)
1332                     || assignments[local_a] == assignments[local_b]
1333                 {
1334                     continue;
1335                 }
1336
1337                 // If they conflict, we will choose one to make ineligible.
1338                 // This is not always optimal; it's just a greedy heuristic that
1339                 // seems to produce good results most of the time.
1340                 let conflicts_b = info.storage_conflicts.count(local_b);
1341                 let (remove, other) =
1342                     if conflicts_a > conflicts_b { (local_a, local_b) } else { (local_b, local_a) };
1343                 ineligible_locals.insert(remove);
1344                 assignments[remove] = Ineligible(None);
1345                 trace!("removing local {:?} due to conflict with {:?}", remove, other);
1346             }
1347         }
1348
1349         // Count the number of variants in use. If only one of them, then it is
1350         // impossible to overlap any locals in our layout. In this case it's
1351         // always better to make the remaining locals ineligible, so we can
1352         // lay them out with the other locals in the prefix and eliminate
1353         // unnecessary padding bytes.
1354         {
1355             let mut used_variants = BitSet::new_empty(info.variant_fields.len());
1356             for assignment in &assignments {
1357                 match assignment {
1358                     Assigned(idx) => {
1359                         used_variants.insert(*idx);
1360                     }
1361                     _ => {}
1362                 }
1363             }
1364             if used_variants.count() < 2 {
1365                 for assignment in assignments.iter_mut() {
1366                     *assignment = Ineligible(None);
1367                 }
1368                 ineligible_locals.insert_all();
1369             }
1370         }
1371
1372         // Write down the order of our locals that will be promoted to the prefix.
1373         {
1374             let mut idx = 0u32;
1375             for local in ineligible_locals.iter() {
1376                 assignments[local] = Ineligible(Some(idx));
1377                 idx += 1;
1378             }
1379         }
1380         debug!("generator saved local assignments: {:?}", assignments);
1381
1382         (ineligible_locals, assignments)
1383     }
1384
1385     /// Compute the full generator layout.
1386     fn generator_layout(
1387         &self,
1388         ty: Ty<'tcx>,
1389         def_id: hir::def_id::DefId,
1390         substs: SubstsRef<'tcx>,
1391     ) -> Result<&'tcx LayoutDetails, LayoutError<'tcx>> {
1392         use SavedLocalEligibility::*;
1393         let tcx = self.tcx;
1394
1395         let subst_field = |ty: Ty<'tcx>| ty.subst(tcx, substs);
1396
1397         let info = tcx.generator_layout(def_id);
1398         let (ineligible_locals, assignments) = self.generator_saved_local_eligibility(&info);
1399
1400         // Build a prefix layout, including "promoting" all ineligible
1401         // locals as part of the prefix. We compute the layout of all of
1402         // these fields at once to get optimal packing.
1403         let discr_index = substs.as_generator().prefix_tys(def_id, tcx).count();
1404         // FIXME(eddyb) set the correct vaidity range for the discriminant.
1405         let discr_layout = self.layout_of(substs.as_generator().discr_ty(tcx))?;
1406         let discr = match &discr_layout.abi {
1407             Abi::Scalar(s) => s.clone(),
1408             _ => bug!(),
1409         };
1410         let promoted_layouts = ineligible_locals
1411             .iter()
1412             .map(|local| subst_field(info.field_tys[local]))
1413             .map(|ty| tcx.mk_maybe_uninit(ty))
1414             .map(|ty| self.layout_of(ty));
1415         let prefix_layouts = substs
1416             .as_generator()
1417             .prefix_tys(def_id, tcx)
1418             .map(|ty| self.layout_of(ty))
1419             .chain(iter::once(Ok(discr_layout)))
1420             .chain(promoted_layouts)
1421             .collect::<Result<Vec<_>, _>>()?;
1422         let prefix = self.univariant_uninterned(
1423             ty,
1424             &prefix_layouts,
1425             &ReprOptions::default(),
1426             StructKind::AlwaysSized,
1427         )?;
1428
1429         let (prefix_size, prefix_align) = (prefix.size, prefix.align);
1430
1431         // Split the prefix layout into the "outer" fields (upvars and
1432         // discriminant) and the "promoted" fields. Promoted fields will
1433         // get included in each variant that requested them in
1434         // GeneratorLayout.
1435         debug!("prefix = {:#?}", prefix);
1436         let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields {
1437             FieldPlacement::Arbitrary { mut offsets, memory_index } => {
1438                 let mut inverse_memory_index = invert_mapping(&memory_index);
1439
1440                 // "a" (`0..b_start`) and "b" (`b_start..`) correspond to
1441                 // "outer" and "promoted" fields respectively.
1442                 let b_start = (discr_index + 1) as u32;
1443                 let offsets_b = offsets.split_off(b_start as usize);
1444                 let offsets_a = offsets;
1445
1446                 // Disentangle the "a" and "b" components of `inverse_memory_index`
1447                 // by preserving the order but keeping only one disjoint "half" each.
1448                 // FIXME(eddyb) build a better abstraction for permutations, if possible.
1449                 let inverse_memory_index_b: Vec<_> =
1450                     inverse_memory_index.iter().filter_map(|&i| i.checked_sub(b_start)).collect();
1451                 inverse_memory_index.retain(|&i| i < b_start);
1452                 let inverse_memory_index_a = inverse_memory_index;
1453
1454                 // Since `inverse_memory_index_{a,b}` each only refer to their
1455                 // respective fields, they can be safely inverted
1456                 let memory_index_a = invert_mapping(&inverse_memory_index_a);
1457                 let memory_index_b = invert_mapping(&inverse_memory_index_b);
1458
1459                 let outer_fields =
1460                     FieldPlacement::Arbitrary { offsets: offsets_a, memory_index: memory_index_a };
1461                 (outer_fields, offsets_b, memory_index_b)
1462             }
1463             _ => bug!(),
1464         };
1465
1466         let mut size = prefix.size;
1467         let mut align = prefix.align;
1468         let variants = info
1469             .variant_fields
1470             .iter_enumerated()
1471             .map(|(index, variant_fields)| {
1472                 // Only include overlap-eligible fields when we compute our variant layout.
1473                 let variant_only_tys = variant_fields
1474                     .iter()
1475                     .filter(|local| match assignments[**local] {
1476                         Unassigned => bug!(),
1477                         Assigned(v) if v == index => true,
1478                         Assigned(_) => bug!("assignment does not match variant"),
1479                         Ineligible(_) => false,
1480                     })
1481                     .map(|local| subst_field(info.field_tys[*local]));
1482
1483                 let mut variant = self.univariant_uninterned(
1484                     ty,
1485                     &variant_only_tys
1486                         .map(|ty| self.layout_of(ty))
1487                         .collect::<Result<Vec<_>, _>>()?,
1488                     &ReprOptions::default(),
1489                     StructKind::Prefixed(prefix_size, prefix_align.abi),
1490                 )?;
1491                 variant.variants = Variants::Single { index };
1492
1493                 let (offsets, memory_index) = match variant.fields {
1494                     FieldPlacement::Arbitrary { offsets, memory_index } => (offsets, memory_index),
1495                     _ => bug!(),
1496                 };
1497
1498                 // Now, stitch the promoted and variant-only fields back together in
1499                 // the order they are mentioned by our GeneratorLayout.
1500                 // Because we only use some subset (that can differ between variants)
1501                 // of the promoted fields, we can't just pick those elements of the
1502                 // `promoted_memory_index` (as we'd end up with gaps).
1503                 // So instead, we build an "inverse memory_index", as if all of the
1504                 // promoted fields were being used, but leave the elements not in the
1505                 // subset as `INVALID_FIELD_IDX`, which we can filter out later to
1506                 // obtain a valid (bijective) mapping.
1507                 const INVALID_FIELD_IDX: u32 = !0;
1508                 let mut combined_inverse_memory_index =
1509                     vec![INVALID_FIELD_IDX; promoted_memory_index.len() + memory_index.len()];
1510                 let mut offsets_and_memory_index = offsets.into_iter().zip(memory_index);
1511                 let combined_offsets = variant_fields
1512                     .iter()
1513                     .enumerate()
1514                     .map(|(i, local)| {
1515                         let (offset, memory_index) = match assignments[*local] {
1516                             Unassigned => bug!(),
1517                             Assigned(_) => {
1518                                 let (offset, memory_index) =
1519                                     offsets_and_memory_index.next().unwrap();
1520                                 (offset, promoted_memory_index.len() as u32 + memory_index)
1521                             }
1522                             Ineligible(field_idx) => {
1523                                 let field_idx = field_idx.unwrap() as usize;
1524                                 (promoted_offsets[field_idx], promoted_memory_index[field_idx])
1525                             }
1526                         };
1527                         combined_inverse_memory_index[memory_index as usize] = i as u32;
1528                         offset
1529                     })
1530                     .collect();
1531
1532                 // Remove the unused slots and invert the mapping to obtain the
1533                 // combined `memory_index` (also see previous comment).
1534                 combined_inverse_memory_index.retain(|&i| i != INVALID_FIELD_IDX);
1535                 let combined_memory_index = invert_mapping(&combined_inverse_memory_index);
1536
1537                 variant.fields = FieldPlacement::Arbitrary {
1538                     offsets: combined_offsets,
1539                     memory_index: combined_memory_index,
1540                 };
1541
1542                 size = size.max(variant.size);
1543                 align = align.max(variant.align);
1544                 Ok(variant)
1545             })
1546             .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
1547
1548         size = size.align_to(align.abi);
1549
1550         let abi = if prefix.abi.is_uninhabited() || variants.iter().all(|v| v.abi.is_uninhabited())
1551         {
1552             Abi::Uninhabited
1553         } else {
1554             Abi::Aggregate { sized: true }
1555         };
1556
1557         let layout = tcx.intern_layout(LayoutDetails {
1558             variants: Variants::Multiple {
1559                 discr,
1560                 discr_kind: DiscriminantKind::Tag,
1561                 discr_index,
1562                 variants,
1563             },
1564             fields: outer_fields,
1565             abi,
1566             largest_niche: prefix.largest_niche,
1567             size,
1568             align,
1569         });
1570         debug!("generator layout ({:?}): {:#?}", ty, layout);
1571         Ok(layout)
1572     }
1573
1574     /// This is invoked by the `layout_raw` query to record the final
1575     /// layout of each type.
1576     #[inline(always)]
1577     fn record_layout_for_printing(&self, layout: TyLayout<'tcx>) {
1578         // If we are running with `-Zprint-type-sizes`, maybe record layouts
1579         // for dumping later.
1580         if self.tcx.sess.opts.debugging_opts.print_type_sizes {
1581             self.record_layout_for_printing_outlined(layout)
1582         }
1583     }
1584
1585     fn record_layout_for_printing_outlined(&self, layout: TyLayout<'tcx>) {
1586         // Ignore layouts that are done with non-empty environments or
1587         // non-monomorphic layouts, as the user only wants to see the stuff
1588         // resulting from the final codegen session.
1589         if layout.ty.has_param_types() || !self.param_env.caller_bounds.is_empty() {
1590             return;
1591         }
1592
1593         // (delay format until we actually need it)
1594         let record = |kind, packed, opt_discr_size, variants| {
1595             let type_desc = format!("{:?}", layout.ty);
1596             self.tcx.sess.code_stats.record_type_size(
1597                 kind,
1598                 type_desc,
1599                 layout.align.abi,
1600                 layout.size,
1601                 packed,
1602                 opt_discr_size,
1603                 variants,
1604             );
1605         };
1606
1607         let adt_def = match layout.ty.kind {
1608             ty::Adt(ref adt_def, _) => {
1609                 debug!("print-type-size t: `{:?}` process adt", layout.ty);
1610                 adt_def
1611             }
1612
1613             ty::Closure(..) => {
1614                 debug!("print-type-size t: `{:?}` record closure", layout.ty);
1615                 record(DataTypeKind::Closure, false, None, vec![]);
1616                 return;
1617             }
1618
1619             _ => {
1620                 debug!("print-type-size t: `{:?}` skip non-nominal", layout.ty);
1621                 return;
1622             }
1623         };
1624
1625         let adt_kind = adt_def.adt_kind();
1626         let adt_packed = adt_def.repr.pack.is_some();
1627
1628         let build_variant_info = |n: Option<Ident>, flds: &[ast::Name], layout: TyLayout<'tcx>| {
1629             let mut min_size = Size::ZERO;
1630             let field_info: Vec<_> = flds
1631                 .iter()
1632                 .enumerate()
1633                 .map(|(i, &name)| match layout.field(self, i) {
1634                     Err(err) => {
1635                         bug!("no layout found for field {}: `{:?}`", name, err);
1636                     }
1637                     Ok(field_layout) => {
1638                         let offset = layout.fields.offset(i);
1639                         let field_end = offset + field_layout.size;
1640                         if min_size < field_end {
1641                             min_size = field_end;
1642                         }
1643                         session::FieldInfo {
1644                             name: name.to_string(),
1645                             offset: offset.bytes(),
1646                             size: field_layout.size.bytes(),
1647                             align: field_layout.align.abi.bytes(),
1648                         }
1649                     }
1650                 })
1651                 .collect();
1652
1653             session::VariantInfo {
1654                 name: n.map(|n| n.to_string()),
1655                 kind: if layout.is_unsized() {
1656                     session::SizeKind::Min
1657                 } else {
1658                     session::SizeKind::Exact
1659                 },
1660                 align: layout.align.abi.bytes(),
1661                 size: if min_size.bytes() == 0 { layout.size.bytes() } else { min_size.bytes() },
1662                 fields: field_info,
1663             }
1664         };
1665
1666         match layout.variants {
1667             Variants::Single { index } => {
1668                 debug!("print-type-size `{:#?}` variant {}", layout, adt_def.variants[index].ident);
1669                 if !adt_def.variants.is_empty() {
1670                     let variant_def = &adt_def.variants[index];
1671                     let fields: Vec<_> = variant_def.fields.iter().map(|f| f.ident.name).collect();
1672                     record(
1673                         adt_kind.into(),
1674                         adt_packed,
1675                         None,
1676                         vec![build_variant_info(Some(variant_def.ident), &fields, layout)],
1677                     );
1678                 } else {
1679                     // (This case arises for *empty* enums; so give it
1680                     // zero variants.)
1681                     record(adt_kind.into(), adt_packed, None, vec![]);
1682                 }
1683             }
1684
1685             Variants::Multiple { ref discr, ref discr_kind, .. } => {
1686                 debug!(
1687                     "print-type-size `{:#?}` adt general variants def {}",
1688                     layout.ty,
1689                     adt_def.variants.len()
1690                 );
1691                 let variant_infos: Vec<_> = adt_def
1692                     .variants
1693                     .iter_enumerated()
1694                     .map(|(i, variant_def)| {
1695                         let fields: Vec<_> =
1696                             variant_def.fields.iter().map(|f| f.ident.name).collect();
1697                         build_variant_info(
1698                             Some(variant_def.ident),
1699                             &fields,
1700                             layout.for_variant(self, i),
1701                         )
1702                     })
1703                     .collect();
1704                 record(
1705                     adt_kind.into(),
1706                     adt_packed,
1707                     match discr_kind {
1708                         DiscriminantKind::Tag => Some(discr.value.size(self)),
1709                         _ => None,
1710                     },
1711                     variant_infos,
1712                 );
1713             }
1714         }
1715     }
1716 }
1717
1718 /// Type size "skeleton", i.e., the only information determining a type's size.
1719 /// While this is conservative, (aside from constant sizes, only pointers,
1720 /// newtypes thereof and null pointer optimized enums are allowed), it is
1721 /// enough to statically check common use cases of transmute.
1722 #[derive(Copy, Clone, Debug)]
1723 pub enum SizeSkeleton<'tcx> {
1724     /// Any statically computable Layout.
1725     Known(Size),
1726
1727     /// A potentially-fat pointer.
1728     Pointer {
1729         /// If true, this pointer is never null.
1730         non_zero: bool,
1731         /// The type which determines the unsized metadata, if any,
1732         /// of this pointer. Either a type parameter or a projection
1733         /// depending on one, with regions erased.
1734         tail: Ty<'tcx>,
1735     },
1736 }
1737
1738 impl<'tcx> SizeSkeleton<'tcx> {
1739     pub fn compute(
1740         ty: Ty<'tcx>,
1741         tcx: TyCtxt<'tcx>,
1742         param_env: ty::ParamEnv<'tcx>,
1743     ) -> Result<SizeSkeleton<'tcx>, LayoutError<'tcx>> {
1744         debug_assert!(!ty.has_infer_types());
1745
1746         // First try computing a static layout.
1747         let err = match tcx.layout_of(param_env.and(ty)) {
1748             Ok(layout) => {
1749                 return Ok(SizeSkeleton::Known(layout.size));
1750             }
1751             Err(err) => err,
1752         };
1753
1754         match ty.kind {
1755             ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => {
1756                 let non_zero = !ty.is_unsafe_ptr();
1757                 let tail = tcx.struct_tail_erasing_lifetimes(pointee, param_env);
1758                 match tail.kind {
1759                     ty::Param(_) | ty::Projection(_) => {
1760                         debug_assert!(tail.has_param_types());
1761                         Ok(SizeSkeleton::Pointer { non_zero, tail: tcx.erase_regions(&tail) })
1762                     }
1763                     _ => bug!(
1764                         "SizeSkeleton::compute({}): layout errored ({}), yet \
1765                               tail `{}` is not a type parameter or a projection",
1766                         ty,
1767                         err,
1768                         tail
1769                     ),
1770                 }
1771             }
1772
1773             ty::Adt(def, substs) => {
1774                 // Only newtypes and enums w/ nullable pointer optimization.
1775                 if def.is_union() || def.variants.is_empty() || def.variants.len() > 2 {
1776                     return Err(err);
1777                 }
1778
1779                 // Get a zero-sized variant or a pointer newtype.
1780                 let zero_or_ptr_variant = |i| {
1781                     let i = VariantIdx::new(i);
1782                     let fields = def.variants[i]
1783                         .fields
1784                         .iter()
1785                         .map(|field| SizeSkeleton::compute(field.ty(tcx, substs), tcx, param_env));
1786                     let mut ptr = None;
1787                     for field in fields {
1788                         let field = field?;
1789                         match field {
1790                             SizeSkeleton::Known(size) => {
1791                                 if size.bytes() > 0 {
1792                                     return Err(err);
1793                                 }
1794                             }
1795                             SizeSkeleton::Pointer { .. } => {
1796                                 if ptr.is_some() {
1797                                     return Err(err);
1798                                 }
1799                                 ptr = Some(field);
1800                             }
1801                         }
1802                     }
1803                     Ok(ptr)
1804                 };
1805
1806                 let v0 = zero_or_ptr_variant(0)?;
1807                 // Newtype.
1808                 if def.variants.len() == 1 {
1809                     if let Some(SizeSkeleton::Pointer { non_zero, tail }) = v0 {
1810                         return Ok(SizeSkeleton::Pointer {
1811                             non_zero: non_zero
1812                                 || match tcx.layout_scalar_valid_range(def.did) {
1813                                     (Bound::Included(start), Bound::Unbounded) => start > 0,
1814                                     (Bound::Included(start), Bound::Included(end)) => {
1815                                         0 < start && start < end
1816                                     }
1817                                     _ => false,
1818                                 },
1819                             tail,
1820                         });
1821                     } else {
1822                         return Err(err);
1823                     }
1824                 }
1825
1826                 let v1 = zero_or_ptr_variant(1)?;
1827                 // Nullable pointer enum optimization.
1828                 match (v0, v1) {
1829                     (Some(SizeSkeleton::Pointer { non_zero: true, tail }), None)
1830                     | (None, Some(SizeSkeleton::Pointer { non_zero: true, tail })) => {
1831                         Ok(SizeSkeleton::Pointer { non_zero: false, tail })
1832                     }
1833                     _ => Err(err),
1834                 }
1835             }
1836
1837             ty::Projection(_) | ty::Opaque(..) => {
1838                 let normalized = tcx.normalize_erasing_regions(param_env, ty);
1839                 if ty == normalized {
1840                     Err(err)
1841                 } else {
1842                     SizeSkeleton::compute(normalized, tcx, param_env)
1843                 }
1844             }
1845
1846             _ => Err(err),
1847         }
1848     }
1849
1850     pub fn same_size(self, other: SizeSkeleton<'_>) -> bool {
1851         match (self, other) {
1852             (SizeSkeleton::Known(a), SizeSkeleton::Known(b)) => a == b,
1853             (SizeSkeleton::Pointer { tail: a, .. }, SizeSkeleton::Pointer { tail: b, .. }) => {
1854                 a == b
1855             }
1856             _ => false,
1857         }
1858     }
1859 }
1860
1861 pub trait HasTyCtxt<'tcx>: HasDataLayout {
1862     fn tcx(&self) -> TyCtxt<'tcx>;
1863 }
1864
1865 pub trait HasParamEnv<'tcx> {
1866     fn param_env(&self) -> ty::ParamEnv<'tcx>;
1867 }
1868
1869 impl<'tcx> HasDataLayout for TyCtxt<'tcx> {
1870     fn data_layout(&self) -> &TargetDataLayout {
1871         &self.data_layout
1872     }
1873 }
1874
1875 impl<'tcx> HasTyCtxt<'tcx> for TyCtxt<'tcx> {
1876     fn tcx(&self) -> TyCtxt<'tcx> {
1877         *self
1878     }
1879 }
1880
1881 impl<'tcx, C> HasParamEnv<'tcx> for LayoutCx<'tcx, C> {
1882     fn param_env(&self) -> ty::ParamEnv<'tcx> {
1883         self.param_env
1884     }
1885 }
1886
1887 impl<'tcx, T: HasDataLayout> HasDataLayout for LayoutCx<'tcx, T> {
1888     fn data_layout(&self) -> &TargetDataLayout {
1889         self.tcx.data_layout()
1890     }
1891 }
1892
1893 impl<'tcx, T: HasTyCtxt<'tcx>> HasTyCtxt<'tcx> for LayoutCx<'tcx, T> {
1894     fn tcx(&self) -> TyCtxt<'tcx> {
1895         self.tcx.tcx()
1896     }
1897 }
1898
1899 pub trait MaybeResult<T> {
1900     type Error;
1901
1902     fn from(x: Result<T, Self::Error>) -> Self;
1903     fn to_result(self) -> Result<T, Self::Error>;
1904 }
1905
1906 impl<T> MaybeResult<T> for T {
1907     type Error = !;
1908
1909     fn from(x: Result<T, Self::Error>) -> Self {
1910         let Ok(x) = x;
1911         x
1912     }
1913     fn to_result(self) -> Result<T, Self::Error> {
1914         Ok(self)
1915     }
1916 }
1917
1918 impl<T, E> MaybeResult<T> for Result<T, E> {
1919     type Error = E;
1920
1921     fn from(x: Result<T, Self::Error>) -> Self {
1922         x
1923     }
1924     fn to_result(self) -> Result<T, Self::Error> {
1925         self
1926     }
1927 }
1928
1929 pub type TyLayout<'tcx> = ::rustc_target::abi::TyLayout<'tcx, Ty<'tcx>>;
1930
1931 impl<'tcx> LayoutOf for LayoutCx<'tcx, TyCtxt<'tcx>> {
1932     type Ty = Ty<'tcx>;
1933     type TyLayout = Result<TyLayout<'tcx>, LayoutError<'tcx>>;
1934
1935     /// Computes the layout of a type. Note that this implicitly
1936     /// executes in "reveal all" mode.
1937     fn layout_of(&self, ty: Ty<'tcx>) -> Self::TyLayout {
1938         let param_env = self.param_env.with_reveal_all();
1939         let ty = self.tcx.normalize_erasing_regions(param_env, ty);
1940         let details = self.tcx.layout_raw(param_env.and(ty))?;
1941         let layout = TyLayout { ty, details };
1942
1943         // N.B., this recording is normally disabled; when enabled, it
1944         // can however trigger recursive invocations of `layout_of`.
1945         // Therefore, we execute it *after* the main query has
1946         // completed, to avoid problems around recursive structures
1947         // and the like. (Admittedly, I wasn't able to reproduce a problem
1948         // here, but it seems like the right thing to do. -nmatsakis)
1949         self.record_layout_for_printing(layout);
1950
1951         Ok(layout)
1952     }
1953 }
1954
1955 impl LayoutOf for LayoutCx<'tcx, ty::query::TyCtxtAt<'tcx>> {
1956     type Ty = Ty<'tcx>;
1957     type TyLayout = Result<TyLayout<'tcx>, LayoutError<'tcx>>;
1958
1959     /// Computes the layout of a type. Note that this implicitly
1960     /// executes in "reveal all" mode.
1961     fn layout_of(&self, ty: Ty<'tcx>) -> Self::TyLayout {
1962         let param_env = self.param_env.with_reveal_all();
1963         let ty = self.tcx.normalize_erasing_regions(param_env, ty);
1964         let details = self.tcx.layout_raw(param_env.and(ty))?;
1965         let layout = TyLayout { ty, details };
1966
1967         // N.B., this recording is normally disabled; when enabled, it
1968         // can however trigger recursive invocations of `layout_of`.
1969         // Therefore, we execute it *after* the main query has
1970         // completed, to avoid problems around recursive structures
1971         // and the like. (Admittedly, I wasn't able to reproduce a problem
1972         // here, but it seems like the right thing to do. -nmatsakis)
1973         let cx = LayoutCx { tcx: *self.tcx, param_env: self.param_env };
1974         cx.record_layout_for_printing(layout);
1975
1976         Ok(layout)
1977     }
1978 }
1979
1980 // Helper (inherent) `layout_of` methods to avoid pushing `LayoutCx` to users.
1981 impl TyCtxt<'tcx> {
1982     /// Computes the layout of a type. Note that this implicitly
1983     /// executes in "reveal all" mode.
1984     #[inline]
1985     pub fn layout_of(
1986         self,
1987         param_env_and_ty: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
1988     ) -> Result<TyLayout<'tcx>, LayoutError<'tcx>> {
1989         let cx = LayoutCx { tcx: self, param_env: param_env_and_ty.param_env };
1990         cx.layout_of(param_env_and_ty.value)
1991     }
1992 }
1993
1994 impl ty::query::TyCtxtAt<'tcx> {
1995     /// Computes the layout of a type. Note that this implicitly
1996     /// executes in "reveal all" mode.
1997     #[inline]
1998     pub fn layout_of(
1999         self,
2000         param_env_and_ty: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
2001     ) -> Result<TyLayout<'tcx>, LayoutError<'tcx>> {
2002         let cx = LayoutCx { tcx: self.at(self.span), param_env: param_env_and_ty.param_env };
2003         cx.layout_of(param_env_and_ty.value)
2004     }
2005 }
2006
2007 impl<'tcx, C> TyLayoutMethods<'tcx, C> for Ty<'tcx>
2008 where
2009     C: LayoutOf<Ty = Ty<'tcx>, TyLayout: MaybeResult<TyLayout<'tcx>>>
2010         + HasTyCtxt<'tcx>
2011         + HasParamEnv<'tcx>,
2012 {
2013     fn for_variant(this: TyLayout<'tcx>, cx: &C, variant_index: VariantIdx) -> TyLayout<'tcx> {
2014         let details = match this.variants {
2015             Variants::Single { index } if index == variant_index => this.details,
2016
2017             Variants::Single { index } => {
2018                 // Deny calling for_variant more than once for non-Single enums.
2019                 if let Ok(layout) = cx.layout_of(this.ty).to_result() {
2020                     assert_eq!(layout.variants, Variants::Single { index });
2021                 }
2022
2023                 let fields = match this.ty.kind {
2024                     ty::Adt(def, _) => def.variants[variant_index].fields.len(),
2025                     _ => bug!(),
2026                 };
2027                 let tcx = cx.tcx();
2028                 tcx.intern_layout(LayoutDetails {
2029                     variants: Variants::Single { index: variant_index },
2030                     fields: FieldPlacement::Union(fields),
2031                     abi: Abi::Uninhabited,
2032                     largest_niche: None,
2033                     align: tcx.data_layout.i8_align,
2034                     size: Size::ZERO,
2035                 })
2036             }
2037
2038             Variants::Multiple { ref variants, .. } => &variants[variant_index],
2039         };
2040
2041         assert_eq!(details.variants, Variants::Single { index: variant_index });
2042
2043         TyLayout { ty: this.ty, details }
2044     }
2045
2046     fn field(this: TyLayout<'tcx>, cx: &C, i: usize) -> C::TyLayout {
2047         let tcx = cx.tcx();
2048         let discr_layout = |discr: &Scalar| -> C::TyLayout {
2049             let layout = LayoutDetails::scalar(cx, discr.clone());
2050             MaybeResult::from(Ok(TyLayout {
2051                 details: tcx.intern_layout(layout),
2052                 ty: discr.value.to_ty(tcx),
2053             }))
2054         };
2055
2056         cx.layout_of(match this.ty.kind {
2057             ty::Bool
2058             | ty::Char
2059             | ty::Int(_)
2060             | ty::Uint(_)
2061             | ty::Float(_)
2062             | ty::FnPtr(_)
2063             | ty::Never
2064             | ty::FnDef(..)
2065             | ty::GeneratorWitness(..)
2066             | ty::Foreign(..)
2067             | ty::Dynamic(..) => bug!("TyLayout::field_type({:?}): not applicable", this),
2068
2069             // Potentially-fat pointers.
2070             ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => {
2071                 assert!(i < this.fields.count());
2072
2073                 // Reuse the fat `*T` type as its own thin pointer data field.
2074                 // This provides information about, e.g., DST struct pointees
2075                 // (which may have no non-DST form), and will work as long
2076                 // as the `Abi` or `FieldPlacement` is checked by users.
2077                 if i == 0 {
2078                     let nil = tcx.mk_unit();
2079                     let ptr_ty = if this.ty.is_unsafe_ptr() {
2080                         tcx.mk_mut_ptr(nil)
2081                     } else {
2082                         tcx.mk_mut_ref(tcx.lifetimes.re_static, nil)
2083                     };
2084                     return MaybeResult::from(cx.layout_of(ptr_ty).to_result().map(
2085                         |mut ptr_layout| {
2086                             ptr_layout.ty = this.ty;
2087                             ptr_layout
2088                         },
2089                     ));
2090                 }
2091
2092                 match tcx.struct_tail_erasing_lifetimes(pointee, cx.param_env()).kind {
2093                     ty::Slice(_) | ty::Str => tcx.types.usize,
2094                     ty::Dynamic(_, _) => {
2095                         tcx.mk_imm_ref(tcx.lifetimes.re_static, tcx.mk_array(tcx.types.usize, 3))
2096                         /* FIXME: use actual fn pointers
2097                         Warning: naively computing the number of entries in the
2098                         vtable by counting the methods on the trait + methods on
2099                         all parent traits does not work, because some methods can
2100                         be not object safe and thus excluded from the vtable.
2101                         Increase this counter if you tried to implement this but
2102                         failed to do it without duplicating a lot of code from
2103                         other places in the compiler: 2
2104                         tcx.mk_tup(&[
2105                             tcx.mk_array(tcx.types.usize, 3),
2106                             tcx.mk_array(Option<fn()>),
2107                         ])
2108                         */
2109                     }
2110                     _ => bug!("TyLayout::field_type({:?}): not applicable", this),
2111                 }
2112             }
2113
2114             // Arrays and slices.
2115             ty::Array(element, _) | ty::Slice(element) => element,
2116             ty::Str => tcx.types.u8,
2117
2118             // Tuples, generators and closures.
2119             ty::Closure(def_id, ref substs) => {
2120                 substs.as_closure().upvar_tys(def_id, tcx).nth(i).unwrap()
2121             }
2122
2123             ty::Generator(def_id, ref substs, _) => match this.variants {
2124                 Variants::Single { index } => substs
2125                     .as_generator()
2126                     .state_tys(def_id, tcx)
2127                     .nth(index.as_usize())
2128                     .unwrap()
2129                     .nth(i)
2130                     .unwrap(),
2131                 Variants::Multiple { ref discr, discr_index, .. } => {
2132                     if i == discr_index {
2133                         return discr_layout(discr);
2134                     }
2135                     substs.as_generator().prefix_tys(def_id, tcx).nth(i).unwrap()
2136                 }
2137             },
2138
2139             ty::Tuple(tys) => tys[i].expect_ty(),
2140
2141             // SIMD vector types.
2142             ty::Adt(def, ..) if def.repr.simd() => this.ty.simd_type(tcx),
2143
2144             // ADTs.
2145             ty::Adt(def, substs) => {
2146                 match this.variants {
2147                     Variants::Single { index } => def.variants[index].fields[i].ty(tcx, substs),
2148
2149                     // Discriminant field for enums (where applicable).
2150                     Variants::Multiple { ref discr, .. } => {
2151                         assert_eq!(i, 0);
2152                         return discr_layout(discr);
2153                     }
2154                 }
2155             }
2156
2157             ty::Projection(_)
2158             | ty::UnnormalizedProjection(..)
2159             | ty::Bound(..)
2160             | ty::Placeholder(..)
2161             | ty::Opaque(..)
2162             | ty::Param(_)
2163             | ty::Infer(_)
2164             | ty::Error => bug!("TyLayout::field_type: unexpected type `{}`", this.ty),
2165         })
2166     }
2167
2168     fn pointee_info_at(this: TyLayout<'tcx>, cx: &C, offset: Size) -> Option<PointeeInfo> {
2169         match this.ty.kind {
2170             ty::RawPtr(mt) if offset.bytes() == 0 => {
2171                 cx.layout_of(mt.ty).to_result().ok().map(|layout| PointeeInfo {
2172                     size: layout.size,
2173                     align: layout.align.abi,
2174                     safe: None,
2175                 })
2176             }
2177
2178             ty::Ref(_, ty, mt) if offset.bytes() == 0 => {
2179                 let tcx = cx.tcx();
2180                 let is_freeze = ty.is_freeze(tcx, cx.param_env(), DUMMY_SP);
2181                 let kind = match mt {
2182                     hir::Mutability::Not => {
2183                         if is_freeze {
2184                             PointerKind::Frozen
2185                         } else {
2186                             PointerKind::Shared
2187                         }
2188                     }
2189                     hir::Mutability::Mut => {
2190                         // Previously we would only emit noalias annotations for LLVM >= 6 or in
2191                         // panic=abort mode. That was deemed right, as prior versions had many bugs
2192                         // in conjunction with unwinding, but later versions didn’t seem to have
2193                         // said issues. See issue #31681.
2194                         //
2195                         // Alas, later on we encountered a case where noalias would generate wrong
2196                         // code altogether even with recent versions of LLVM in *safe* code with no
2197                         // unwinding involved. See #54462.
2198                         //
2199                         // For now, do not enable mutable_noalias by default at all, while the
2200                         // issue is being figured out.
2201                         let mutable_noalias =
2202                             tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(false);
2203                         if mutable_noalias {
2204                             PointerKind::UniqueBorrowed
2205                         } else {
2206                             PointerKind::Shared
2207                         }
2208                     }
2209                 };
2210
2211                 cx.layout_of(ty).to_result().ok().map(|layout| PointeeInfo {
2212                     size: layout.size,
2213                     align: layout.align.abi,
2214                     safe: Some(kind),
2215                 })
2216             }
2217
2218             _ => {
2219                 let mut data_variant = match this.variants {
2220                     // Within the discriminant field, only the niche itself is
2221                     // always initialized, so we only check for a pointer at its
2222                     // offset.
2223                     //
2224                     // If the niche is a pointer, it's either valid (according
2225                     // to its type), or null (which the niche field's scalar
2226                     // validity range encodes).  This allows using
2227                     // `dereferenceable_or_null` for e.g., `Option<&T>`, and
2228                     // this will continue to work as long as we don't start
2229                     // using more niches than just null (e.g., the first page of
2230                     // the address space, or unaligned pointers).
2231                     Variants::Multiple {
2232                         discr_kind: DiscriminantKind::Niche { dataful_variant, .. },
2233                         discr_index,
2234                         ..
2235                     } if this.fields.offset(discr_index) == offset => {
2236                         Some(this.for_variant(cx, dataful_variant))
2237                     }
2238                     _ => Some(this),
2239                 };
2240
2241                 if let Some(variant) = data_variant {
2242                     // We're not interested in any unions.
2243                     if let FieldPlacement::Union(_) = variant.fields {
2244                         data_variant = None;
2245                     }
2246                 }
2247
2248                 let mut result = None;
2249
2250                 if let Some(variant) = data_variant {
2251                     let ptr_end = offset + Pointer.size(cx);
2252                     for i in 0..variant.fields.count() {
2253                         let field_start = variant.fields.offset(i);
2254                         if field_start <= offset {
2255                             let field = variant.field(cx, i);
2256                             result = field.to_result().ok().and_then(|field| {
2257                                 if ptr_end <= field_start + field.size {
2258                                     // We found the right field, look inside it.
2259                                     field.pointee_info_at(cx, offset - field_start)
2260                                 } else {
2261                                     None
2262                                 }
2263                             });
2264                             if result.is_some() {
2265                                 break;
2266                             }
2267                         }
2268                     }
2269                 }
2270
2271                 // FIXME(eddyb) This should be for `ptr::Unique<T>`, not `Box<T>`.
2272                 if let Some(ref mut pointee) = result {
2273                     if let ty::Adt(def, _) = this.ty.kind {
2274                         if def.is_box() && offset.bytes() == 0 {
2275                             pointee.safe = Some(PointerKind::UniqueOwned);
2276                         }
2277                     }
2278                 }
2279
2280                 result
2281             }
2282         }
2283     }
2284 }
2285
2286 impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for LayoutError<'tcx> {
2287     fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
2288         use crate::ty::layout::LayoutError::*;
2289         mem::discriminant(self).hash_stable(hcx, hasher);
2290
2291         match *self {
2292             Unknown(t) | SizeOverflow(t) => t.hash_stable(hcx, hasher),
2293         }
2294     }
2295 }
2296
2297 impl<'tcx> ty::Instance<'tcx> {
2298     // NOTE(eddyb) this is private to avoid using it from outside of
2299     // `FnAbi::of_instance` - any other uses are either too high-level
2300     // for `Instance` (e.g. typeck would use `Ty::fn_sig` instead),
2301     // or should go through `FnAbi` instead, to avoid losing any
2302     // adjustments `FnAbi::of_instance` might be performing.
2303     fn fn_sig_for_fn_abi(&self, tcx: TyCtxt<'tcx>) -> ty::PolyFnSig<'tcx> {
2304         let ty = self.monomorphic_ty(tcx);
2305         match ty.kind {
2306             ty::FnDef(..) |
2307             // Shims currently have type FnPtr. Not sure this should remain.
2308             ty::FnPtr(_) => {
2309                 let mut sig = ty.fn_sig(tcx);
2310                 if let ty::InstanceDef::VtableShim(..) = self.def {
2311                     // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`.
2312                     sig = sig.map_bound(|mut sig| {
2313                         let mut inputs_and_output = sig.inputs_and_output.to_vec();
2314                         inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]);
2315                         sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
2316                         sig
2317                     });
2318                 }
2319                 sig
2320             }
2321             ty::Closure(def_id, substs) => {
2322                 let sig = substs.as_closure().sig(def_id, tcx);
2323
2324                 let env_ty = tcx.closure_env_ty(def_id, substs).unwrap();
2325                 sig.map_bound(|sig| tcx.mk_fn_sig(
2326                     iter::once(*env_ty.skip_binder()).chain(sig.inputs().iter().cloned()),
2327                     sig.output(),
2328                     sig.c_variadic,
2329                     sig.unsafety,
2330                     sig.abi
2331                 ))
2332             }
2333             ty::Generator(def_id, substs, _) => {
2334                 let sig = substs.as_generator().poly_sig(def_id, tcx);
2335
2336                 let env_region = ty::ReLateBound(ty::INNERMOST, ty::BrEnv);
2337                 let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty);
2338
2339                 let pin_did = tcx.lang_items().pin_type().unwrap();
2340                 let pin_adt_ref = tcx.adt_def(pin_did);
2341                 let pin_substs = tcx.intern_substs(&[env_ty.into()]);
2342                 let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs);
2343
2344                 sig.map_bound(|sig| {
2345                     let state_did = tcx.lang_items().gen_state().unwrap();
2346                     let state_adt_ref = tcx.adt_def(state_did);
2347                     let state_substs = tcx.intern_substs(&[
2348                         sig.yield_ty.into(),
2349                         sig.return_ty.into(),
2350                     ]);
2351                     let ret_ty = tcx.mk_adt(state_adt_ref, state_substs);
2352
2353                     tcx.mk_fn_sig(iter::once(env_ty),
2354                         ret_ty,
2355                         false,
2356                         hir::Unsafety::Normal,
2357                         rustc_target::spec::abi::Abi::Rust
2358                     )
2359                 })
2360             }
2361             _ => bug!("unexpected type {:?} in Instance::fn_sig", ty)
2362         }
2363     }
2364 }
2365
2366 pub trait FnAbiExt<'tcx, C>
2367 where
2368     C: LayoutOf<Ty = Ty<'tcx>, TyLayout = TyLayout<'tcx>>
2369         + HasDataLayout
2370         + HasTargetSpec
2371         + HasTyCtxt<'tcx>
2372         + HasParamEnv<'tcx>,
2373 {
2374     /// Compute a `FnAbi` suitable for indirect calls, i.e. to `fn` pointers.
2375     ///
2376     /// NB: this doesn't handle virtual calls - those should use `FnAbi::of_instance`
2377     /// instead, where the instance is a `InstanceDef::Virtual`.
2378     fn of_fn_ptr(cx: &C, sig: ty::PolyFnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self;
2379
2380     /// Compute a `FnAbi` suitable for declaring/defining an `fn` instance, and for
2381     /// direct calls to an `fn`.
2382     ///
2383     /// NB: that includes virtual calls, which are represented by "direct calls"
2384     /// to a `InstanceDef::Virtual` instance (of `<dyn Trait as Trait>::fn`).
2385     fn of_instance(cx: &C, instance: ty::Instance<'tcx>, extra_args: &[Ty<'tcx>]) -> Self;
2386
2387     fn new_internal(
2388         cx: &C,
2389         sig: ty::PolyFnSig<'tcx>,
2390         extra_args: &[Ty<'tcx>],
2391         caller_location: Option<Ty<'tcx>>,
2392         mk_arg_type: impl Fn(Ty<'tcx>, Option<usize>) -> ArgAbi<'tcx, Ty<'tcx>>,
2393     ) -> Self;
2394     fn adjust_for_abi(&mut self, cx: &C, abi: SpecAbi);
2395 }
2396
2397 impl<'tcx, C> FnAbiExt<'tcx, C> for call::FnAbi<'tcx, Ty<'tcx>>
2398 where
2399     C: LayoutOf<Ty = Ty<'tcx>, TyLayout = TyLayout<'tcx>>
2400         + HasDataLayout
2401         + HasTargetSpec
2402         + HasTyCtxt<'tcx>
2403         + HasParamEnv<'tcx>,
2404 {
2405     fn of_fn_ptr(cx: &C, sig: ty::PolyFnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self {
2406         call::FnAbi::new_internal(cx, sig, extra_args, None, |ty, _| ArgAbi::new(cx.layout_of(ty)))
2407     }
2408
2409     fn of_instance(cx: &C, instance: ty::Instance<'tcx>, extra_args: &[Ty<'tcx>]) -> Self {
2410         let sig = instance.fn_sig_for_fn_abi(cx.tcx());
2411
2412         let caller_location = if instance.def.requires_caller_location(cx.tcx()) {
2413             Some(cx.tcx().caller_location_ty())
2414         } else {
2415             None
2416         };
2417
2418         call::FnAbi::new_internal(cx, sig, extra_args, caller_location, |ty, arg_idx| {
2419             let mut layout = cx.layout_of(ty);
2420             // Don't pass the vtable, it's not an argument of the virtual fn.
2421             // Instead, pass just the data pointer, but give it the type `*const/mut dyn Trait`
2422             // or `&/&mut dyn Trait` because this is special-cased elsewhere in codegen
2423             if let (ty::InstanceDef::Virtual(..), Some(0)) = (&instance.def, arg_idx) {
2424                 let fat_pointer_ty = if layout.is_unsized() {
2425                     // unsized `self` is passed as a pointer to `self`
2426                     // FIXME (mikeyhew) change this to use &own if it is ever added to the language
2427                     cx.tcx().mk_mut_ptr(layout.ty)
2428                 } else {
2429                     match layout.abi {
2430                         Abi::ScalarPair(..) => (),
2431                         _ => bug!("receiver type has unsupported layout: {:?}", layout),
2432                     }
2433
2434                     // In the case of Rc<Self>, we need to explicitly pass a *mut RcBox<Self>
2435                     // with a Scalar (not ScalarPair) ABI. This is a hack that is understood
2436                     // elsewhere in the compiler as a method on a `dyn Trait`.
2437                     // To get the type `*mut RcBox<Self>`, we just keep unwrapping newtypes until we
2438                     // get a built-in pointer type
2439                     let mut fat_pointer_layout = layout;
2440                     'descend_newtypes: while !fat_pointer_layout.ty.is_unsafe_ptr()
2441                         && !fat_pointer_layout.ty.is_region_ptr()
2442                     {
2443                         for i in 0..fat_pointer_layout.fields.count() {
2444                             let field_layout = fat_pointer_layout.field(cx, i);
2445
2446                             if !field_layout.is_zst() {
2447                                 fat_pointer_layout = field_layout;
2448                                 continue 'descend_newtypes;
2449                             }
2450                         }
2451
2452                         bug!("receiver has no non-zero-sized fields {:?}", fat_pointer_layout);
2453                     }
2454
2455                     fat_pointer_layout.ty
2456                 };
2457
2458                 // we now have a type like `*mut RcBox<dyn Trait>`
2459                 // change its layout to that of `*mut ()`, a thin pointer, but keep the same type
2460                 // this is understood as a special case elsewhere in the compiler
2461                 let unit_pointer_ty = cx.tcx().mk_mut_ptr(cx.tcx().mk_unit());
2462                 layout = cx.layout_of(unit_pointer_ty);
2463                 layout.ty = fat_pointer_ty;
2464             }
2465             ArgAbi::new(layout)
2466         })
2467     }
2468
2469     fn new_internal(
2470         cx: &C,
2471         sig: ty::PolyFnSig<'tcx>,
2472         extra_args: &[Ty<'tcx>],
2473         caller_location: Option<Ty<'tcx>>,
2474         mk_arg_type: impl Fn(Ty<'tcx>, Option<usize>) -> ArgAbi<'tcx, Ty<'tcx>>,
2475     ) -> Self {
2476         debug!("FnAbi::new_internal({:?}, {:?})", sig, extra_args);
2477
2478         let sig = cx.tcx().normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig);
2479
2480         use rustc_target::spec::abi::Abi::*;
2481         let conv = match cx.tcx().sess.target.target.adjust_abi(sig.abi) {
2482             RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust,
2483
2484             // It's the ABI's job to select this, not ours.
2485             System => bug!("system abi should be selected elsewhere"),
2486             EfiApi => bug!("eficall abi should be selected elsewhere"),
2487
2488             Stdcall => Conv::X86Stdcall,
2489             Fastcall => Conv::X86Fastcall,
2490             Vectorcall => Conv::X86VectorCall,
2491             Thiscall => Conv::X86ThisCall,
2492             C => Conv::C,
2493             Unadjusted => Conv::C,
2494             Win64 => Conv::X86_64Win64,
2495             SysV64 => Conv::X86_64SysV,
2496             Aapcs => Conv::ArmAapcs,
2497             PtxKernel => Conv::PtxKernel,
2498             Msp430Interrupt => Conv::Msp430Intr,
2499             X86Interrupt => Conv::X86Intr,
2500             AmdGpuKernel => Conv::AmdGpuKernel,
2501
2502             // These API constants ought to be more specific...
2503             Cdecl => Conv::C,
2504         };
2505
2506         let mut inputs = sig.inputs();
2507         let extra_args = if sig.abi == RustCall {
2508             assert!(!sig.c_variadic && extra_args.is_empty());
2509
2510             if let Some(input) = sig.inputs().last() {
2511                 if let ty::Tuple(tupled_arguments) = input.kind {
2512                     inputs = &sig.inputs()[0..sig.inputs().len() - 1];
2513                     tupled_arguments.iter().map(|k| k.expect_ty()).collect()
2514                 } else {
2515                     bug!(
2516                         "argument to function with \"rust-call\" ABI \
2517                             is not a tuple"
2518                     );
2519                 }
2520             } else {
2521                 bug!(
2522                     "argument to function with \"rust-call\" ABI \
2523                         is not a tuple"
2524                 );
2525             }
2526         } else {
2527             assert!(sig.c_variadic || extra_args.is_empty());
2528             extra_args.to_vec()
2529         };
2530
2531         let target = &cx.tcx().sess.target.target;
2532         let win_x64_gnu =
2533             target.target_os == "windows" && target.arch == "x86_64" && target.target_env == "gnu";
2534         let linux_s390x =
2535             target.target_os == "linux" && target.arch == "s390x" && target.target_env == "gnu";
2536         let linux_sparc64 =
2537             target.target_os == "linux" && target.arch == "sparc64" && target.target_env == "gnu";
2538         let rust_abi = match sig.abi {
2539             RustIntrinsic | PlatformIntrinsic | Rust | RustCall => true,
2540             _ => false,
2541         };
2542
2543         // Handle safe Rust thin and fat pointers.
2544         let adjust_for_rust_scalar = |attrs: &mut ArgAttributes,
2545                                       scalar: &Scalar,
2546                                       layout: TyLayout<'tcx>,
2547                                       offset: Size,
2548                                       is_return: bool| {
2549             // Booleans are always an i1 that needs to be zero-extended.
2550             if scalar.is_bool() {
2551                 attrs.set(ArgAttribute::ZExt);
2552                 return;
2553             }
2554
2555             // Only pointer types handled below.
2556             if scalar.value != Pointer {
2557                 return;
2558             }
2559
2560             if scalar.valid_range.start() < scalar.valid_range.end() {
2561                 if *scalar.valid_range.start() > 0 {
2562                     attrs.set(ArgAttribute::NonNull);
2563                 }
2564             }
2565
2566             if let Some(pointee) = layout.pointee_info_at(cx, offset) {
2567                 if let Some(kind) = pointee.safe {
2568                     attrs.pointee_align = Some(pointee.align);
2569
2570                     // `Box` (`UniqueBorrowed`) are not necessarily dereferencable
2571                     // for the entire duration of the function as they can be deallocated
2572                     // any time. Set their valid size to 0.
2573                     attrs.pointee_size = match kind {
2574                         PointerKind::UniqueOwned => Size::ZERO,
2575                         _ => pointee.size,
2576                     };
2577
2578                     // `Box` pointer parameters never alias because ownership is transferred
2579                     // `&mut` pointer parameters never alias other parameters,
2580                     // or mutable global data
2581                     //
2582                     // `&T` where `T` contains no `UnsafeCell<U>` is immutable,
2583                     // and can be marked as both `readonly` and `noalias`, as
2584                     // LLVM's definition of `noalias` is based solely on memory
2585                     // dependencies rather than pointer equality
2586                     let no_alias = match kind {
2587                         PointerKind::Shared => false,
2588                         PointerKind::UniqueOwned => true,
2589                         PointerKind::Frozen | PointerKind::UniqueBorrowed => !is_return,
2590                     };
2591                     if no_alias {
2592                         attrs.set(ArgAttribute::NoAlias);
2593                     }
2594
2595                     if kind == PointerKind::Frozen && !is_return {
2596                         attrs.set(ArgAttribute::ReadOnly);
2597                     }
2598                 }
2599             }
2600         };
2601
2602         let arg_of = |ty: Ty<'tcx>, arg_idx: Option<usize>| {
2603             let is_return = arg_idx.is_none();
2604             let mut arg = mk_arg_type(ty, arg_idx);
2605             if arg.layout.is_zst() {
2606                 // For some forsaken reason, x86_64-pc-windows-gnu
2607                 // doesn't ignore zero-sized struct arguments.
2608                 // The same is true for s390x-unknown-linux-gnu
2609                 // and sparc64-unknown-linux-gnu.
2610                 if is_return || rust_abi || (!win_x64_gnu && !linux_s390x && !linux_sparc64) {
2611                     arg.mode = PassMode::Ignore;
2612                 }
2613             }
2614
2615             // FIXME(eddyb) other ABIs don't have logic for scalar pairs.
2616             if !is_return && rust_abi {
2617                 if let Abi::ScalarPair(ref a, ref b) = arg.layout.abi {
2618                     let mut a_attrs = ArgAttributes::new();
2619                     let mut b_attrs = ArgAttributes::new();
2620                     adjust_for_rust_scalar(&mut a_attrs, a, arg.layout, Size::ZERO, false);
2621                     adjust_for_rust_scalar(
2622                         &mut b_attrs,
2623                         b,
2624                         arg.layout,
2625                         a.value.size(cx).align_to(b.value.align(cx).abi),
2626                         false,
2627                     );
2628                     arg.mode = PassMode::Pair(a_attrs, b_attrs);
2629                     return arg;
2630                 }
2631             }
2632
2633             if let Abi::Scalar(ref scalar) = arg.layout.abi {
2634                 if let PassMode::Direct(ref mut attrs) = arg.mode {
2635                     adjust_for_rust_scalar(attrs, scalar, arg.layout, Size::ZERO, is_return);
2636                 }
2637             }
2638
2639             arg
2640         };
2641
2642         let mut fn_abi = FnAbi {
2643             ret: arg_of(sig.output(), None),
2644             args: inputs
2645                 .iter()
2646                 .cloned()
2647                 .chain(extra_args)
2648                 .chain(caller_location)
2649                 .enumerate()
2650                 .map(|(i, ty)| arg_of(ty, Some(i)))
2651                 .collect(),
2652             c_variadic: sig.c_variadic,
2653             conv,
2654         };
2655         fn_abi.adjust_for_abi(cx, sig.abi);
2656         fn_abi
2657     }
2658
2659     fn adjust_for_abi(&mut self, cx: &C, abi: SpecAbi) {
2660         if abi == SpecAbi::Unadjusted {
2661             return;
2662         }
2663
2664         if abi == SpecAbi::Rust
2665             || abi == SpecAbi::RustCall
2666             || abi == SpecAbi::RustIntrinsic
2667             || abi == SpecAbi::PlatformIntrinsic
2668         {
2669             let fixup = |arg: &mut ArgAbi<'tcx, Ty<'tcx>>| {
2670                 if arg.is_ignore() {
2671                     return;
2672                 }
2673
2674                 match arg.layout.abi {
2675                     Abi::Aggregate { .. } => {}
2676
2677                     // This is a fun case! The gist of what this is doing is
2678                     // that we want callers and callees to always agree on the
2679                     // ABI of how they pass SIMD arguments. If we were to *not*
2680                     // make these arguments indirect then they'd be immediates
2681                     // in LLVM, which means that they'd used whatever the
2682                     // appropriate ABI is for the callee and the caller. That
2683                     // means, for example, if the caller doesn't have AVX
2684                     // enabled but the callee does, then passing an AVX argument
2685                     // across this boundary would cause corrupt data to show up.
2686                     //
2687                     // This problem is fixed by unconditionally passing SIMD
2688                     // arguments through memory between callers and callees
2689                     // which should get them all to agree on ABI regardless of
2690                     // target feature sets. Some more information about this
2691                     // issue can be found in #44367.
2692                     //
2693                     // Note that the platform intrinsic ABI is exempt here as
2694                     // that's how we connect up to LLVM and it's unstable
2695                     // anyway, we control all calls to it in libstd.
2696                     Abi::Vector { .. }
2697                         if abi != SpecAbi::PlatformIntrinsic
2698                             && cx.tcx().sess.target.target.options.simd_types_indirect =>
2699                     {
2700                         arg.make_indirect();
2701                         return;
2702                     }
2703
2704                     _ => return,
2705                 }
2706
2707                 let size = arg.layout.size;
2708                 if arg.layout.is_unsized() || size > Pointer.size(cx) {
2709                     arg.make_indirect();
2710                 } else {
2711                     // We want to pass small aggregates as immediates, but using
2712                     // a LLVM aggregate type for this leads to bad optimizations,
2713                     // so we pick an appropriately sized integer type instead.
2714                     arg.cast_to(Reg { kind: RegKind::Integer, size });
2715                 }
2716             };
2717             fixup(&mut self.ret);
2718             for arg in &mut self.args {
2719                 fixup(arg);
2720             }
2721             if let PassMode::Indirect(ref mut attrs, _) = self.ret.mode {
2722                 attrs.set(ArgAttribute::StructRet);
2723             }
2724             return;
2725         }
2726
2727         if let Err(msg) = self.adjust_for_cabi(cx, abi) {
2728             cx.tcx().sess.fatal(&msg);
2729         }
2730     }
2731 }