1 use crate::abi::{self, Abi, Align, FieldsShape, Size};
2 use crate::abi::{HasDataLayout, LayoutOf, TyAbiInterface, TyAndLayout};
3 use crate::spec::{self, HasTargetSpec};
27 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
29 /// Ignore the argument.
31 /// The argument is either uninhabited or a ZST.
33 /// Pass the argument directly.
35 /// The argument has a layout abi of `Scalar`, `Vector` or in rare cases `Aggregate`.
36 Direct(ArgAttributes),
37 /// Pass a pair's elements directly in two arguments.
39 /// The argument has a layout abi of `ScalarPair`.
40 Pair(ArgAttributes, ArgAttributes),
41 /// Pass the argument after casting it, to either
42 /// a single uniform or a pair of registers.
44 /// Pass the argument indirectly via a hidden pointer.
45 /// The `extra_attrs` value, if any, is for the extra data (vtable or length)
46 /// which indicates that it refers to an unsized rvalue.
47 /// `on_stack` defines that the the value should be passed at a fixed
48 /// stack offset in accordance to the ABI rather than passed using a
49 /// pointer. This corresponds to the `byval` LLVM argument attribute.
50 Indirect { attrs: ArgAttributes, extra_attrs: Option<ArgAttributes>, on_stack: bool },
53 // Hack to disable non_upper_case_globals only for the bitflags! and not for the rest
55 pub use attr_impl::ArgAttribute;
57 #[allow(non_upper_case_globals)]
60 // The subset of llvm::Attribute needed for arguments, packed into a bitfield.
63 pub struct ArgAttribute: u16 {
64 const NoAlias = 1 << 1;
65 const NoCapture = 1 << 2;
66 const NonNull = 1 << 3;
67 const ReadOnly = 1 << 4;
69 // NoAlias on &mut arguments can only be used with LLVM >= 12 due to miscompiles
70 // in earlier versions. FIXME: Remove this distinction once possible.
71 const NoAliasMutRef = 1 << 6;
76 /// Sometimes an ABI requires small integers to be extended to a full or partial register. This enum
77 /// defines if this extension should be zero-extension or sign-extension when necessary. When it is
78 /// not necessary to extend the argument, this enum is ignored.
79 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
80 pub enum ArgExtension {
86 /// A compact representation of LLVM attributes (at least those relevant for this module)
87 /// that can be manipulated without interacting with LLVM's Attribute machinery.
88 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
89 pub struct ArgAttributes {
90 pub regular: ArgAttribute,
91 pub arg_ext: ArgExtension,
92 /// The minimum size of the pointee, guaranteed to be valid for the duration of the whole call
93 /// (corresponding to LLVM's dereferenceable and dereferenceable_or_null attributes).
94 pub pointee_size: Size,
95 pub pointee_align: Option<Align>,
99 pub fn new() -> Self {
101 regular: ArgAttribute::default(),
102 arg_ext: ArgExtension::None,
103 pointee_size: Size::ZERO,
108 pub fn ext(&mut self, ext: ArgExtension) -> &mut Self {
110 self.arg_ext == ArgExtension::None || self.arg_ext == ext,
111 "cannot set {:?} when {:?} is already set",
119 pub fn set(&mut self, attr: ArgAttribute) -> &mut Self {
120 self.regular |= attr;
124 pub fn contains(&self, attr: ArgAttribute) -> bool {
125 self.regular.contains(attr)
129 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
136 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
142 macro_rules! reg_ctor {
143 ($name:ident, $kind:ident, $bits:expr) => {
144 pub fn $name() -> Reg {
145 Reg { kind: RegKind::$kind, size: Size::from_bits($bits) }
151 reg_ctor!(i8, Integer, 8);
152 reg_ctor!(i16, Integer, 16);
153 reg_ctor!(i32, Integer, 32);
154 reg_ctor!(i64, Integer, 64);
155 reg_ctor!(i128, Integer, 128);
157 reg_ctor!(f32, Float, 32);
158 reg_ctor!(f64, Float, 64);
162 pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align {
163 let dl = cx.data_layout();
165 RegKind::Integer => match self.size.bits() {
166 1 => dl.i1_align.abi,
167 2..=8 => dl.i8_align.abi,
168 9..=16 => dl.i16_align.abi,
169 17..=32 => dl.i32_align.abi,
170 33..=64 => dl.i64_align.abi,
171 65..=128 => dl.i128_align.abi,
172 _ => panic!("unsupported integer: {:?}", self),
174 RegKind::Float => match self.size.bits() {
175 32 => dl.f32_align.abi,
176 64 => dl.f64_align.abi,
177 _ => panic!("unsupported float: {:?}", self),
179 RegKind::Vector => dl.vector_align(self.size).abi,
184 /// An argument passed entirely registers with the
185 /// same kind (e.g., HFA / HVA on PPC64 and AArch64).
186 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
190 /// The total size of the argument, which can be:
191 /// * equal to `unit.size` (one scalar/vector),
192 /// * a multiple of `unit.size` (an array of scalar/vectors),
193 /// * if `unit.kind` is `Integer`, the last element
194 /// can be shorter, i.e., `{ i64, i64, i32 }` for
195 /// 64-bit integers with a total size of 20 bytes.
199 impl From<Reg> for Uniform {
200 fn from(unit: Reg) -> Uniform {
201 Uniform { unit, total: unit.size }
206 pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align {
211 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
212 pub struct CastTarget {
213 pub prefix: [Option<RegKind>; 8],
214 pub prefix_chunk_size: Size,
218 impl From<Reg> for CastTarget {
219 fn from(unit: Reg) -> CastTarget {
220 CastTarget::from(Uniform::from(unit))
224 impl From<Uniform> for CastTarget {
225 fn from(uniform: Uniform) -> CastTarget {
226 CastTarget { prefix: [None; 8], prefix_chunk_size: Size::ZERO, rest: uniform }
231 pub fn pair(a: Reg, b: Reg) -> CastTarget {
233 prefix: [Some(a.kind), None, None, None, None, None, None, None],
234 prefix_chunk_size: a.size,
235 rest: Uniform::from(b),
239 pub fn size<C: HasDataLayout>(&self, cx: &C) -> Size {
240 (self.prefix_chunk_size * self.prefix.iter().filter(|x| x.is_some()).count() as u64)
241 .align_to(self.rest.align(cx))
245 pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align {
248 .filter_map(|x| x.map(|kind| Reg { kind, size: self.prefix_chunk_size }.align(cx)))
249 .fold(cx.data_layout().aggregate_align.abi.max(self.rest.align(cx)), |acc, align| {
255 /// Return value from the `homogeneous_aggregate` test function.
256 #[derive(Copy, Clone, Debug)]
257 pub enum HomogeneousAggregate {
258 /// Yes, all the "leaf fields" of this struct are passed in the
259 /// same way (specified in the `Reg` value).
262 /// There are no leaf fields at all.
266 /// Error from the `homogeneous_aggregate` test function, indicating
267 /// there are distinct leaf fields passed in different ways,
268 /// or this is uninhabited.
269 #[derive(Copy, Clone, Debug)]
270 pub struct Heterogeneous;
272 impl HomogeneousAggregate {
273 /// If this is a homogeneous aggregate, returns the homogeneous
274 /// unit, else `None`.
275 pub fn unit(self) -> Option<Reg> {
277 HomogeneousAggregate::Homogeneous(reg) => Some(reg),
278 HomogeneousAggregate::NoData => None,
282 /// Try to combine two `HomogeneousAggregate`s, e.g. from two fields in
283 /// the same `struct`. Only succeeds if only one of them has any data,
284 /// or both units are identical.
285 fn merge(self, other: HomogeneousAggregate) -> Result<HomogeneousAggregate, Heterogeneous> {
286 match (self, other) {
287 (x, HomogeneousAggregate::NoData) | (HomogeneousAggregate::NoData, x) => Ok(x),
289 (HomogeneousAggregate::Homogeneous(a), HomogeneousAggregate::Homogeneous(b)) => {
291 return Err(Heterogeneous);
299 impl<'a, Ty> TyAndLayout<'a, Ty> {
300 fn is_aggregate(&self) -> bool {
302 Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } => false,
303 Abi::ScalarPair(..) | Abi::Aggregate { .. } => true,
307 /// Returns `Homogeneous` if this layout is an aggregate containing fields of
308 /// only a single type (e.g., `(u32, u32)`). Such aggregates are often
309 /// special-cased in ABIs.
311 /// Note: We generally ignore fields of zero-sized type when computing
312 /// this value (see #56877).
314 /// This is public so that it can be used in unit tests, but
315 /// should generally only be relevant to the ABI details of
316 /// specific targets.
317 pub fn homogeneous_aggregate<C>(&self, cx: &C) -> Result<HomogeneousAggregate, Heterogeneous>
319 Ty: TyAbiInterface<'a, C> + Copy,
320 C: LayoutOf<'a, Ty = Ty, TyAndLayout = Self>,
323 Abi::Uninhabited => Err(Heterogeneous),
325 // The primitive for this algorithm.
326 Abi::Scalar(ref scalar) => {
327 let kind = match scalar.value {
328 abi::Int(..) | abi::Pointer => RegKind::Integer,
329 abi::F32 | abi::F64 => RegKind::Float,
331 Ok(HomogeneousAggregate::Homogeneous(Reg { kind, size: self.size }))
334 Abi::Vector { .. } => {
335 assert!(!self.is_zst());
336 Ok(HomogeneousAggregate::Homogeneous(Reg {
337 kind: RegKind::Vector,
342 Abi::ScalarPair(..) | Abi::Aggregate { .. } => {
343 // Helper for computing `homogeneous_aggregate`, allowing a custom
344 // starting offset (used below for handling variants).
348 -> Result<(HomogeneousAggregate, Size), Heterogeneous> {
349 let is_union = match layout.fields {
350 FieldsShape::Primitive => {
351 unreachable!("aggregates can't have `FieldsShape::Primitive`")
353 FieldsShape::Array { count, .. } => {
354 assert_eq!(start, Size::ZERO);
356 let result = if count > 0 {
357 layout.field(cx, 0).homogeneous_aggregate(cx)?
359 HomogeneousAggregate::NoData
361 return Ok((result, layout.size));
363 FieldsShape::Union(_) => true,
364 FieldsShape::Arbitrary { .. } => false,
367 let mut result = HomogeneousAggregate::NoData;
368 let mut total = start;
370 for i in 0..layout.fields.count() {
371 if !is_union && total != layout.fields.offset(i) {
372 return Err(Heterogeneous);
375 let field = layout.field(cx, i);
377 result = result.merge(field.homogeneous_aggregate(cx)?)?;
379 // Keep track of the offset (without padding).
380 let size = field.size;
382 total = total.max(size);
391 let (mut result, mut total) = from_fields_at(*self, Size::ZERO)?;
393 match &self.variants {
394 abi::Variants::Single { .. } => {}
395 abi::Variants::Multiple { variants, .. } => {
396 // Treat enum variants like union members.
397 // HACK(eddyb) pretend the `enum` field (discriminant)
398 // is at the start of every variant (otherwise the gap
399 // at the start of all variants would disqualify them).
401 // NB: for all tagged `enum`s (which include all non-C-like
402 // `enum`s with defined FFI representation), this will
403 // match the homogeneous computation on the equivalent
404 // `struct { tag; union { variant1; ... } }` and/or
405 // `union { struct { tag; variant1; } ... }`
406 // (the offsets of variant fields should be identical
407 // between the two for either to be a homogeneous aggregate).
408 let variant_start = total;
409 for variant_idx in variants.indices() {
410 let (variant_result, variant_total) =
411 from_fields_at(self.for_variant(cx, variant_idx), variant_start)?;
413 result = result.merge(variant_result)?;
414 total = total.max(variant_total);
419 // There needs to be no padding.
420 if total != self.size {
424 HomogeneousAggregate::Homogeneous(_) => {
425 assert_ne!(total, Size::ZERO);
427 HomogeneousAggregate::NoData => {
428 assert_eq!(total, Size::ZERO);
438 /// Information about how to pass an argument to,
439 /// or return a value from, a function, under some ABI.
441 pub struct ArgAbi<'a, Ty> {
442 pub layout: TyAndLayout<'a, Ty>,
444 /// Dummy argument, which is emitted before the real argument.
445 pub pad: Option<Reg>,
450 impl<'a, Ty> ArgAbi<'a, Ty> {
452 cx: &impl HasDataLayout,
453 layout: TyAndLayout<'a, Ty>,
454 scalar_attrs: impl Fn(&TyAndLayout<'a, Ty>, &abi::Scalar, Size) -> ArgAttributes,
456 let mode = match &layout.abi {
457 Abi::Uninhabited => PassMode::Ignore,
458 Abi::Scalar(scalar) => PassMode::Direct(scalar_attrs(&layout, scalar, Size::ZERO)),
459 Abi::ScalarPair(a, b) => PassMode::Pair(
460 scalar_attrs(&layout, a, Size::ZERO),
461 scalar_attrs(&layout, b, a.value.size(cx).align_to(b.value.align(cx).abi)),
463 Abi::Vector { .. } => PassMode::Direct(ArgAttributes::new()),
464 Abi::Aggregate { .. } => PassMode::Direct(ArgAttributes::new()),
466 ArgAbi { layout, pad: None, mode }
469 fn indirect_pass_mode(layout: &TyAndLayout<'a, Ty>) -> PassMode {
470 let mut attrs = ArgAttributes::new();
472 // For non-immediate arguments the callee gets its own copy of
473 // the value on the stack, so there are no aliases. It's also
474 // program-invisible so can't possibly capture
475 attrs.set(ArgAttribute::NoAlias).set(ArgAttribute::NoCapture).set(ArgAttribute::NonNull);
476 attrs.pointee_size = layout.size;
477 // FIXME(eddyb) We should be doing this, but at least on
478 // i686-pc-windows-msvc, it results in wrong stack offsets.
479 // attrs.pointee_align = Some(layout.align.abi);
481 let extra_attrs = layout.is_unsized().then_some(ArgAttributes::new());
483 PassMode::Indirect { attrs, extra_attrs, on_stack: false }
486 pub fn make_indirect(&mut self) {
488 PassMode::Direct(_) | PassMode::Pair(_, _) => {}
489 PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: false } => return,
490 _ => panic!("Tried to make {:?} indirect", self.mode),
493 self.mode = Self::indirect_pass_mode(&self.layout);
496 pub fn make_indirect_byval(&mut self) {
497 self.make_indirect();
499 PassMode::Indirect { attrs: _, extra_attrs: _, ref mut on_stack } => {
506 pub fn extend_integer_width_to(&mut self, bits: u64) {
507 // Only integers have signedness
508 if let Abi::Scalar(ref scalar) = self.layout.abi {
509 if let abi::Int(i, signed) = scalar.value {
510 if i.size().bits() < bits {
511 if let PassMode::Direct(ref mut attrs) = self.mode {
513 attrs.ext(ArgExtension::Sext)
515 attrs.ext(ArgExtension::Zext)
523 pub fn cast_to<T: Into<CastTarget>>(&mut self, target: T) {
524 self.mode = PassMode::Cast(target.into());
527 pub fn pad_with(&mut self, reg: Reg) {
528 self.pad = Some(reg);
531 pub fn is_indirect(&self) -> bool {
532 matches!(self.mode, PassMode::Indirect { .. })
535 pub fn is_sized_indirect(&self) -> bool {
536 matches!(self.mode, PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ })
539 pub fn is_unsized_indirect(&self) -> bool {
540 matches!(self.mode, PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ })
543 pub fn is_ignore(&self) -> bool {
544 matches!(self.mode, PassMode::Ignore)
548 #[derive(Copy, Clone, PartialEq, Debug)]
550 // General language calling conventions, for which every target
551 // should have its own backend (e.g. LLVM) support.
555 // Target-specific calling conventions.
574 AvrNonBlockingInterrupt,
577 /// Metadata describing how the arguments to a native function
578 /// should be passed in order to respect the native ABI.
580 /// I will do my best to describe this structure, but these
581 /// comments are reverse-engineered and may be inaccurate. -NDM
583 pub struct FnAbi<'a, Ty> {
584 /// The LLVM types of each argument.
585 pub args: Vec<ArgAbi<'a, Ty>>,
587 /// LLVM return type.
588 pub ret: ArgAbi<'a, Ty>,
590 pub c_variadic: bool,
592 /// The count of non-variadic arguments.
594 /// Should only be different from args.len() when c_variadic is true.
595 /// This can be used to know whether an argument is variadic or not.
596 pub fixed_count: usize,
600 pub can_unwind: bool,
603 impl<'a, Ty> FnAbi<'a, Ty> {
604 pub fn adjust_for_cabi<C>(&mut self, cx: &C, abi: spec::abi::Abi) -> Result<(), String>
606 Ty: TyAbiInterface<'a, C> + Copy,
607 C: LayoutOf<'a, Ty = Ty, TyAndLayout = TyAndLayout<'a, Ty>> + HasDataLayout + HasTargetSpec,
609 if abi == spec::abi::Abi::X86Interrupt {
610 if let Some(arg) = self.args.first_mut() {
611 arg.make_indirect_byval();
616 match &cx.target_spec().arch[..] {
618 let flavor = if abi == spec::abi::Abi::Fastcall {
619 x86::Flavor::Fastcall
623 x86::compute_abi_info(cx, self, flavor);
626 if abi == spec::abi::Abi::SysV64 {
627 x86_64::compute_abi_info(cx, self);
628 } else if abi == spec::abi::Abi::Win64 || cx.target_spec().is_like_windows {
629 x86_win64::compute_abi_info(self);
631 x86_64::compute_abi_info(cx, self);
634 "aarch64" => aarch64::compute_abi_info(cx, self),
635 "amdgpu" => amdgpu::compute_abi_info(cx, self),
636 "arm" => arm::compute_abi_info(cx, self),
637 "avr" => avr::compute_abi_info(self),
638 "mips" => mips::compute_abi_info(cx, self),
639 "mips64" => mips64::compute_abi_info(cx, self),
640 "powerpc" => powerpc::compute_abi_info(self),
641 "powerpc64" => powerpc64::compute_abi_info(cx, self),
642 "s390x" => s390x::compute_abi_info(cx, self),
643 "msp430" => msp430::compute_abi_info(self),
644 "sparc" => sparc::compute_abi_info(cx, self),
645 "sparc64" => sparc64::compute_abi_info(cx, self),
646 "nvptx" => nvptx::compute_abi_info(self),
647 "nvptx64" => nvptx64::compute_abi_info(self),
648 "hexagon" => hexagon::compute_abi_info(self),
649 "riscv32" | "riscv64" => riscv::compute_abi_info(cx, self),
650 "wasm32" | "wasm64" => {
651 if cx.target_spec().adjust_abi(abi) == spec::abi::Abi::Wasm {
652 wasm::compute_wasm_abi_info(self)
654 wasm::compute_c_abi_info(cx, self)
657 "asmjs" => wasm::compute_c_abi_info(cx, self),
658 "bpf" => bpf::compute_abi_info(self),
659 a => return Err(format!("unrecognized arch \"{}\" in target specification", a)),