1 use crate::builder::Builder;
2 use crate::type_::Type;
3 use crate::type_of::LayoutLlvmExt;
4 use crate::value::Value;
5 use rustc_codegen_ssa::mir::operand::OperandRef;
6 use rustc_codegen_ssa::{
8 traits::{BaseTypeMethods, BuilderMethods, ConstMethods, DerivedTypeMethods},
10 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
11 use rustc_middle::ty::Ty;
12 use rustc_target::abi::{Align, Endian, HasDataLayout, Size};
14 fn round_pointer_up_to_alignment<'ll>(
15 bx: &mut Builder<'_, 'll, '_>,
20 let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize());
21 ptr_as_int = bx.add(ptr_as_int, bx.cx().const_i32(align.bytes() as i32 - 1));
22 ptr_as_int = bx.and(ptr_as_int, bx.cx().const_i32(-(align.bytes() as i32)));
23 bx.inttoptr(ptr_as_int, ptr_ty)
26 fn emit_direct_ptr_va_arg<'ll, 'tcx>(
27 bx: &mut Builder<'_, 'll, 'tcx>,
28 list: OperandRef<'tcx, &'ll Value>,
33 allow_higher_align: bool,
34 ) -> (&'ll Value, Align) {
35 let va_list_ty = bx.type_i8p();
36 let va_list_ptr_ty = bx.type_ptr_to(va_list_ty);
37 let va_list_addr = if list.layout.llvm_type(bx.cx) != va_list_ptr_ty {
38 bx.bitcast(list.immediate(), va_list_ptr_ty)
43 let ptr = bx.load(va_list_ty, va_list_addr, bx.tcx().data_layout.pointer_align.abi);
45 let (addr, addr_align) = if allow_higher_align && align > slot_size {
46 (round_pointer_up_to_alignment(bx, ptr, align, bx.cx().type_i8p()), align)
51 let aligned_size = size.align_to(slot_size).bytes() as i32;
52 let full_direct_size = bx.cx().const_i32(aligned_size);
53 let next = bx.inbounds_gep(bx.type_i8(), addr, &[full_direct_size]);
54 bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi);
56 if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big {
57 let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32);
58 let adjusted = bx.inbounds_gep(bx.type_i8(), addr, &[adjusted_size]);
59 (bx.bitcast(adjusted, bx.cx().type_ptr_to(llty)), addr_align)
61 (bx.bitcast(addr, bx.cx().type_ptr_to(llty)), addr_align)
65 fn emit_ptr_va_arg<'ll, 'tcx>(
66 bx: &mut Builder<'_, 'll, 'tcx>,
67 list: OperandRef<'tcx, &'ll Value>,
71 allow_higher_align: bool,
73 let layout = bx.cx.layout_of(target_ty);
74 let (llty, size, align) = if indirect {
76 bx.cx.layout_of(bx.cx.tcx.mk_imm_ptr(target_ty)).llvm_type(bx.cx),
77 bx.cx.data_layout().pointer_size,
78 bx.cx.data_layout().pointer_align,
81 (layout.llvm_type(bx.cx), layout.size, layout.align)
83 let (addr, addr_align) =
84 emit_direct_ptr_va_arg(bx, list, llty, size, align.abi, slot_size, allow_higher_align);
86 let tmp_ret = bx.load(llty, addr, addr_align);
87 bx.load(bx.cx.layout_of(target_ty).llvm_type(bx.cx), tmp_ret, align.abi)
89 bx.load(llty, addr, addr_align)
93 fn emit_aapcs_va_arg<'ll, 'tcx>(
94 bx: &mut Builder<'_, 'll, 'tcx>,
95 list: OperandRef<'tcx, &'ll Value>,
98 // Implementation of the AAPCS64 calling convention for va_args see
99 // https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst
100 let va_list_addr = list.immediate();
101 let va_list_layout = list.deref(bx.cx).layout;
102 let va_list_ty = va_list_layout.llvm_type(bx);
103 let layout = bx.cx.layout_of(target_ty);
105 let maybe_reg = bx.append_sibling_block("va_arg.maybe_reg");
106 let in_reg = bx.append_sibling_block("va_arg.in_reg");
107 let on_stack = bx.append_sibling_block("va_arg.on_stack");
108 let end = bx.append_sibling_block("va_arg.end");
109 let zero = bx.const_i32(0);
110 let offset_align = Align::from_bytes(4).unwrap();
112 let gr_type = target_ty.is_any_ptr() || target_ty.is_integral();
113 let (reg_off, reg_top_index, slot_size) = if gr_type {
115 bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3));
116 let nreg = (layout.size.bytes() + 7) / 8;
117 (gr_offs, va_list_layout.llvm_field_index(bx.cx, 1), nreg * 8)
120 bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 4));
121 let nreg = (layout.size.bytes() + 15) / 16;
122 (vr_off, va_list_layout.llvm_field_index(bx.cx, 2), nreg * 16)
125 // if the offset >= 0 then the value will be on the stack
126 let mut reg_off_v = bx.load(bx.type_i32(), reg_off, offset_align);
127 let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero);
128 bx.cond_br(use_stack, on_stack, maybe_reg);
130 // The value at this point might be in a register, but there is a chance that
131 // it could be on the stack so we have to update the offset and then check
134 bx.switch_to_block(maybe_reg);
135 if gr_type && layout.align.abi.bytes() > 8 {
136 reg_off_v = bx.add(reg_off_v, bx.const_i32(15));
137 reg_off_v = bx.and(reg_off_v, bx.const_i32(-16));
139 let new_reg_off_v = bx.add(reg_off_v, bx.const_i32(slot_size as i32));
141 bx.store(new_reg_off_v, reg_off, offset_align);
143 // Check to see if we have overflowed the registers as a result of this.
144 // If we have then we need to use the stack for this value
145 let use_stack = bx.icmp(IntPredicate::IntSGT, new_reg_off_v, zero);
146 bx.cond_br(use_stack, on_stack, in_reg);
148 bx.switch_to_block(in_reg);
149 let top_type = bx.type_i8p();
150 let top = bx.struct_gep(va_list_ty, va_list_addr, reg_top_index);
151 let top = bx.load(top_type, top, bx.tcx().data_layout.pointer_align.abi);
153 // reg_value = *(@top + reg_off_v);
154 let mut reg_addr = bx.gep(bx.type_i8(), top, &[reg_off_v]);
155 if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size {
156 // On big-endian systems the value is right-aligned in its slot.
157 let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32);
158 reg_addr = bx.gep(bx.type_i8(), reg_addr, &[offset]);
160 let reg_type = layout.llvm_type(bx);
161 let reg_addr = bx.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type));
162 let reg_value = bx.load(reg_type, reg_addr, layout.align.abi);
166 bx.switch_to_block(on_stack);
168 emit_ptr_va_arg(bx, list, target_ty, false, Align::from_bytes(8).unwrap(), true);
171 bx.switch_to_block(end);
173 bx.phi(layout.immediate_llvm_type(bx), &[reg_value, stack_value], &[in_reg, on_stack]);
178 fn emit_s390x_va_arg<'ll, 'tcx>(
179 bx: &mut Builder<'_, 'll, 'tcx>,
180 list: OperandRef<'tcx, &'ll Value>,
183 // Implementation of the s390x ELF ABI calling convention for va_args see
184 // https://github.com/IBM/s390x-abi (chapter 1.2.4)
185 let va_list_addr = list.immediate();
186 let va_list_layout = list.deref(bx.cx).layout;
187 let va_list_ty = va_list_layout.llvm_type(bx);
188 let layout = bx.cx.layout_of(target_ty);
190 let in_reg = bx.append_sibling_block("va_arg.in_reg");
191 let in_mem = bx.append_sibling_block("va_arg.in_mem");
192 let end = bx.append_sibling_block("va_arg.end");
194 // FIXME: vector ABI not yet supported.
195 let target_ty_size = bx.cx.size_of(target_ty).bytes();
196 let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two();
197 let unpadded_size = if indirect { 8 } else { target_ty_size };
199 let padding = padded_size - unpadded_size;
201 let gpr_type = indirect || !layout.is_single_fp_element(bx.cx);
202 let (max_regs, reg_count_field, reg_save_index, reg_padding) =
203 if gpr_type { (5, 0, 2, padding) } else { (4, 1, 16, 0) };
205 // Check whether the value was passed in a register or in memory.
206 let reg_count = bx.struct_gep(
209 va_list_layout.llvm_field_index(bx.cx, reg_count_field),
211 let reg_count_v = bx.load(bx.type_i64(), reg_count, Align::from_bytes(8).unwrap());
212 let use_regs = bx.icmp(IntPredicate::IntULT, reg_count_v, bx.const_u64(max_regs));
213 bx.cond_br(use_regs, in_reg, in_mem);
215 // Emit code to load the value if it was passed in a register.
216 bx.switch_to_block(in_reg);
218 // Work out the address of the value in the register save area.
220 bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3));
221 let reg_ptr_v = bx.load(bx.type_i8p(), reg_ptr, bx.tcx().data_layout.pointer_align.abi);
222 let scaled_reg_count = bx.mul(reg_count_v, bx.const_u64(8));
223 let reg_off = bx.add(scaled_reg_count, bx.const_u64(reg_save_index * 8 + reg_padding));
224 let reg_addr = bx.gep(bx.type_i8(), reg_ptr_v, &[reg_off]);
226 // Update the register count.
227 let new_reg_count_v = bx.add(reg_count_v, bx.const_u64(1));
228 bx.store(new_reg_count_v, reg_count, Align::from_bytes(8).unwrap());
231 // Emit code to load the value if it was passed in memory.
232 bx.switch_to_block(in_mem);
234 // Work out the address of the value in the argument overflow area.
236 bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 2));
237 let arg_ptr_v = bx.load(bx.type_i8p(), arg_ptr, bx.tcx().data_layout.pointer_align.abi);
238 let arg_off = bx.const_u64(padding);
239 let mem_addr = bx.gep(bx.type_i8(), arg_ptr_v, &[arg_off]);
241 // Update the argument overflow area pointer.
242 let arg_size = bx.cx().const_u64(padded_size);
243 let new_arg_ptr_v = bx.inbounds_gep(bx.type_i8(), arg_ptr_v, &[arg_size]);
244 bx.store(new_arg_ptr_v, arg_ptr, bx.tcx().data_layout.pointer_align.abi);
247 // Return the appropriate result.
248 bx.switch_to_block(end);
249 let val_addr = bx.phi(bx.type_i8p(), &[reg_addr, mem_addr], &[in_reg, in_mem]);
250 let val_type = layout.llvm_type(bx);
251 let val_addr = if indirect {
252 let ptr_type = bx.cx.type_ptr_to(val_type);
253 let ptr_addr = bx.bitcast(val_addr, bx.cx.type_ptr_to(ptr_type));
254 bx.load(ptr_type, ptr_addr, bx.tcx().data_layout.pointer_align.abi)
256 bx.bitcast(val_addr, bx.cx.type_ptr_to(val_type))
258 bx.load(val_type, val_addr, layout.align.abi)
261 pub(super) fn emit_va_arg<'ll, 'tcx>(
262 bx: &mut Builder<'_, 'll, 'tcx>,
263 addr: OperandRef<'tcx, &'ll Value>,
266 // Determine the va_arg implementation to use. The LLVM va_arg instruction
267 // is lacking in some instances, so we should only use it as a fallback.
268 let target = &bx.cx.tcx.sess.target;
269 let arch = &bx.cx.tcx.sess.target.arch;
272 "x86" if target.is_like_windows => {
273 emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), false)
276 "x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true),
278 "aarch64" if target.is_like_windows => {
279 emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false)
281 // macOS / iOS AArch64
282 "aarch64" if target.is_like_osx => {
283 emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true)
285 "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty),
286 "s390x" => emit_s390x_va_arg(bx, addr, target_ty),
288 "x86_64" if target.is_like_windows => {
289 let target_ty_size = bx.cx.size_of(target_ty).bytes();
290 let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two();
291 emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false)
293 // For all other architecture/OS combinations fall back to using
294 // the LLVM va_arg instruction.
295 // https://llvm.org/docs/LangRef.html#va-arg-instruction
296 _ => bx.va_arg(addr.immediate(), bx.cx.layout_of(target_ty).llvm_type(bx.cx)),