3 use rustc_errors::ErrorReported;
5 use rustc_middle::mir::interpret::ErrorHandled;
6 use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt, TyAndLayout};
7 use rustc_middle::ty::{self, Instance, Ty, TypeFoldable};
8 use rustc_target::abi::call::{FnAbi, PassMode};
12 use rustc_index::bit_set::BitSet;
13 use rustc_index::vec::IndexVec;
15 use self::analyze::CleanupKind;
16 use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo};
17 use self::place::PlaceRef;
18 use rustc_middle::mir::traversal;
20 use self::operand::{OperandRef, OperandValue};
22 /// Master context for codegenning from MIR.
23 pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
24 instance: Instance<'tcx>,
26 mir: &'tcx mir::Body<'tcx>,
28 debug_context: Option<FunctionDebugContext<Bx::DIScope>>,
32 cx: &'a Bx::CodegenCx,
34 fn_abi: FnAbi<'tcx, Ty<'tcx>>,
36 /// When unwinding is initiated, we have to store this personality
37 /// value somewhere so that we can load it and re-use it in the
38 /// resume instruction. The personality is (afaik) some kind of
39 /// value used for C++ unwinding, which must filter by type: we
40 /// don't really care about it very much. Anyway, this value
41 /// contains an alloca into which the personality is stored and
42 /// then later loaded when generating the DIVERGE_BLOCK.
43 personality_slot: Option<PlaceRef<'tcx, Bx::Value>>,
45 /// A `Block` for each MIR `BasicBlock`
46 blocks: IndexVec<mir::BasicBlock, Bx::BasicBlock>,
48 /// The funclet status of each basic block
49 cleanup_kinds: IndexVec<mir::BasicBlock, analyze::CleanupKind>,
51 /// When targeting MSVC, this stores the cleanup info for each funclet
52 /// BB. This is initialized as we compute the funclets' head block in RPO.
53 funclets: IndexVec<mir::BasicBlock, Option<Bx::Funclet>>,
55 /// This stores the landing-pad block for a given BB, computed lazily on GNU
56 /// and eagerly on MSVC.
57 landing_pads: IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>>,
59 /// Cached unreachable block
60 unreachable_block: Option<Bx::BasicBlock>,
62 /// The location where each MIR arg/var/tmp/ret is stored. This is
63 /// usually an `PlaceRef` representing an alloca, but not always:
64 /// sometimes we can skip the alloca and just store the value
65 /// directly using an `OperandRef`, which makes for tighter LLVM
66 /// IR. The conditions for using an `OperandRef` are as follows:
68 /// - the type of the local must be judged "immediate" by `is_llvm_immediate`
69 /// - the operand must never be referenced indirectly
70 /// - we should not take its address using the `&` operator
71 /// - nor should it appear in a place path like `tmp.a`
72 /// - the operand must be defined by an rvalue that can generate immediate
75 /// Avoiding allocs can also be important for certain intrinsics,
77 locals: IndexVec<mir::Local, LocalRef<'tcx, Bx::Value>>,
79 /// All `VarDebugInfo` from the MIR body, partitioned by `Local`.
80 /// This is `None` if no var`#[non_exhaustive]`iable debuginfo/names are needed.
81 per_local_var_debug_info:
82 Option<IndexVec<mir::Local, Vec<PerLocalVarDebugInfo<'tcx, Bx::DIVariable>>>>,
84 /// Caller location propagated if this function has `#[track_caller]`.
85 caller_location: Option<OperandRef<'tcx, Bx::Value>>,
88 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
89 pub fn monomorphize<T>(&self, value: &T) -> T
91 T: Copy + TypeFoldable<'tcx>,
93 debug!("monomorphize: self.instance={:?}", self.instance);
94 if let Some(substs) = self.instance.substs_for_mir_body() {
95 self.cx.tcx().subst_and_normalize_erasing_regions(
97 ty::ParamEnv::reveal_all(),
101 self.cx.tcx().normalize_erasing_regions(ty::ParamEnv::reveal_all(), *value)
106 enum LocalRef<'tcx, V> {
107 Place(PlaceRef<'tcx, V>),
108 /// `UnsizedPlace(p)`: `p` itself is a thin pointer (indirect place).
109 /// `*p` is the fat pointer that references the actual unsized place.
110 /// Every time it is initialized, we have to reallocate the place
111 /// and update the fat pointer. That's the reason why it is indirect.
112 UnsizedPlace(PlaceRef<'tcx, V>),
113 Operand(Option<OperandRef<'tcx, V>>),
116 impl<'a, 'tcx, V: CodegenObject> LocalRef<'tcx, V> {
117 fn new_operand<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
119 layout: TyAndLayout<'tcx>,
120 ) -> LocalRef<'tcx, V> {
122 // Zero-size temporaries aren't always initialized, which
123 // doesn't matter because they don't contain data, but
124 // we need something in the operand.
125 LocalRef::Operand(Some(OperandRef::new_zst(bx, layout)))
127 LocalRef::Operand(None)
132 ///////////////////////////////////////////////////////////////////////////
134 pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
135 cx: &'a Bx::CodegenCx,
136 instance: Instance<'tcx>,
138 assert!(!instance.substs.needs_infer());
140 let llfn = cx.get_fn(instance);
142 let mir = cx.tcx().instance_mir(instance.def);
144 let fn_abi = FnAbi::of_instance(cx, instance, &[]);
145 debug!("fn_abi: {:?}", fn_abi);
147 let debug_context = cx.create_function_debug_context(instance, &fn_abi, llfn, &mir);
149 let mut bx = Bx::new_block(cx, llfn, "start");
151 if mir.basic_blocks().iter().any(|bb| bb.is_cleanup) {
152 bx.set_personality_fn(cx.eh_personality());
157 let cleanup_kinds = analyze::cleanup_kinds(&mir);
158 // Allocate a `Block` for every basic block, except
159 // the start block, if nothing loops back to it.
160 let reentrant_start_block = !mir.predecessors()[mir::START_BLOCK].is_empty();
161 let block_bxs: IndexVec<mir::BasicBlock, Bx::BasicBlock> = mir
165 if bb == mir::START_BLOCK && !reentrant_start_block {
168 bx.build_sibling_block(&format!("{:?}", bb)).llbb()
173 let (landing_pads, funclets) = create_funclets(&mir, &mut bx, &cleanup_kinds, &block_bxs);
174 let mut fx = FunctionCx {
180 personality_slot: None,
182 unreachable_block: None,
186 locals: IndexVec::new(),
188 per_local_var_debug_info: None,
189 caller_location: None,
192 fx.per_local_var_debug_info = fx.compute_per_local_var_debug_info();
194 for const_ in &mir.required_consts {
195 if let Err(err) = fx.eval_mir_constant(const_) {
197 // errored or at least linted
198 ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted => {}
199 ErrorHandled::TooGeneric => {
200 span_bug!(const_.span, "codgen encountered polymorphic constant: {:?}", err)
206 let memory_locals = analyze::non_ssa_locals(&fx);
208 // Allocate variable and temp allocas
210 let args = arg_local_refs(&mut bx, &mut fx, &memory_locals);
212 let mut allocate_local = |local| {
213 let decl = &mir.local_decls[local];
214 let layout = bx.layout_of(fx.monomorphize(&decl.ty));
215 assert!(!layout.ty.has_erasable_regions());
217 if local == mir::RETURN_PLACE && fx.fn_abi.ret.is_indirect() {
218 debug!("alloc: {:?} (return place) -> place", local);
219 let llretptr = bx.get_param(0);
220 return LocalRef::Place(PlaceRef::new_sized(llretptr, layout));
223 if memory_locals.contains(local) {
224 debug!("alloc: {:?} -> place", local);
225 if layout.is_unsized() {
226 LocalRef::UnsizedPlace(PlaceRef::alloca_unsized_indirect(&mut bx, layout))
228 LocalRef::Place(PlaceRef::alloca(&mut bx, layout))
231 debug!("alloc: {:?} -> operand", local);
232 LocalRef::new_operand(&mut bx, layout)
236 let retptr = allocate_local(mir::RETURN_PLACE);
238 .chain(args.into_iter())
239 .chain(mir.vars_and_temps_iter().map(allocate_local))
243 // Apply debuginfo to the newly allocated locals.
244 fx.debug_introduce_locals(&mut bx);
246 // Branch to the START block, if it's not the entry block.
247 if reentrant_start_block {
248 bx.br(fx.blocks[mir::START_BLOCK]);
251 let rpo = traversal::reverse_postorder(&mir);
252 let mut visited = BitSet::new_empty(mir.basic_blocks().len());
254 // Codegen the body of each block using reverse postorder
256 visited.insert(bb.index());
257 fx.codegen_block(bb);
260 // Remove blocks that haven't been visited, or have no
262 for bb in mir.basic_blocks().indices() {
264 if !visited.contains(bb.index()) {
265 debug!("codegen_mir: block {:?} was not visited", bb);
267 bx.delete_basic_block(fx.blocks[bb]);
273 fn create_funclets<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
274 mir: &'tcx mir::Body<'tcx>,
276 cleanup_kinds: &IndexVec<mir::BasicBlock, CleanupKind>,
277 block_bxs: &IndexVec<mir::BasicBlock, Bx::BasicBlock>,
279 IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>>,
280 IndexVec<mir::BasicBlock, Option<Bx::Funclet>>,
285 .map(|((bb, &llbb), cleanup_kind)| {
286 match *cleanup_kind {
287 CleanupKind::Funclet if base::wants_msvc_seh(bx.sess()) => {}
288 _ => return (None, None),
293 match mir[bb].terminator.as_ref().map(|t| &t.kind) {
294 // This is a basic block that we're aborting the program for,
295 // notably in an `extern` function. These basic blocks are inserted
296 // so that we assert that `extern` functions do indeed not panic,
297 // and if they do we abort the process.
299 // On MSVC these are tricky though (where we're doing funclets). If
300 // we were to do a cleanuppad (like below) the normal functions like
301 // `longjmp` would trigger the abort logic, terminating the
302 // program. Instead we insert the equivalent of `catch(...)` for C++
303 // which magically doesn't trigger when `longjmp` files over this
306 // Lots more discussion can be found on #48251 but this codegen is
307 // modeled after clang's for:
314 Some(&mir::TerminatorKind::Abort) => {
315 let mut cs_bx = bx.build_sibling_block(&format!("cs_funclet{:?}", bb));
316 let mut cp_bx = bx.build_sibling_block(&format!("cp_funclet{:?}", bb));
317 ret_llbb = cs_bx.llbb();
319 let cs = cs_bx.catch_switch(None, None, 1);
320 cs_bx.add_handler(cs, cp_bx.llbb());
322 // The "null" here is actually a RTTI type descriptor for the
323 // C++ personality function, but `catch (...)` has no type so
324 // it's null. The 64 here is actually a bitfield which
325 // represents that this is a catch-all block.
326 let null = bx.const_null(bx.type_i8p());
327 let sixty_four = bx.const_i32(64);
328 funclet = cp_bx.catch_pad(cs, &[null, sixty_four, null]);
332 let mut cleanup_bx = bx.build_sibling_block(&format!("funclet_{:?}", bb));
333 ret_llbb = cleanup_bx.llbb();
334 funclet = cleanup_bx.cleanup_pad(None, &[]);
339 (Some(ret_llbb), Some(funclet))
344 /// Produces, for each argument, a `Value` pointing at the
345 /// argument's value. As arguments are places, these are always
347 fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
349 fx: &mut FunctionCx<'a, 'tcx, Bx>,
350 memory_locals: &BitSet<mir::Local>,
351 ) -> Vec<LocalRef<'tcx, Bx::Value>> {
354 let mut llarg_idx = fx.fn_abi.ret.is_indirect() as usize;
359 .map(|(arg_index, local)| {
360 let arg_decl = &mir.local_decls[local];
362 if Some(local) == mir.spread_arg {
363 // This argument (e.g., the last argument in the "rust-call" ABI)
364 // is a tuple that was spread at the ABI level and now we have
365 // to reconstruct it into a tuple local variable, from multiple
366 // individual LLVM function arguments.
368 let arg_ty = fx.monomorphize(&arg_decl.ty);
369 let tupled_arg_tys = match arg_ty.kind {
370 ty::Tuple(ref tys) => tys,
371 _ => bug!("spread argument isn't a tuple?!"),
374 let place = PlaceRef::alloca(bx, bx.layout_of(arg_ty));
375 for i in 0..tupled_arg_tys.len() {
376 let arg = &fx.fn_abi.args[idx];
378 if arg.pad.is_some() {
381 let pr_field = place.project_field(bx, i);
382 bx.store_fn_arg(arg, &mut llarg_idx, pr_field);
385 return LocalRef::Place(place);
388 if fx.fn_abi.c_variadic && arg_index == fx.fn_abi.args.len() {
389 let arg_ty = fx.monomorphize(&arg_decl.ty);
391 let va_list = PlaceRef::alloca(bx, bx.layout_of(arg_ty));
392 bx.va_start(va_list.llval);
394 return LocalRef::Place(va_list);
397 let arg = &fx.fn_abi.args[idx];
399 if arg.pad.is_some() {
403 if !memory_locals.contains(local) {
404 // We don't have to cast or keep the argument in the alloca.
405 // FIXME(eddyb): We should figure out how to use llvm.dbg.value instead
406 // of putting everything in allocas just so we can use llvm.dbg.declare.
407 let local = |op| LocalRef::Operand(Some(op));
409 PassMode::Ignore => {
410 return local(OperandRef::new_zst(bx, arg.layout));
412 PassMode::Direct(_) => {
413 let llarg = bx.get_param(llarg_idx);
415 return local(OperandRef::from_immediate_or_packed_pair(
416 bx, llarg, arg.layout,
419 PassMode::Pair(..) => {
420 let (a, b) = (bx.get_param(llarg_idx), bx.get_param(llarg_idx + 1));
423 return local(OperandRef {
424 val: OperandValue::Pair(a, b),
432 if arg.is_sized_indirect() {
433 // Don't copy an indirect argument to an alloca, the caller
434 // already put it in a temporary alloca and gave it up.
436 let llarg = bx.get_param(llarg_idx);
438 LocalRef::Place(PlaceRef::new_sized(llarg, arg.layout))
439 } else if arg.is_unsized_indirect() {
440 // As the storage for the indirect argument lives during
441 // the whole function call, we just copy the fat pointer.
442 let llarg = bx.get_param(llarg_idx);
444 let llextra = bx.get_param(llarg_idx);
446 let indirect_operand = OperandValue::Pair(llarg, llextra);
448 let tmp = PlaceRef::alloca_unsized_indirect(bx, arg.layout);
449 indirect_operand.store(bx, tmp);
450 LocalRef::UnsizedPlace(tmp)
452 let tmp = PlaceRef::alloca(bx, arg.layout);
453 bx.store_fn_arg(arg, &mut llarg_idx, tmp);
457 .collect::<Vec<_>>();
459 if fx.instance.def.requires_caller_location(bx.tcx()) {
461 fx.fn_abi.args.len(),
463 "#[track_caller] fn's must have 1 more argument in their ABI than in their MIR",
466 let arg = fx.fn_abi.args.last().unwrap();
468 PassMode::Direct(_) => (),
469 _ => bug!("caller location must be PassMode::Direct, found {:?}", arg.mode),
472 fx.caller_location = Some(OperandRef {
473 val: OperandValue::Immediate(bx.get_param(llarg_idx)),