1 use crate::errors::DumpVTableEntries;
2 use crate::traits::{impossible_predicates, is_vtable_safe_method};
3 use rustc_hir::def_id::DefId;
4 use rustc_hir::lang_items::LangItem;
5 use rustc_infer::traits::util::PredicateSet;
6 use rustc_infer::traits::ImplSource;
7 use rustc_middle::ty::visit::TypeVisitable;
8 use rustc_middle::ty::InternalSubsts;
9 use rustc_middle::ty::{self, GenericParamDefKind, ToPredicate, Ty, TyCtxt, VtblEntry};
10 use rustc_span::{sym, Span};
11 use smallvec::SmallVec;
14 use std::ops::ControlFlow;
16 #[derive(Clone, Debug)]
17 pub(super) enum VtblSegment<'tcx> {
19 TraitOwnEntries { trait_ref: ty::PolyTraitRef<'tcx>, emit_vptr: bool },
22 /// Prepare the segments for a vtable
23 pub(super) fn prepare_vtable_segments<'tcx, T>(
25 trait_ref: ty::PolyTraitRef<'tcx>,
26 mut segment_visitor: impl FnMut(VtblSegment<'tcx>) -> ControlFlow<T>,
28 // The following constraints holds for the final arrangement.
29 // 1. The whole virtual table of the first direct super trait is included as the
30 // the prefix. If this trait doesn't have any super traits, then this step
31 // consists of the dsa metadata.
32 // 2. Then comes the proper pointer metadata(vptr) and all own methods for all
33 // other super traits except those already included as part of the first
34 // direct super trait virtual table.
35 // 3. finally, the own methods of this trait.
37 // This has the advantage that trait upcasting to the first direct super trait on each level
38 // is zero cost, and to another trait includes only replacing the pointer with one level indirection,
39 // while not using too much extra memory.
41 // For a single inheritance relationship like this,
42 // D --> C --> B --> A
43 // The resulting vtable will consists of these segments:
46 // For a multiple inheritance relationship like this,
49 // The resulting vtable will consists of these segments:
50 // DSA, A, B, B-vptr, C, D
52 // For a diamond inheritance relationship like this,
55 // The resulting vtable will consists of these segments:
56 // DSA, A, B, C, C-vptr, D
58 // For a more complex inheritance relationship like this:
59 // O --> G --> C --> A
67 // The resulting vtable will consists of these segments:
68 // DSA, A, B, B-vptr, C, D, D-vptr, E, E-vptr, F, F-vptr, G,
69 // H, H-vptr, I, I-vptr, J, J-vptr, K, K-vptr, L, L-vptr, M, M-vptr,
72 // emit dsa segment first.
73 if let ControlFlow::Break(v) = (segment_visitor)(VtblSegment::MetadataDSA) {
77 let mut emit_vptr_on_new_entry = false;
78 let mut visited = PredicateSet::new(tcx);
79 let predicate = trait_ref.without_const().to_predicate(tcx);
80 let mut stack: SmallVec<[(ty::PolyTraitRef<'tcx>, _, _); 5]> =
81 smallvec![(trait_ref, emit_vptr_on_new_entry, None)];
82 visited.insert(predicate);
84 // the main traversal loop:
85 // basically we want to cut the inheritance directed graph into a few non-overlapping slices of nodes
86 // that each node is emitted after all its descendents have been emitted.
87 // so we convert the directed graph into a tree by skipping all previously visited nodes using a visited set.
88 // this is done on the fly.
89 // Each loop run emits a slice - it starts by find a "childless" unvisited node, backtracking upwards, and it
90 // stops after it finds a node that has a next-sibling node.
91 // This next-sibling node will used as the starting point of next slice.
94 // For a diamond inheritance relationship like this,
95 // D#1 --> B#0 --> A#0
98 // Starting point 0 stack [D]
99 // Loop run #0: Stack after diving in is [D B A], A is "childless"
100 // after this point, all newly visited nodes won't have a vtable that equals to a prefix of this one.
101 // Loop run #0: Emitting the slice [B A] (in reverse order), B has a next-sibling node, so this slice stops here.
102 // Loop run #0: Stack after exiting out is [D C], C is the next starting point.
103 // Loop run #1: Stack after diving in is [D C], C is "childless", since its child A is skipped(already emitted).
104 // Loop run #1: Emitting the slice [D C] (in reverse order). No one has a next-sibling node.
105 // Loop run #1: Stack after exiting out is []. Now the function exits.
108 // dive deeper into the stack, recording the path
110 if let Some((inner_most_trait_ref, _, _)) = stack.last() {
111 let inner_most_trait_ref = *inner_most_trait_ref;
112 let mut direct_super_traits_iter = tcx
113 .super_predicates_of(inner_most_trait_ref.def_id())
116 .filter_map(move |(pred, _)| {
117 pred.subst_supertrait(tcx, &inner_most_trait_ref).to_opt_poly_trait_pred()
120 'diving_in_skip_visited_traits: loop {
121 if let Some(next_super_trait) = direct_super_traits_iter.next() {
122 if visited.insert(next_super_trait.to_predicate(tcx)) {
123 // We're throwing away potential constness of super traits here.
124 // FIXME: handle ~const super traits
125 let next_super_trait = next_super_trait.map_bound(|t| t.trait_ref);
128 emit_vptr_on_new_entry,
129 Some(direct_super_traits_iter),
131 break 'diving_in_skip_visited_traits;
133 continue 'diving_in_skip_visited_traits;
142 // Other than the left-most path, vptr should be emitted for each trait.
143 emit_vptr_on_new_entry = true;
145 // emit innermost item, move to next sibling and stop there if possible, otherwise jump to outer level.
147 if let Some((inner_most_trait_ref, emit_vptr, siblings_opt)) = stack.last_mut() {
148 if let ControlFlow::Break(v) = (segment_visitor)(VtblSegment::TraitOwnEntries {
149 trait_ref: *inner_most_trait_ref,
150 emit_vptr: *emit_vptr,
155 'exiting_out_skip_visited_traits: loop {
156 if let Some(siblings) = siblings_opt {
157 if let Some(next_inner_most_trait_ref) = siblings.next() {
158 if visited.insert(next_inner_most_trait_ref.to_predicate(tcx)) {
159 // We're throwing away potential constness of super traits here.
160 // FIXME: handle ~const super traits
161 let next_inner_most_trait_ref =
162 next_inner_most_trait_ref.map_bound(|t| t.trait_ref);
163 *inner_most_trait_ref = next_inner_most_trait_ref;
164 *emit_vptr = emit_vptr_on_new_entry;
167 continue 'exiting_out_skip_visited_traits;
172 continue 'exiting_out;
181 fn dump_vtable_entries<'tcx>(
184 trait_ref: ty::PolyTraitRef<'tcx>,
185 entries: &[VtblEntry<'tcx>],
187 tcx.sess.emit_err(DumpVTableEntries {
190 entries: format!("{:#?}", entries),
194 fn own_existential_vtable_entries<'tcx>(tcx: TyCtxt<'tcx>, trait_def_id: DefId) -> &'tcx [DefId] {
195 let trait_methods = tcx
196 .associated_items(trait_def_id)
197 .in_definition_order()
198 .filter(|item| item.kind == ty::AssocKind::Fn);
199 // Now list each method's DefId (for within its trait).
200 let own_entries = trait_methods.filter_map(move |trait_method| {
201 debug!("own_existential_vtable_entry: trait_method={:?}", trait_method);
202 let def_id = trait_method.def_id;
204 // Some methods cannot be called on an object; skip those.
205 if !is_vtable_safe_method(tcx, trait_def_id, &trait_method) {
206 debug!("own_existential_vtable_entry: not vtable safe");
213 tcx.arena.alloc_from_iter(own_entries.into_iter())
216 /// Given a trait `trait_ref`, iterates the vtable entries
217 /// that come from `trait_ref`, including its supertraits.
218 fn vtable_entries<'tcx>(
220 trait_ref: ty::PolyTraitRef<'tcx>,
221 ) -> &'tcx [VtblEntry<'tcx>] {
222 debug!("vtable_entries({:?})", trait_ref);
224 let mut entries = vec![];
226 let vtable_segment_callback = |segment| -> ControlFlow<()> {
228 VtblSegment::MetadataDSA => {
229 entries.extend(TyCtxt::COMMON_VTABLE_ENTRIES);
231 VtblSegment::TraitOwnEntries { trait_ref, emit_vptr } => {
232 let existential_trait_ref = trait_ref
233 .map_bound(|trait_ref| ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref));
235 // Lookup the shape of vtable for the trait.
236 let own_existential_entries =
237 tcx.own_existential_vtable_entries(existential_trait_ref.def_id());
239 let own_entries = own_existential_entries.iter().copied().map(|def_id| {
240 debug!("vtable_entries: trait_method={:?}", def_id);
242 // The method may have some early-bound lifetimes; add regions for those.
243 let substs = trait_ref.map_bound(|trait_ref| {
244 InternalSubsts::for_item(tcx, def_id, |param, _| match param.kind {
245 GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
246 GenericParamDefKind::Type { .. }
247 | GenericParamDefKind::Const { .. } => {
248 trait_ref.substs[param.index as usize]
253 // The trait type may have higher-ranked lifetimes in it;
254 // erase them if they appear, so that we get the type
255 // at some particular call site.
257 .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), substs);
259 // It's possible that the method relies on where-clauses that
260 // do not hold for this particular set of type parameters.
261 // Note that this method could then never be called, so we
262 // do not want to try and codegen it, in that case (see #23435).
263 let predicates = tcx.predicates_of(def_id).instantiate_own(tcx, substs);
264 if impossible_predicates(tcx, predicates.predicates) {
265 debug!("vtable_entries: predicates do not hold");
266 return VtblEntry::Vacant;
269 let instance = ty::Instance::resolve_for_vtable(
271 ty::ParamEnv::reveal_all(),
275 .expect("resolution failed during building vtable representation");
276 VtblEntry::Method(instance)
279 entries.extend(own_entries);
282 entries.push(VtblEntry::TraitVPtr(trait_ref));
287 ControlFlow::Continue(())
290 let _ = prepare_vtable_segments(tcx, trait_ref, vtable_segment_callback);
292 if tcx.has_attr(trait_ref.def_id(), sym::rustc_dump_vtable) {
293 let sp = tcx.def_span(trait_ref.def_id());
294 dump_vtable_entries(tcx, sp, trait_ref, &entries);
297 tcx.arena.alloc_from_iter(entries.into_iter())
300 /// Find slot base for trait methods within vtable entries of another trait
301 pub(super) fn vtable_trait_first_method_offset<'tcx>(
304 ty::PolyTraitRef<'tcx>, // trait_to_be_found
305 ty::PolyTraitRef<'tcx>, // trait_owning_vtable
308 let (trait_to_be_found, trait_owning_vtable) = key;
311 let trait_to_be_found_erased = tcx.erase_regions(trait_to_be_found);
313 let vtable_segment_callback = {
314 let mut vtable_base = 0;
318 VtblSegment::MetadataDSA => {
319 vtable_base += TyCtxt::COMMON_VTABLE_ENTRIES.len();
321 VtblSegment::TraitOwnEntries { trait_ref, emit_vptr } => {
322 if tcx.erase_regions(trait_ref) == trait_to_be_found_erased {
323 return ControlFlow::Break(vtable_base);
325 vtable_base += count_own_vtable_entries(tcx, trait_ref);
331 ControlFlow::Continue(())
335 if let Some(vtable_base) =
336 prepare_vtable_segments(tcx, trait_owning_vtable, vtable_segment_callback)
340 bug!("Failed to find info for expected trait in vtable");
344 /// Find slot offset for trait vptr within vtable entries of another trait
345 pub(crate) fn vtable_trait_upcasting_coercion_new_vptr_slot<'tcx>(
348 Ty<'tcx>, // trait object type whose trait owning vtable
349 Ty<'tcx>, // trait object for supertrait
352 let (source, target) = key;
353 assert!(matches!(&source.kind(), &ty::Dynamic(..)) && !source.needs_infer());
354 assert!(matches!(&target.kind(), &ty::Dynamic(..)) && !target.needs_infer());
356 // this has been typecked-before, so diagnostics is not really needed.
357 let unsize_trait_did = tcx.require_lang_item(LangItem::Unsize, None);
359 let trait_ref = tcx.mk_trait_ref(unsize_trait_did, [source, target]);
361 match tcx.codegen_select_candidate((ty::ParamEnv::reveal_all(), ty::Binder::dummy(trait_ref))) {
362 Ok(ImplSource::TraitUpcasting(implsrc_traitcasting)) => {
363 implsrc_traitcasting.vtable_vptr_slot
365 otherwise => bug!("expected TraitUpcasting candidate, got {otherwise:?}"),
369 /// Given a trait `trait_ref`, returns the number of vtable entries
370 /// that come from `trait_ref`, excluding its supertraits. Used in
371 /// computing the vtable base for an upcast trait of a trait object.
372 pub(crate) fn count_own_vtable_entries<'tcx>(
374 trait_ref: ty::PolyTraitRef<'tcx>,
376 tcx.own_existential_vtable_entries(trait_ref.def_id()).len()
379 pub(super) fn provide(providers: &mut ty::query::Providers) {
380 *providers = ty::query::Providers {
381 own_existential_vtable_entries,
383 vtable_trait_upcasting_coercion_new_vptr_slot,