1 //! The implementation of the query system itself. This defines the macros that
2 //! generate the actual methods on tcx which find and execute the provider,
3 //! manage the caches, and so forth.
5 use crate::dep_graph::{DepNodeIndex, DepNode, DepKind, SerializedDepNodeIndex};
7 use crate::ty::{self, TyCtxt};
8 use crate::ty::query::Query;
9 use crate::ty::query::config::{QueryConfig, QueryDescription};
10 use crate::ty::query::job::{QueryJob, QueryResult, QueryInfo};
12 use crate::util::common::{profq_msg, ProfileQueriesMsg, QueryMsg};
14 use errors::DiagnosticBuilder;
16 use errors::Diagnostic;
17 use errors::FatalError;
18 use rustc_data_structures::fx::{FxHashMap};
19 use rustc_data_structures::sync::{Lrc, Lock};
20 use rustc_data_structures::thin_vec::ThinVec;
21 #[cfg(not(parallel_compiler))]
22 use rustc_data_structures::cold_path;
25 use std::collections::hash_map::Entry;
27 use syntax::source_map::DUMMY_SP;
29 pub struct QueryCache<'tcx, D: QueryConfig<'tcx> + ?Sized> {
30 pub(super) results: FxHashMap<D::Key, QueryValue<D::Value>>,
31 pub(super) active: FxHashMap<D::Key, QueryResult<'tcx>>,
32 #[cfg(debug_assertions)]
33 pub(super) cache_hits: usize,
36 pub(super) struct QueryValue<T> {
38 pub(super) index: DepNodeIndex,
41 impl<T> QueryValue<T> {
42 pub(super) fn new(value: T,
43 dep_node_index: DepNodeIndex)
47 index: dep_node_index,
52 impl<'tcx, M: QueryConfig<'tcx>> Default for QueryCache<'tcx, M> {
53 fn default() -> QueryCache<'tcx, M> {
55 results: FxHashMap::default(),
56 active: FxHashMap::default(),
57 #[cfg(debug_assertions)]
63 // If enabled, send a message to the profile-queries thread
64 macro_rules! profq_msg {
65 ($tcx:expr, $msg:expr) => {
66 if cfg!(debug_assertions) {
67 if $tcx.sess.profile_queries() {
68 profq_msg($tcx.sess, $msg)
74 // If enabled, format a key using its debug string, which can be
75 // expensive to compute (in terms of time).
76 macro_rules! profq_query_msg {
77 ($query:expr, $tcx:expr, $key:expr) => {{
78 let msg = if cfg!(debug_assertions) {
79 if $tcx.sess.profile_queries_and_keys() {
80 Some(format!("{:?}", $key))
90 /// A type representing the responsibility to execute the job in the `job` field.
91 /// This will poison the relevant query if dropped.
92 pub(super) struct JobOwner<'a, 'tcx: 'a, Q: QueryDescription<'tcx> + 'a> {
93 cache: &'a Lock<QueryCache<'tcx, Q>>,
95 job: Lrc<QueryJob<'tcx>>,
98 impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> {
99 /// Either gets a `JobOwner` corresponding the query, allowing us to
100 /// start executing the query, or it returns with the result of the query.
101 /// If the query is executing elsewhere, this will wait for it.
102 /// If the query panicked, this will silently panic.
104 /// This function is inlined because that results in a noticeable speed-up
105 /// for some compile-time benchmarks.
107 pub(super) fn try_get(
108 tcx: TyCtxt<'tcx, '_>,
111 ) -> TryGetJob<'a, 'tcx, Q> {
112 let cache = Q::query_cache(tcx);
114 let mut lock = cache.borrow_mut();
115 if let Some(value) = lock.results.get(key) {
116 profq_msg!(tcx, ProfileQueriesMsg::CacheHit);
117 tcx.sess.profiler(|p| p.record_query_hit(Q::NAME));
118 let result = (value.value.clone(), value.index);
119 #[cfg(debug_assertions)]
121 lock.cache_hits += 1;
123 return TryGetJob::JobCompleted(result);
125 let job = match lock.active.entry((*key).clone()) {
126 Entry::Occupied(entry) => {
128 QueryResult::Started(ref job) => {
129 // For parallel queries, we'll block and wait until the query running
130 // in another thread has completed. Record how long we wait in the
132 #[cfg(parallel_compiler)]
133 tcx.sess.profiler(|p| p.query_blocked_start(Q::NAME));
137 QueryResult::Poisoned => FatalError.raise(),
140 Entry::Vacant(entry) => {
141 // No job entry for this query. Return a new one to be started later.
142 return tls::with_related_context(tcx, |icx| {
143 // Create the `parent` variable before `info`. This allows LLVM
144 // to elide the move of `info`
145 let parent = icx.query.clone();
146 let info = QueryInfo {
148 query: Q::query(key.clone()),
150 let job = Lrc::new(QueryJob::new(info, parent));
151 let owner = JobOwner {
156 entry.insert(QueryResult::Started(job));
157 TryGetJob::NotYetStarted(owner)
163 // If we are single-threaded we know that we have cycle error,
164 // so we just return the error.
165 #[cfg(not(parallel_compiler))]
166 return TryGetJob::Cycle(cold_path(|| {
167 Q::handle_cycle_error(tcx, job.find_cycle_in_stack(tcx, span))
170 // With parallel queries we might just have to wait on some other
172 #[cfg(parallel_compiler)]
174 let result = job.r#await(tcx, span);
175 tcx.sess.profiler(|p| p.query_blocked_end(Q::NAME));
177 if let Err(cycle) = result {
178 return TryGetJob::Cycle(Q::handle_cycle_error(tcx, cycle));
184 /// Completes the query by updating the query cache with the `result`,
185 /// signals the waiter and forgets the JobOwner, so it won't poison the query
187 pub(super) fn complete(self, result: &Q::Value, dep_node_index: DepNodeIndex) {
188 // We can move out of `self` here because we `mem::forget` it below
189 let key = unsafe { ptr::read(&self.key) };
190 let job = unsafe { ptr::read(&self.job) };
191 let cache = self.cache;
193 // Forget ourself so our destructor won't poison the query
196 let value = QueryValue::new(result.clone(), dep_node_index);
198 let mut lock = cache.borrow_mut();
199 lock.active.remove(&key);
200 lock.results.insert(key, value);
203 job.signal_complete();
208 fn with_diagnostics<F, R>(f: F) -> (R, ThinVec<Diagnostic>)
210 F: FnOnce(Option<&Lock<ThinVec<Diagnostic>>>) -> R
212 let diagnostics = Lock::new(ThinVec::new());
213 let result = f(Some(&diagnostics));
214 (result, diagnostics.into_inner())
217 impl<'a, 'tcx, Q: QueryDescription<'tcx>> Drop for JobOwner<'a, 'tcx, Q> {
221 // Poison the query so jobs waiting on it panic
222 self.cache.borrow_mut().active.insert(self.key.clone(), QueryResult::Poisoned);
223 // Also signal the completion of the job, so waiters
224 // will continue execution
225 self.job.signal_complete();
230 pub struct CycleError<'tcx> {
231 /// The query and related span which uses the cycle
232 pub(super) usage: Option<(Span, Query<'tcx>)>,
233 pub(super) cycle: Vec<QueryInfo<'tcx>>,
236 /// The result of `try_get_lock`
237 pub(super) enum TryGetJob<'a, 'tcx: 'a, D: QueryDescription<'tcx> + 'a> {
238 /// The query is not yet started. Contains a guard to the cache eventually used to start it.
239 NotYetStarted(JobOwner<'a, 'tcx, D>),
241 /// The query was already completed.
242 /// Returns the result of the query and its dep node index
243 /// if it succeeded or a cycle error if it failed
244 JobCompleted((D::Value, DepNodeIndex)),
246 /// Trying to execute the query resulted in a cycle.
250 impl<'gcx, 'tcx> TyCtxt<'gcx, 'tcx> {
251 /// Executes a job by changing the ImplicitCtxt to point to the
252 /// new query job while it executes. It returns the diagnostics
253 /// captured during execution and the actual result.
255 pub(super) fn start_query<F, R>(
257 job: Lrc<QueryJob<'gcx>>,
258 diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
262 F: for<'lcx> FnOnce(TyCtxt<'gcx, 'lcx>) -> R,
264 // The TyCtxt stored in TLS has the same global interner lifetime
265 // as `self`, so we use `with_related_context` to relate the 'gcx lifetimes
266 // when accessing the ImplicitCtxt
267 tls::with_related_context(self, move |current_icx| {
268 // Update the ImplicitCtxt to point to our new query job
269 let new_icx = tls::ImplicitCtxt {
270 tcx: self.global_tcx(),
273 layout_depth: current_icx.layout_depth,
274 task_deps: current_icx.task_deps,
277 // Use the ImplicitCtxt while we execute the query
278 tls::enter_context(&new_icx, |_| {
279 compute(self.global_tcx())
286 pub(super) fn report_cycle(
288 CycleError { usage, cycle: stack }: CycleError<'gcx>,
289 ) -> DiagnosticBuilder<'tcx> {
290 assert!(!stack.is_empty());
292 let fix_span = |span: Span, query: &Query<'gcx>| {
293 self.sess.source_map().def_span(query.default_span(self, span))
296 // Disable naming impls with types in this path, since that
297 // sometimes cycles itself, leading to extra cycle errors.
298 // (And cycle errors around impls tend to occur during the
299 // collect/coherence phases anyhow.)
300 ty::print::with_forced_impl_filename_line(|| {
301 let span = fix_span(stack[1 % stack.len()].span, &stack[0].query);
302 let mut err = struct_span_err!(self.sess,
305 "cycle detected when {}",
306 stack[0].query.describe(self));
308 for i in 1..stack.len() {
309 let query = &stack[i].query;
310 let span = fix_span(stack[(i + 1) % stack.len()].span, query);
311 err.span_note(span, &format!("...which requires {}...", query.describe(self)));
314 err.note(&format!("...which again requires {}, completing the cycle",
315 stack[0].query.describe(self)));
317 if let Some((span, query)) = usage {
318 err.span_note(fix_span(span, &query),
319 &format!("cycle used when {}", query.describe(self)));
326 pub fn try_print_query_stack() {
327 eprintln!("query stack during panic:");
329 tls::with_context_opt(|icx| {
330 if let Some(icx) = icx {
331 let mut current_query = icx.query.clone();
334 while let Some(query) = current_query {
335 let mut db = DiagnosticBuilder::new(icx.tcx.sess.diagnostic(),
337 &format!("#{} [{}] {}",
339 query.info.query.name(),
340 query.info.query.describe(icx.tcx)));
341 db.set_span(icx.tcx.sess.source_map().def_span(query.info.span));
342 icx.tcx.sess.diagnostic().force_print_db(db);
344 current_query = query.parent.clone();
350 eprintln!("end of query stack");
354 pub(super) fn get_query<Q: QueryDescription<'gcx>>(
359 debug!("ty::query::get_query<{}>(key={:?}, span={:?})",
365 ProfileQueriesMsg::QueryBegin(
367 profq_query_msg!(Q::NAME.as_str(), self, key),
371 let job = match JobOwner::try_get(self, span, &key) {
372 TryGetJob::NotYetStarted(job) => job,
373 TryGetJob::Cycle(result) => return result,
374 TryGetJob::JobCompleted((v, index)) => {
375 self.dep_graph.read_index(index);
380 // Fast path for when incr. comp. is off. `to_dep_node` is
381 // expensive for some DepKinds.
382 if !self.dep_graph.is_fully_enabled() {
383 let null_dep_node = DepNode::new_no_params(crate::dep_graph::DepKind::Null);
384 return self.force_query_with_job::<Q>(key, job, null_dep_node).0;
387 let dep_node = Q::to_dep_node(self, &key);
389 if dep_node.kind.is_anon() {
390 profq_msg!(self, ProfileQueriesMsg::ProviderBegin);
391 self.sess.profiler(|p| p.start_query(Q::NAME));
393 let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
394 self.start_query(job.job.clone(), diagnostics, |tcx| {
395 tcx.dep_graph.with_anon_task(dep_node.kind, || {
396 Q::compute(tcx.global_tcx(), key)
401 self.sess.profiler(|p| p.end_query(Q::NAME));
402 profq_msg!(self, ProfileQueriesMsg::ProviderEnd);
404 self.dep_graph.read_index(dep_node_index);
406 if unlikely!(!diagnostics.is_empty()) {
407 self.queries.on_disk_cache
408 .store_diagnostics_for_anon_node(dep_node_index, diagnostics);
411 job.complete(&result, dep_node_index);
416 if !dep_node.kind.is_eval_always() {
417 // The diagnostics for this query will be
418 // promoted to the current session during
419 // try_mark_green(), so we can ignore them here.
420 let loaded = self.start_query(job.job.clone(), None, |tcx| {
421 let marked = tcx.dep_graph.try_mark_green_and_read(tcx, &dep_node);
422 marked.map(|(prev_dep_node_index, dep_node_index)| {
423 (tcx.load_from_disk_and_cache_in_memory::<Q>(
431 if let Some((result, dep_node_index)) = loaded {
432 job.complete(&result, dep_node_index);
437 let (result, dep_node_index) = self.force_query_with_job::<Q>(key, job, dep_node);
438 self.dep_graph.read_index(dep_node_index);
442 fn load_from_disk_and_cache_in_memory<Q: QueryDescription<'gcx>>(
445 prev_dep_node_index: SerializedDepNodeIndex,
446 dep_node_index: DepNodeIndex,
450 // Note this function can be called concurrently from the same query
451 // We must ensure that this is handled correctly
453 debug_assert!(self.dep_graph.is_green(dep_node));
455 // First we try to load the result from the on-disk cache
456 let result = if Q::cache_on_disk(self.global_tcx(), key.clone()) &&
457 self.sess.opts.debugging_opts.incremental_queries {
458 self.sess.profiler(|p| p.incremental_load_result_start(Q::NAME));
459 let result = Q::try_load_from_disk(self.global_tcx(), prev_dep_node_index);
460 self.sess.profiler(|p| p.incremental_load_result_end(Q::NAME));
462 // We always expect to find a cached result for things that
463 // can be forced from DepNode.
464 debug_assert!(!dep_node.kind.can_reconstruct_query_key() ||
466 "Missing on-disk cache entry for {:?}",
470 // Some things are never cached on disk.
474 let result = if let Some(result) = result {
475 profq_msg!(self, ProfileQueriesMsg::CacheHit);
476 self.sess.profiler(|p| p.record_query_hit(Q::NAME));
480 // We could not load a result from the on-disk cache, so
483 self.sess.profiler(|p| p.start_query(Q::NAME));
485 // The dep-graph for this computation is already in
487 let result = self.dep_graph.with_ignore(|| {
488 Q::compute(self, key)
491 self.sess.profiler(|p| p.end_query(Q::NAME));
495 // If -Zincremental-verify-ich is specified, re-hash results from
496 // the cache and make sure that they have the expected fingerprint.
497 if unlikely!(self.sess.opts.debugging_opts.incremental_verify_ich) {
498 self.incremental_verify_ich::<Q>(&result, dep_node, dep_node_index);
501 if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) {
502 self.dep_graph.mark_loaded_from_cache(dep_node_index, true);
510 fn incremental_verify_ich<Q: QueryDescription<'gcx>>(
514 dep_node_index: DepNodeIndex,
516 use crate::ich::Fingerprint;
518 assert!(Some(self.dep_graph.fingerprint_of(dep_node_index)) ==
519 self.dep_graph.prev_fingerprint_of(dep_node),
520 "Fingerprint for green query instance not loaded \
521 from cache: {:?}", dep_node);
523 debug!("BEGIN verify_ich({:?})", dep_node);
524 let mut hcx = self.create_stable_hashing_context();
526 let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
527 debug!("END verify_ich({:?})", dep_node);
529 let old_hash = self.dep_graph.fingerprint_of(dep_node_index);
531 assert!(new_hash == old_hash, "Found unstable fingerprints \
532 for {:?}", dep_node);
536 fn force_query_with_job<Q: QueryDescription<'gcx>>(
539 job: JobOwner<'_, 'gcx, Q>,
541 -> (Q::Value, DepNodeIndex) {
542 // If the following assertion triggers, it can have two reasons:
543 // 1. Something is wrong with DepNode creation, either here or
544 // in DepGraph::try_mark_green()
545 // 2. Two distinct query keys get mapped to the same DepNode
546 // (see for example #48923)
547 assert!(!self.dep_graph.dep_node_exists(&dep_node),
548 "Forcing query with already existing DepNode.\n\
553 profq_msg!(self, ProfileQueriesMsg::ProviderBegin);
554 self.sess.profiler(|p| p.start_query(Q::NAME));
556 let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
557 self.start_query(job.job.clone(), diagnostics, |tcx| {
558 if dep_node.kind.is_eval_always() {
559 tcx.dep_graph.with_eval_always_task(dep_node,
565 tcx.dep_graph.with_task(dep_node,
574 self.sess.profiler(|p| p.end_query(Q::NAME));
575 profq_msg!(self, ProfileQueriesMsg::ProviderEnd);
577 if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) {
578 self.dep_graph.mark_loaded_from_cache(dep_node_index, false);
581 if dep_node.kind != crate::dep_graph::DepKind::Null {
582 if unlikely!(!diagnostics.is_empty()) {
583 self.queries.on_disk_cache
584 .store_diagnostics(dep_node_index, diagnostics);
588 job.complete(&result, dep_node_index);
590 (result, dep_node_index)
593 /// Ensure that either this query has all green inputs or been executed.
594 /// Executing query::ensure(D) is considered a read of the dep-node D.
596 /// This function is particularly useful when executing passes for their
597 /// side-effects -- e.g., in order to report errors for erroneous programs.
599 /// Note: The optimization is only available during incr. comp.
600 pub(super) fn ensure_query<Q: QueryDescription<'gcx>>(self, key: Q::Key) -> () {
601 let dep_node = Q::to_dep_node(self, &key);
603 if dep_node.kind.is_eval_always() {
604 let _ = self.get_query::<Q>(DUMMY_SP, key);
608 // Ensuring an anonymous query makes no sense
609 assert!(!dep_node.kind.is_anon());
610 if self.dep_graph.try_mark_green_and_read(self, &dep_node).is_none() {
611 // A None return from `try_mark_green_and_read` means that this is either
612 // a new dep node or that the dep node has already been marked red.
613 // Either way, we can't call `dep_graph.read()` as we don't have the
614 // DepNodeIndex. We must invoke the query itself. The performance cost
615 // this introduces should be negligible as we'll immediately hit the
616 // in-memory cache, or another query down the line will.
618 let _ = self.get_query::<Q>(DUMMY_SP, key);
620 profq_msg!(self, ProfileQueriesMsg::CacheHit);
621 self.sess.profiler(|p| p.record_query_hit(Q::NAME));
626 fn force_query<Q: QueryDescription<'gcx>>(
634 ProfileQueriesMsg::QueryBegin(span.data(),
635 profq_query_msg!(Q::NAME.as_str(), self, key))
638 // We may be concurrently trying both execute and force a query.
639 // Ensure that only one of them runs the query.
640 let job = match JobOwner::try_get(self, span, &key) {
641 TryGetJob::NotYetStarted(job) => job,
642 TryGetJob::Cycle(_) |
643 TryGetJob::JobCompleted(_) => {
647 self.force_query_with_job::<Q>(key, job, dep_node);
651 macro_rules! handle_cycle_error {
652 ([][$tcx: expr, $error:expr]) => {{
653 $tcx.report_cycle($error).emit();
654 Value::from_cycle_error($tcx.global_tcx())
656 ([fatal_cycle$(, $modifiers:ident)*][$tcx:expr, $error:expr]) => {{
657 $tcx.report_cycle($error).emit();
658 $tcx.sess.abort_if_errors();
661 ([cycle_delay_bug$(, $modifiers:ident)*][$tcx:expr, $error:expr]) => {{
662 $tcx.report_cycle($error).delay_as_bug();
663 Value::from_cycle_error($tcx.global_tcx())
665 ([$other:ident$(, $modifiers:ident)*][$($args:tt)*]) => {
666 handle_cycle_error!([$($modifiers),*][$($args)*])
670 macro_rules! hash_result {
671 ([][$hcx:expr, $result:expr]) => {{
672 dep_graph::hash_result($hcx, &$result)
674 ([no_hash$(, $modifiers:ident)*][$hcx:expr, $result:expr]) => {{
677 ([$other:ident$(, $modifiers:ident)*][$($args:tt)*]) => {
678 hash_result!([$($modifiers),*][$($args)*])
682 macro_rules! define_queries {
683 (<$tcx:tt> $($category:tt {
684 $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*
686 define_queries_inner! { <$tcx>
687 $($( $(#[$attr])* category<$category> [$($modifiers)*] fn $name: $node($K) -> $V,)*)*
692 macro_rules! define_queries_inner {
694 $($(#[$attr:meta])* category<$category:tt>
695 [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*) => {
698 #[cfg(parallel_compiler)]
699 use ty::query::job::QueryResult;
700 use rustc_data_structures::sync::Lock;
702 rustc_data_structures::stable_hasher::HashStable,
703 rustc_data_structures::stable_hasher::StableHasherResult,
704 rustc_data_structures::stable_hasher::StableHasher,
705 ich::StableHashingContext
707 use crate::util::profiling::ProfileCategory;
709 define_queries_struct! {
711 input: ($(([$($modifiers)*] [$($attr)*] [$name]))*)
714 impl<$tcx> Queries<$tcx> {
716 providers: IndexVec<CrateNum, Providers<$tcx>>,
717 fallback_extern_providers: Providers<$tcx>,
718 on_disk_cache: OnDiskCache<'tcx>,
722 fallback_extern_providers: Box::new(fallback_extern_providers),
724 $($name: Default::default()),*
728 #[cfg(parallel_compiler)]
729 pub fn collect_active_jobs(&self) -> Vec<Lrc<QueryJob<$tcx>>> {
730 let mut jobs = Vec::new();
732 // We use try_lock here since we are only called from the
733 // deadlock handler, and this shouldn't be locked.
736 self.$name.try_lock().unwrap().active.values().filter_map(|v|
737 if let QueryResult::Started(ref job) = *v {
749 pub fn print_stats(&self) {
750 let mut queries = Vec::new();
757 key_type: &'static str,
759 value_type: &'static str,
763 fn stats<'tcx, Q: QueryConfig<'tcx>>(
765 map: &QueryCache<'tcx, Q>
769 #[cfg(debug_assertions)]
770 cache_hits: map.cache_hits,
771 #[cfg(not(debug_assertions))]
773 key_size: mem::size_of::<Q::Key>(),
774 key_type: unsafe { type_name::<Q::Key>() },
775 value_size: mem::size_of::<Q::Value>(),
776 value_type: unsafe { type_name::<Q::Value>() },
777 entry_count: map.results.len(),
782 queries.push(stats::<queries::$name<'_>>(
788 if cfg!(debug_assertions) {
789 let hits: usize = queries.iter().map(|s| s.cache_hits).sum();
790 let results: usize = queries.iter().map(|s| s.entry_count).sum();
791 println!("\nQuery cache hit rate: {}", hits as f64 / (hits + results) as f64);
794 let mut query_key_sizes = queries.clone();
795 query_key_sizes.sort_by_key(|q| q.key_size);
796 println!("\nLarge query keys:");
797 for q in query_key_sizes.iter().rev()
798 .filter(|q| q.key_size > 8) {
800 " {} - {} x {} - {}",
808 let mut query_value_sizes = queries.clone();
809 query_value_sizes.sort_by_key(|q| q.value_size);
810 println!("\nLarge query values:");
811 for q in query_value_sizes.iter().rev()
812 .filter(|q| q.value_size > 8) {
814 " {} - {} x {} - {}",
822 if cfg!(debug_assertions) {
823 let mut query_cache_hits = queries.clone();
824 query_cache_hits.sort_by_key(|q| q.cache_hits);
825 println!("\nQuery cache hits:");
826 for q in query_cache_hits.iter().rev() {
831 q.cache_hits as f64 / (q.cache_hits + q.entry_count) as f64
836 let mut query_value_count = queries.clone();
837 query_value_count.sort_by_key(|q| q.entry_count);
838 println!("\nQuery value count:");
839 for q in query_value_count.iter().rev() {
840 println!(" {} - {}", q.name, q.entry_count);
845 #[allow(nonstandard_style)]
846 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
852 pub fn register_with_profiler(profiler: &crate::util::profiling::SelfProfiler) {
853 $(profiler.register_query_name(QueryName::$name);)*
856 pub fn as_str(&self) -> &'static str {
858 $(QueryName::$name => stringify!($name),)*
863 #[allow(nonstandard_style)]
864 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
865 pub enum Query<$tcx> {
866 $($(#[$attr])* $name($K)),*
869 impl<$tcx> Query<$tcx> {
870 pub fn name(&self) -> &'static str {
872 $(Query::$name(_) => stringify!($name),)*
876 pub fn describe(&self, tcx: TyCtxt<'_, '_>) -> Cow<'static, str> {
877 let (r, name) = match *self {
878 $(Query::$name(key) => {
879 (queries::$name::describe(tcx, key), stringify!($name))
882 if tcx.sess.verbose() {
883 format!("{} [{}]", r, name).into()
889 // FIXME(eddyb) Get more valid Span's on queries.
890 pub fn default_span(&self, tcx: TyCtxt<$tcx, '_>, span: Span) -> Span {
891 if !span.is_dummy() {
894 // The def_span query is used to calculate default_span,
895 // so exit to avoid infinite recursion
896 if let Query::def_span(..) = *self {
900 $(Query::$name(key) => key.default_span(tcx),)*
904 pub fn query_name(&self) -> QueryName {
906 $(Query::$name(_) => QueryName::$name,)*
911 impl<'a, $tcx> HashStable<StableHashingContext<'a>> for Query<$tcx> {
912 fn hash_stable<W: StableHasherResult>(&self,
913 hcx: &mut StableHashingContext<'a>,
914 hasher: &mut StableHasher<W>) {
915 mem::discriminant(self).hash_stable(hcx, hasher);
917 $(Query::$name(key) => key.hash_stable(hcx, hasher),)*
923 use std::marker::PhantomData;
925 $(#[allow(nonstandard_style)]
926 pub struct $name<$tcx> {
927 data: PhantomData<&$tcx ()>
931 // This module and the functions in it exist only to provide a
932 // predictable symbol name prefix for query providers. This is helpful
933 // for analyzing queries in profilers.
934 pub(super) mod __query_compute {
936 pub fn $name<F: FnOnce() -> R, R>(f: F) -> R {
941 $(impl<$tcx> QueryConfig<$tcx> for queries::$name<$tcx> {
945 const NAME: QueryName = QueryName::$name;
946 const CATEGORY: ProfileCategory = $category;
949 impl<$tcx> QueryAccessors<$tcx> for queries::$name<$tcx> {
951 fn query(key: Self::Key) -> Query<'tcx> {
956 fn query_cache<'a>(tcx: TyCtxt<$tcx, '_>) -> &'a Lock<QueryCache<$tcx, Self>> {
962 fn to_dep_node(tcx: TyCtxt<$tcx, '_>, key: &Self::Key) -> DepNode {
963 use crate::dep_graph::DepConstructor::*;
965 DepNode::new(tcx, $node(*key))
969 fn compute(tcx: TyCtxt<'tcx, '_>, key: Self::Key) -> Self::Value {
970 __query_compute::$name(move || {
971 let provider = tcx.queries.providers.get(key.query_crate())
972 // HACK(eddyb) it's possible crates may be loaded after
973 // the query engine is created, and because crate loading
974 // is not yet integrated with the query engine, such crates
975 // would be missing appropriate entries in `providers`.
976 .unwrap_or(&tcx.queries.fallback_extern_providers)
978 provider(tcx.global_tcx(), key)
983 _hcx: &mut StableHashingContext<'_>,
984 _result: &Self::Value
985 ) -> Option<Fingerprint> {
986 hash_result!([$($modifiers)*][_hcx, _result])
989 fn handle_cycle_error(
990 tcx: TyCtxt<'tcx, '_>,
991 error: CycleError<'tcx>
993 handle_cycle_error!([$($modifiers)*][tcx, error])
997 #[derive(Copy, Clone)]
998 pub struct TyCtxtEnsure<'gcx, 'tcx> {
999 pub tcx: TyCtxt<'gcx, 'tcx>,
1002 impl TyCtxtEnsure<$tcx, 'lcx> {
1005 pub fn $name(self, key: $K) {
1006 self.tcx.ensure_query::<queries::$name<'_>>(key)
1010 #[derive(Copy, Clone)]
1011 pub struct TyCtxtAt<'gcx, 'tcx> {
1012 pub tcx: TyCtxt<'gcx, 'tcx>,
1016 impl Deref for TyCtxtAt<'gcx, 'tcx> {
1017 type Target = TyCtxt<'gcx, 'tcx>;
1019 fn deref(&self) -> &Self::Target {
1024 impl TyCtxt<$tcx, 'lcx> {
1025 /// Returns a transparent wrapper for `TyCtxt`, which ensures queries
1026 /// are executed instead of just returing their results.
1028 pub fn ensure(self) -> TyCtxtEnsure<$tcx, 'lcx> {
1034 /// Returns a transparent wrapper for `TyCtxt` which uses
1035 /// `span` as the location of queries performed through it.
1037 pub fn at(self, span: Span) -> TyCtxtAt<$tcx, 'lcx> {
1046 pub fn $name(self, key: $K) -> $V {
1047 self.at(DUMMY_SP).$name(key)
1051 impl TyCtxtAt<$tcx, 'lcx> {
1054 pub fn $name(self, key: $K) -> $V {
1055 self.tcx.get_query::<queries::$name<'_>>(self.span, key)
1059 define_provider_struct! {
1061 input: ($(([$($modifiers)*] [$name] [$K] [$V]))*)
1064 impl<$tcx> Copy for Providers<$tcx> {}
1065 impl<$tcx> Clone for Providers<$tcx> {
1066 fn clone(&self) -> Self { *self }
1071 macro_rules! define_queries_struct {
1073 input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
1074 pub struct Queries<$tcx> {
1075 /// This provides access to the incrimental comilation on-disk cache for query results.
1076 /// Do not access this directly. It is only meant to be used by
1077 /// `DepGraph::try_mark_green()` and the query infrastructure.
1078 pub(crate) on_disk_cache: OnDiskCache<'tcx>,
1080 providers: IndexVec<CrateNum, Providers<$tcx>>,
1081 fallback_extern_providers: Box<Providers<$tcx>>,
1083 $($(#[$attr])* $name: Lock<QueryCache<$tcx, queries::$name<$tcx>>>,)*
1088 macro_rules! define_provider_struct {
1090 input: ($(([$($modifiers:tt)*] [$name:ident] [$K:ty] [$R:ty]))*)) => {
1091 pub struct Providers<$tcx> {
1092 $(pub $name: fn(TyCtxt<$tcx, $tcx>, $K) -> $R,)*
1095 impl<$tcx> Default for Providers<$tcx> {
1096 fn default() -> Self {
1097 $(fn $name<$tcx>(_: TyCtxt<$tcx, $tcx>, key: $K) -> $R {
1098 bug!("tcx.{}({:?}) unsupported by its crate",
1099 stringify!($name), key);
1101 Providers { $($name),* }
1108 /// The red/green evaluation system will try to mark a specific DepNode in the
1109 /// dependency graph as green by recursively trying to mark the dependencies of
1110 /// that DepNode as green. While doing so, it will sometimes encounter a DepNode
1111 /// where we don't know if it is red or green and we therefore actually have
1112 /// to recompute its value in order to find out. Since the only piece of
1113 /// information that we have at that point is the DepNode we are trying to
1114 /// re-evaluate, we need some way to re-run a query from just that. This is what
1115 /// `force_from_dep_node()` implements.
1117 /// In the general case, a DepNode consists of a DepKind and an opaque
1118 /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint
1119 /// is usually constructed by computing a stable hash of the query-key that the
1120 /// DepNode corresponds to. Consequently, it is not in general possible to go
1121 /// back from hash to query-key (since hash functions are not reversible). For
1122 /// this reason `force_from_dep_node()` is expected to fail from time to time
1123 /// because we just cannot find out, from the DepNode alone, what the
1124 /// corresponding query-key is and therefore cannot re-run the query.
1126 /// The system deals with this case letting `try_mark_green` fail which forces
1127 /// the root query to be re-evaluated.
1129 /// Now, if force_from_dep_node() would always fail, it would be pretty useless.
1130 /// Fortunately, we can use some contextual information that will allow us to
1131 /// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we
1132 /// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a
1133 /// valid `DefPathHash`. Since we also always build a huge table that maps every
1134 /// `DefPathHash` in the current codebase to the corresponding `DefId`, we have
1135 /// everything we need to re-run the query.
1137 /// Take the `mir_validated` query as an example. Like many other queries, it
1138 /// just has a single parameter: the `DefId` of the item it will compute the
1139 /// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode`
1140 /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode`
1141 /// is actually a `DefPathHash`, and can therefore just look up the corresponding
1142 /// `DefId` in `tcx.def_path_hash_to_def_id`.
1144 /// When you implement a new query, it will likely have a corresponding new
1145 /// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As
1146 /// a rule of thumb, if your query takes a `DefId` or `DefIndex` as sole parameter,
1147 /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
1148 /// add it to the "We don't have enough information to reconstruct..." group in
1149 /// the match below.
1150 pub fn force_from_dep_node<'tcx>(tcx: TyCtxt<'tcx, 'tcx>, dep_node: &DepNode) -> bool {
1151 use crate::dep_graph::RecoverKey;
1153 // We must avoid ever having to call force_from_dep_node() for a
1154 // DepNode::codegen_unit:
1155 // Since we cannot reconstruct the query key of a DepNode::codegen_unit, we
1156 // would always end up having to evaluate the first caller of the
1157 // `codegen_unit` query that *is* reconstructible. This might very well be
1158 // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
1159 // to re-trigger calling the `codegen_unit` query with the right key. At
1160 // that point we would already have re-done all the work we are trying to
1161 // avoid doing in the first place.
1162 // The solution is simple: Just explicitly call the `codegen_unit` query for
1163 // each CGU, right after partitioning. This way `try_mark_green` will always
1164 // hit the cache instead of having to go through `force_from_dep_node`.
1165 // This assertion makes sure, we actually keep applying the solution above.
1166 debug_assert!(dep_node.kind != DepKind::codegen_unit,
1167 "calling force_from_dep_node() on DepKind::codegen_unit");
1169 if !dep_node.kind.can_reconstruct_query_key() {
1173 macro_rules! def_id {
1175 if let Some(def_id) = dep_node.extract_def_id(tcx) {
1178 // return from the whole function
1184 macro_rules! krate {
1185 () => { (def_id!()).krate }
1188 macro_rules! force_ex {
1189 ($tcx:expr, $query:ident, $key:expr) => {
1191 $tcx.force_query::<crate::ty::query::queries::$query<'_>>(
1200 macro_rules! force {
1201 ($query:ident, $key:expr) => { force_ex!(tcx, $query, $key) }
1204 rustc_dep_node_force!([dep_node, tcx]
1205 // These are inputs that are expected to be pre-allocated and that
1206 // should therefore always be red or green already
1207 DepKind::AllLocalTraitImpls |
1209 DepKind::CrateMetadata |
1213 // This are anonymous nodes
1214 DepKind::TraitSelect |
1216 // We don't have enough information to reconstruct the query key of
1218 DepKind::CompileCodegenUnit => {
1219 bug!("force_from_dep_node() - Encountered {:?}", dep_node)
1222 DepKind::Analysis => { force!(analysis, krate!()); }
1229 // FIXME(#45015): Another piece of boilerplate code that could be generated in
1230 // a combined define_dep_nodes!()/define_queries!() macro.
1231 macro_rules! impl_load_from_cache {
1232 ($($dep_kind:ident => $query_name:ident,)*) => {
1234 // Check whether the query invocation corresponding to the given
1235 // DepNode is eligible for on-disk-caching.
1236 pub fn cache_on_disk(&self, tcx: TyCtxt<'_, '_>) -> bool {
1237 use crate::ty::query::queries;
1238 use crate::ty::query::QueryDescription;
1241 $(DepKind::$dep_kind => {
1242 let def_id = self.extract_def_id(tcx).unwrap();
1243 queries::$query_name::cache_on_disk(tcx.global_tcx(), def_id)
1249 // This is method will execute the query corresponding to the given
1250 // DepNode. It is only expected to work for DepNodes where the
1251 // above `cache_on_disk` methods returns true.
1252 // Also, as a sanity check, it expects that the corresponding query
1253 // invocation has been marked as green already.
1254 pub fn load_from_on_disk_cache(&self, tcx: TyCtxt<'_, '_>) {
1256 $(DepKind::$dep_kind => {
1257 debug_assert!(tcx.dep_graph
1259 .map(|c| c.is_green())
1262 let def_id = self.extract_def_id(tcx).unwrap();
1263 let _ = tcx.$query_name(def_id);
1274 impl_load_from_cache!(
1275 typeck_tables_of => typeck_tables_of,
1276 optimized_mir => optimized_mir,
1277 unsafety_check_result => unsafety_check_result,
1278 borrowck => borrowck,
1279 mir_borrowck => mir_borrowck,
1280 mir_const_qualif => mir_const_qualif,
1281 const_is_rvalue_promotable_to_static => const_is_rvalue_promotable_to_static,
1282 check_match => check_match,
1284 generics_of => generics_of,
1285 predicates_of => predicates_of,
1286 used_trait_imports => used_trait_imports,
1287 codegen_fn_attrs => codegen_fn_attrs,
1288 specialization_graph_of => specialization_graph_of,