src/librustc/ty/query/plumbing.rs

   1 //! The implementation of the query system itself. This defines the macros that
   2 //! generate the actual methods on tcx which find and execute the provider,
   3 //! manage the caches, and so forth.
   4
   5 use crate::dep_graph::{DepNodeIndex, DepNode, DepKind, SerializedDepNodeIndex};
   6 use crate::ty::tls;
   7 use crate::ty::{self, TyCtxt};
   8 use crate::ty::query::Query;
   9 use crate::ty::query::config::{QueryConfig, QueryDescription};
  10 use crate::ty::query::job::{QueryJob, QueryResult, QueryInfo};
  11
  12 use crate::util::common::{profq_msg, ProfileQueriesMsg, QueryMsg};
  13
  14 use errors::DiagnosticBuilder;
  15 use errors::Level;
  16 use errors::Diagnostic;
  17 use errors::FatalError;
  18 use rustc_data_structures::fx::{FxHashMap};
  19 use rustc_data_structures::sync::{Lrc, Lock};
  20 use rustc_data_structures::thin_vec::ThinVec;
  21 #[cfg(not(parallel_compiler))]
  22 use rustc_data_structures::cold_path;
  23 use std::mem;
  24 use std::ptr;
  25 use std::collections::hash_map::Entry;
  26 use syntax_pos::Span;
  27 use syntax::source_map::DUMMY_SP;
  28
  29 pub struct QueryCache<'tcx, D: QueryConfig<'tcx> + ?Sized> {
  30     pub(super) results: FxHashMap<D::Key, QueryValue<D::Value>>,
  31     pub(super) active: FxHashMap<D::Key, QueryResult<'tcx>>,
  32     #[cfg(debug_assertions)]
  33     pub(super) cache_hits: usize,
  34 }
  35
  36 pub(super) struct QueryValue<T> {
  37     pub(super) value: T,
  38     pub(super) index: DepNodeIndex,
  39 }
  40
  41 impl<T> QueryValue<T> {
  42     pub(super) fn new(value: T,
  43                       dep_node_index: DepNodeIndex)
  44                       -> QueryValue<T> {
  45         QueryValue {
  46             value,
  47             index: dep_node_index,
  48         }
  49     }
  50 }
  51
  52 impl<'tcx, M: QueryConfig<'tcx>> Default for QueryCache<'tcx, M> {
  53     fn default() -> QueryCache<'tcx, M> {
  54         QueryCache {
  55             results: FxHashMap::default(),
  56             active: FxHashMap::default(),
  57             #[cfg(debug_assertions)]
  58             cache_hits: 0,
  59         }
  60     }
  61 }
  62
  63 // If enabled, send a message to the profile-queries thread
  64 macro_rules! profq_msg {
  65     ($tcx:expr, $msg:expr) => {
  66         if cfg!(debug_assertions) {
  67             if $tcx.sess.profile_queries() {
  68                 profq_msg($tcx.sess, $msg)
  69             }
  70         }
  71     }
  72 }
  73
  74 // If enabled, format a key using its debug string, which can be
  75 // expensive to compute (in terms of time).
  76 macro_rules! profq_query_msg {
  77     ($query:expr, $tcx:expr, $key:expr) => {{
  78         let msg = if cfg!(debug_assertions) {
  79             if $tcx.sess.profile_queries_and_keys() {
  80                 Some(format!("{:?}", $key))
  81             } else { None }
  82         } else { None };
  83         QueryMsg {
  84             query: $query,
  85             msg,
  86         }
  87     }}
  88 }
  89
  90 /// A type representing the responsibility to execute the job in the `job` field.
  91 /// This will poison the relevant query if dropped.
  92 pub(super) struct JobOwner<'a, 'tcx: 'a, Q: QueryDescription<'tcx> + 'a> {
  93     cache: &'a Lock<QueryCache<'tcx, Q>>,
  94     key: Q::Key,
  95     job: Lrc<QueryJob<'tcx>>,
  96 }
  97
  98 impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> {
  99     /// Either gets a JobOwner corresponding the query, allowing us to
 100     /// start executing the query, or it returns with the result of the query.
 101     /// If the query is executing elsewhere, this will wait for it.
 102     /// If the query panicked, this will silently panic.
 103     ///
 104     /// This function is inlined because that results in a noticeable speedup
 105     /// for some compile-time benchmarks.
 106     #[inline(always)]
 107     pub(super) fn try_get(
 108         tcx: TyCtxt<'a, 'tcx, '_>,
 109         span: Span,
 110         key: &Q::Key,
 111     ) -> TryGetJob<'a, 'tcx, Q> {
 112         let cache = Q::query_cache(tcx);
 113         loop {
 114             let mut lock = cache.borrow_mut();
 115             if let Some(value) = lock.results.get(key) {
 116                 profq_msg!(tcx, ProfileQueriesMsg::CacheHit);
 117                 tcx.sess.profiler(|p| p.record_query_hit(Q::NAME, Q::CATEGORY));
 118                 let result = (value.value.clone(), value.index);
 119                 #[cfg(debug_assertions)]
 120                 {
 121                     lock.cache_hits += 1;
 122                 }
 123                 return TryGetJob::JobCompleted(result);
 124             }
 125             let job = match lock.active.entry((*key).clone()) {
 126                 Entry::Occupied(entry) => {
 127                     match *entry.get() {
 128                         QueryResult::Started(ref job) => {
 129                             //For parallel queries, we'll block and wait until the query running
 130                             //in another thread has completed. Record how long we wait in the
 131                             //self-profiler
 132                             #[cfg(parallel_compiler)]
 133                             tcx.sess.profiler(|p| p.query_blocked_start(Q::NAME, Q::CATEGORY));
 134
 135                             job.clone()
 136                         },
 137                         QueryResult::Poisoned => FatalError.raise(),
 138                     }
 139                 }
 140                 Entry::Vacant(entry) => {
 141                     // No job entry for this query. Return a new one to be started later
 142                     return tls::with_related_context(tcx, |icx| {
 143                         // Create the `parent` variable before `info`. This allows LLVM
 144                         // to elide the move of `info`
 145                         let parent = icx.query.clone();
 146                         let info = QueryInfo {
 147                             span,
 148                             query: Q::query(key.clone()),
 149                         };
 150                         let job = Lrc::new(QueryJob::new(info, parent));
 151                         let owner = JobOwner {
 152                             cache,
 153                             job: job.clone(),
 154                             key: (*key).clone(),
 155                         };
 156                         entry.insert(QueryResult::Started(job));
 157                         TryGetJob::NotYetStarted(owner)
 158                     })
 159                 }
 160             };
 161             mem::drop(lock);
 162
 163             // If we are single-threaded we know that we have cycle error,
 164             // so we just return the error
 165             #[cfg(not(parallel_compiler))]
 166             return TryGetJob::Cycle(cold_path(|| {
 167                 Q::handle_cycle_error(tcx, job.find_cycle_in_stack(tcx, span))
 168             }));
 169
 170             // With parallel queries we might just have to wait on some other
 171             // thread
 172             #[cfg(parallel_compiler)]
 173             {
 174                 let result = job.r#await(tcx, span);
 175                 tcx.sess.profiler(|p| p.query_blocked_end(Q::NAME, Q::CATEGORY));
 176
 177                 if let Err(cycle) = result {
 178                     return TryGetJob::Cycle(Q::handle_cycle_error(tcx, cycle));
 179                 }
 180             }
 181         }
 182     }
 183
 184     /// Completes the query by updating the query cache with the `result`,
 185     /// signals the waiter and forgets the JobOwner, so it won't poison the query
 186     #[inline(always)]
 187     pub(super) fn complete(self, result: &Q::Value, dep_node_index: DepNodeIndex) {
 188         // We can move out of `self` here because we `mem::forget` it below
 189         let key = unsafe { ptr::read(&self.key) };
 190         let job = unsafe { ptr::read(&self.job) };
 191         let cache = self.cache;
 192
 193         // Forget ourself so our destructor won't poison the query
 194         mem::forget(self);
 195
 196         let value = QueryValue::new(result.clone(), dep_node_index);
 197         {
 198             let mut lock = cache.borrow_mut();
 199             lock.active.remove(&key);
 200             lock.results.insert(key, value);
 201         }
 202
 203         job.signal_complete();
 204     }
 205 }
 206
 207 #[inline(always)]
 208 fn with_diagnostics<F, R>(f: F) -> (R, ThinVec<Diagnostic>)
 209 where
 210     F: FnOnce(Option<&Lock<ThinVec<Diagnostic>>>) -> R
 211 {
 212     let diagnostics = Lock::new(ThinVec::new());
 213     let result = f(Some(&diagnostics));
 214     (result, diagnostics.into_inner())
 215 }
 216
 217 impl<'a, 'tcx, Q: QueryDescription<'tcx>> Drop for JobOwner<'a, 'tcx, Q> {
 218     #[inline(never)]
 219     #[cold]
 220     fn drop(&mut self) {
 221         // Poison the query so jobs waiting on it panic
 222         self.cache.borrow_mut().active.insert(self.key.clone(), QueryResult::Poisoned);
 223         // Also signal the completion of the job, so waiters
 224         // will continue execution
 225         self.job.signal_complete();
 226     }
 227 }
 228
 229 #[derive(Clone)]
 230 pub struct CycleError<'tcx> {
 231     /// The query and related span which uses the cycle
 232     pub(super) usage: Option<(Span, Query<'tcx>)>,
 233     pub(super) cycle: Vec<QueryInfo<'tcx>>,
 234 }
 235
 236 /// The result of `try_get_lock`
 237 pub(super) enum TryGetJob<'a, 'tcx: 'a, D: QueryDescription<'tcx> + 'a> {
 238     /// The query is not yet started. Contains a guard to the cache eventually used to start it.
 239     NotYetStarted(JobOwner<'a, 'tcx, D>),
 240
 241     /// The query was already completed.
 242     /// Returns the result of the query and its dep node index
 243     /// if it succeeded or a cycle error if it failed
 244     JobCompleted((D::Value, DepNodeIndex)),
 245
 246     /// Trying to execute the query resulted in a cycle.
 247     Cycle(D::Value),
 248 }
 249
 250 impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
 251     /// Executes a job by changing the ImplicitCtxt to point to the
 252     /// new query job while it executes. It returns the diagnostics
 253     /// captured during execution and the actual result.
 254     #[inline(always)]
 255     pub(super) fn start_query<F, R>(
 256         self,
 257         job: Lrc<QueryJob<'gcx>>,
 258         diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
 259         compute: F)
 260     -> R
 261     where
 262         F: for<'b, 'lcx> FnOnce(TyCtxt<'b, 'gcx, 'lcx>) -> R
 263     {
 264         // The TyCtxt stored in TLS has the same global interner lifetime
 265         // as `self`, so we use `with_related_context` to relate the 'gcx lifetimes
 266         // when accessing the ImplicitCtxt
 267         tls::with_related_context(self, move |current_icx| {
 268             // Update the ImplicitCtxt to point to our new query job
 269             let new_icx = tls::ImplicitCtxt {
 270                 tcx: self.global_tcx(),
 271                 query: Some(job),
 272                 diagnostics,
 273                 layout_depth: current_icx.layout_depth,
 274                 task_deps: current_icx.task_deps,
 275             };
 276
 277             // Use the ImplicitCtxt while we execute the query
 278             tls::enter_context(&new_icx, |_| {
 279                 compute(self.global_tcx())
 280             })
 281         })
 282     }
 283
 284     #[inline(never)]
 285     #[cold]
 286     pub(super) fn report_cycle(
 287         self,
 288         CycleError { usage, cycle: stack }: CycleError<'gcx>
 289     ) -> DiagnosticBuilder<'a>
 290     {
 291         assert!(!stack.is_empty());
 292
 293         let fix_span = |span: Span, query: &Query<'gcx>| {
 294             self.sess.source_map().def_span(query.default_span(self, span))
 295         };
 296
 297         // Disable naming impls with types in this path, since that
 298         // sometimes cycles itself, leading to extra cycle errors.
 299         // (And cycle errors around impls tend to occur during the
 300         // collect/coherence phases anyhow.)
 301         ty::print::with_forced_impl_filename_line(|| {
 302             let span = fix_span(stack[1 % stack.len()].span, &stack[0].query);
 303             let mut err = struct_span_err!(self.sess,
 304                                            span,
 305                                            E0391,
 306                                            "cycle detected when {}",
 307                                            stack[0].query.describe(self));
 308
 309             for i in 1..stack.len() {
 310                 let query = &stack[i].query;
 311                 let span = fix_span(stack[(i + 1) % stack.len()].span, query);
 312                 err.span_note(span, &format!("...which requires {}...", query.describe(self)));
 313             }
 314
 315             err.note(&format!("...which again requires {}, completing the cycle",
 316                               stack[0].query.describe(self)));
 317
 318             if let Some((span, query)) = usage {
 319                 err.span_note(fix_span(span, &query),
 320                               &format!("cycle used when {}", query.describe(self)));
 321             }
 322
 323             err
 324         })
 325     }
 326
 327     pub fn try_print_query_stack() {
 328         eprintln!("query stack during panic:");
 329
 330         tls::with_context_opt(|icx| {
 331             if let Some(icx) = icx {
 332                 let mut current_query = icx.query.clone();
 333                 let mut i = 0;
 334
 335                 while let Some(query) = current_query {
 336                     let mut db = DiagnosticBuilder::new(icx.tcx.sess.diagnostic(),
 337                         Level::FailureNote,
 338                         &format!("#{} [{}] {}",
 339                                  i,
 340                                  query.info.query.name(),
 341                                  query.info.query.describe(icx.tcx)));
 342                     db.set_span(icx.tcx.sess.source_map().def_span(query.info.span));
 343                     icx.tcx.sess.diagnostic().force_print_db(db);
 344
 345                     current_query = query.parent.clone();
 346                     i += 1;
 347                 }
 348             }
 349         });
 350
 351         eprintln!("end of query stack");
 352     }
 353
 354     #[inline(never)]
 355     pub(super) fn get_query<Q: QueryDescription<'gcx>>(
 356         self,
 357         span: Span,
 358         key: Q::Key)
 359     -> Q::Value {
 360         debug!("ty::query::get_query<{}>(key={:?}, span={:?})",
 361                Q::NAME,
 362                key,
 363                span);
 364
 365         profq_msg!(self,
 366             ProfileQueriesMsg::QueryBegin(
 367                 span.data(),
 368                 profq_query_msg!(Q::NAME, self, key),
 369             )
 370         );
 371
 372         let job = match JobOwner::try_get(self, span, &key) {
 373             TryGetJob::NotYetStarted(job) => job,
 374             TryGetJob::Cycle(result) => return result,
 375             TryGetJob::JobCompleted((v, index)) => {
 376                 self.dep_graph.read_index(index);
 377                 return v
 378             }
 379         };
 380
 381         // Fast path for when incr. comp. is off. `to_dep_node` is
 382         // expensive for some DepKinds.
 383         if !self.dep_graph.is_fully_enabled() {
 384             let null_dep_node = DepNode::new_no_params(crate::dep_graph::DepKind::Null);
 385             return self.force_query_with_job::<Q>(key, job, null_dep_node).0;
 386         }
 387
 388         let dep_node = Q::to_dep_node(self, &key);
 389
 390         if dep_node.kind.is_anon() {
 391             profq_msg!(self, ProfileQueriesMsg::ProviderBegin);
 392             self.sess.profiler(|p| p.start_query(Q::NAME, Q::CATEGORY));
 393
 394             let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
 395                 self.start_query(job.job.clone(), diagnostics, |tcx| {
 396                     tcx.dep_graph.with_anon_task(dep_node.kind, || {
 397                         Q::compute(tcx.global_tcx(), key)
 398                     })
 399                 })
 400             });
 401
 402             self.sess.profiler(|p| p.end_query(Q::NAME, Q::CATEGORY));
 403             profq_msg!(self, ProfileQueriesMsg::ProviderEnd);
 404
 405             self.dep_graph.read_index(dep_node_index);
 406
 407             if unlikely!(!diagnostics.is_empty()) {
 408                 self.queries.on_disk_cache
 409                     .store_diagnostics_for_anon_node(dep_node_index, diagnostics);
 410             }
 411
 412             job.complete(&result, dep_node_index);
 413
 414             return result;
 415         }
 416
 417         if !dep_node.kind.is_input() {
 418             // The diagnostics for this query will be
 419             // promoted to the current session during
 420             // try_mark_green(), so we can ignore them here.
 421             let loaded = self.start_query(job.job.clone(), None, |tcx| {
 422                 let marked = tcx.dep_graph.try_mark_green_and_read(tcx, &dep_node);
 423                 marked.map(|(prev_dep_node_index, dep_node_index)| {
 424                     (tcx.load_from_disk_and_cache_in_memory::<Q>(
 425                         key.clone(),
 426                         prev_dep_node_index,
 427                         dep_node_index,
 428                         &dep_node
 429                     ), dep_node_index)
 430                 })
 431             });
 432             if let Some((result, dep_node_index)) = loaded {
 433                 job.complete(&result, dep_node_index);
 434                 return result;
 435             }
 436         }
 437
 438         let (result, dep_node_index) = self.force_query_with_job::<Q>(key, job, dep_node);
 439         self.dep_graph.read_index(dep_node_index);
 440         result
 441     }
 442
 443     fn load_from_disk_and_cache_in_memory<Q: QueryDescription<'gcx>>(
 444         self,
 445         key: Q::Key,
 446         prev_dep_node_index: SerializedDepNodeIndex,
 447         dep_node_index: DepNodeIndex,
 448         dep_node: &DepNode
 449     ) -> Q::Value
 450     {
 451         // Note this function can be called concurrently from the same query
 452         // We must ensure that this is handled correctly
 453
 454         debug_assert!(self.dep_graph.is_green(dep_node));
 455
 456         // First we try to load the result from the on-disk cache
 457         let result = if Q::cache_on_disk(self.global_tcx(), key.clone()) &&
 458                         self.sess.opts.debugging_opts.incremental_queries {
 459             self.sess.profiler(|p| p.incremental_load_result_start(Q::NAME));
 460             let result = Q::try_load_from_disk(self.global_tcx(), prev_dep_node_index);
 461             self.sess.profiler(|p| p.incremental_load_result_end(Q::NAME));
 462
 463             // We always expect to find a cached result for things that
 464             // can be forced from DepNode.
 465             debug_assert!(!dep_node.kind.can_reconstruct_query_key() ||
 466                           result.is_some(),
 467                           "Missing on-disk cache entry for {:?}",
 468                           dep_node);
 469             result
 470         } else {
 471             // Some things are never cached on disk.
 472             None
 473         };
 474
 475         let result = if let Some(result) = result {
 476             profq_msg!(self, ProfileQueriesMsg::CacheHit);
 477             self.sess.profiler(|p| p.record_query_hit(Q::NAME, Q::CATEGORY));
 478
 479             result
 480         } else {
 481             // We could not load a result from the on-disk cache, so
 482             // recompute.
 483
 484             self.sess.profiler(|p| p.start_query(Q::NAME, Q::CATEGORY));
 485
 486             // The dep-graph for this computation is already in
 487             // place
 488             let result = self.dep_graph.with_ignore(|| {
 489                 Q::compute(self, key)
 490             });
 491
 492             self.sess.profiler(|p| p.end_query(Q::NAME, Q::CATEGORY));
 493             result
 494         };
 495
 496         // If -Zincremental-verify-ich is specified, re-hash results from
 497         // the cache and make sure that they have the expected fingerprint.
 498         if unlikely!(self.sess.opts.debugging_opts.incremental_verify_ich) {
 499             self.incremental_verify_ich::<Q>(&result, dep_node, dep_node_index);
 500         }
 501
 502         if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) {
 503             self.dep_graph.mark_loaded_from_cache(dep_node_index, true);
 504         }
 505
 506         result
 507     }
 508
 509     #[inline(never)]
 510     #[cold]
 511     fn incremental_verify_ich<Q: QueryDescription<'gcx>>(
 512         self,
 513         result: &Q::Value,
 514         dep_node: &DepNode,
 515         dep_node_index: DepNodeIndex,
 516     ) {
 517         use crate::ich::Fingerprint;
 518
 519         assert!(Some(self.dep_graph.fingerprint_of(dep_node_index)) ==
 520                 self.dep_graph.prev_fingerprint_of(dep_node),
 521                 "Fingerprint for green query instance not loaded \
 522                     from cache: {:?}", dep_node);
 523
 524         debug!("BEGIN verify_ich({:?})", dep_node);
 525         let mut hcx = self.create_stable_hashing_context();
 526
 527         let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
 528         debug!("END verify_ich({:?})", dep_node);
 529
 530         let old_hash = self.dep_graph.fingerprint_of(dep_node_index);
 531
 532         assert!(new_hash == old_hash, "Found unstable fingerprints \
 533             for {:?}", dep_node);
 534     }
 535
 536     #[inline(always)]
 537     fn force_query_with_job<Q: QueryDescription<'gcx>>(
 538         self,
 539         key: Q::Key,
 540         job: JobOwner<'_, 'gcx, Q>,
 541         dep_node: DepNode)
 542     -> (Q::Value, DepNodeIndex) {
 543         // If the following assertion triggers, it can have two reasons:
 544         // 1. Something is wrong with DepNode creation, either here or
 545         //    in DepGraph::try_mark_green()
 546         // 2. Two distinct query keys get mapped to the same DepNode
 547         //    (see for example #48923)
 548         assert!(!self.dep_graph.dep_node_exists(&dep_node),
 549                 "Forcing query with already existing DepNode.\n\
 550                  - query-key: {:?}\n\
 551                  - dep-node: {:?}",
 552                 key, dep_node);
 553
 554         profq_msg!(self, ProfileQueriesMsg::ProviderBegin);
 555         self.sess.profiler(|p| p.start_query(Q::NAME, Q::CATEGORY));
 556
 557         let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
 558             self.start_query(job.job.clone(), diagnostics, |tcx| {
 559                 if dep_node.kind.is_eval_always() {
 560                     tcx.dep_graph.with_eval_always_task(dep_node,
 561                                                         tcx,
 562                                                         key,
 563                                                         Q::compute,
 564                                                         Q::hash_result)
 565                 } else {
 566                     tcx.dep_graph.with_task(dep_node,
 567                                             tcx,
 568                                             key,
 569                                             Q::compute,
 570                                             Q::hash_result)
 571                 }
 572             })
 573         });
 574
 575         self.sess.profiler(|p| p.end_query(Q::NAME, Q::CATEGORY));
 576         profq_msg!(self, ProfileQueriesMsg::ProviderEnd);
 577
 578         if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) {
 579             self.dep_graph.mark_loaded_from_cache(dep_node_index, false);
 580         }
 581
 582         if dep_node.kind != crate::dep_graph::DepKind::Null {
 583             if unlikely!(!diagnostics.is_empty()) {
 584                 self.queries.on_disk_cache
 585                     .store_diagnostics(dep_node_index, diagnostics);
 586             }
 587         }
 588
 589         job.complete(&result, dep_node_index);
 590
 591         (result, dep_node_index)
 592     }
 593
 594     /// Ensure that either this query has all green inputs or been executed.
 595     /// Executing query::ensure(D) is considered a read of the dep-node D.
 596     ///
 597     /// This function is particularly useful when executing passes for their
 598     /// side-effects -- e.g., in order to report errors for erroneous programs.
 599     ///
 600     /// Note: The optimization is only available during incr. comp.
 601     pub(super) fn ensure_query<Q: QueryDescription<'gcx>>(self, key: Q::Key) -> () {
 602         let dep_node = Q::to_dep_node(self, &key);
 603
 604         // Ensuring an "input" or anonymous query makes no sense
 605         assert!(!dep_node.kind.is_anon());
 606         assert!(!dep_node.kind.is_input());
 607         if self.dep_graph.try_mark_green_and_read(self, &dep_node).is_none() {
 608             // A None return from `try_mark_green_and_read` means that this is either
 609             // a new dep node or that the dep node has already been marked red.
 610             // Either way, we can't call `dep_graph.read()` as we don't have the
 611             // DepNodeIndex. We must invoke the query itself. The performance cost
 612             // this introduces should be negligible as we'll immediately hit the
 613             // in-memory cache, or another query down the line will.
 614
 615             let _ = self.get_query::<Q>(DUMMY_SP, key);
 616         } else {
 617             profq_msg!(self, ProfileQueriesMsg::CacheHit);
 618             self.sess.profiler(|p| p.record_query_hit(Q::NAME, Q::CATEGORY));
 619         }
 620     }
 621
 622     #[allow(dead_code)]
 623     fn force_query<Q: QueryDescription<'gcx>>(
 624         self,
 625         key: Q::Key,
 626         span: Span,
 627         dep_node: DepNode
 628     ) {
 629         profq_msg!(
 630             self,
 631             ProfileQueriesMsg::QueryBegin(span.data(), profq_query_msg!(Q::NAME, self, key))
 632         );
 633
 634         // We may be concurrently trying both execute and force a query
 635         // Ensure that only one of them runs the query
 636         let job = match JobOwner::try_get(self, span, &key) {
 637             TryGetJob::NotYetStarted(job) => job,
 638             TryGetJob::Cycle(_) |
 639             TryGetJob::JobCompleted(_) => {
 640                 return
 641             }
 642         };
 643         self.force_query_with_job::<Q>(key, job, dep_node);
 644     }
 645 }
 646
 647 macro_rules! handle_cycle_error {
 648     ([][$tcx: expr, $error:expr]) => {{
 649         $tcx.report_cycle($error).emit();
 650         Value::from_cycle_error($tcx.global_tcx())
 651     }};
 652     ([fatal_cycle$(, $modifiers:ident)*][$tcx:expr, $error:expr]) => {{
 653         $tcx.report_cycle($error).emit();
 654         $tcx.sess.abort_if_errors();
 655         unreachable!()
 656     }};
 657     ([cycle_delay_bug$(, $modifiers:ident)*][$tcx:expr, $error:expr]) => {{
 658         $tcx.report_cycle($error).delay_as_bug();
 659         Value::from_cycle_error($tcx.global_tcx())
 660     }};
 661     ([$other:ident$(, $modifiers:ident)*][$($args:tt)*]) => {
 662         handle_cycle_error!([$($modifiers),*][$($args)*])
 663     };
 664 }
 665
 666 macro_rules! hash_result {
 667     ([][$hcx:expr, $result:expr]) => {{
 668         dep_graph::hash_result($hcx, &$result)
 669     }};
 670     ([no_hash$(, $modifiers:ident)*][$hcx:expr, $result:expr]) => {{
 671         None
 672     }};
 673     ([$other:ident$(, $modifiers:ident)*][$($args:tt)*]) => {
 674         hash_result!([$($modifiers),*][$($args)*])
 675     };
 676 }
 677
 678 macro_rules! define_queries {
 679     (<$tcx:tt> $($category:tt {
 680         $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*
 681     },)*) => {
 682         define_queries_inner! { <$tcx>
 683             $($( $(#[$attr])* category<$category> [$($modifiers)*] fn $name: $node($K) -> $V,)*)*
 684         }
 685     }
 686 }
 687
 688 macro_rules! define_queries_inner {
 689     (<$tcx:tt>
 690      $($(#[$attr:meta])* category<$category:tt>
 691         [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*) => {
 692
 693         use std::mem;
 694         #[cfg(parallel_compiler)]
 695         use ty::query::job::QueryResult;
 696         use rustc_data_structures::sync::Lock;
 697         use crate::{
 698             rustc_data_structures::stable_hasher::HashStable,
 699             rustc_data_structures::stable_hasher::StableHasherResult,
 700             rustc_data_structures::stable_hasher::StableHasher,
 701             ich::StableHashingContext
 702         };
 703         use crate::util::profiling::ProfileCategory;
 704
 705         define_queries_struct! {
 706             tcx: $tcx,
 707             input: ($(([$($modifiers)*] [$($attr)*] [$name]))*)
 708         }
 709
 710         impl<$tcx> Queries<$tcx> {
 711             pub fn new(
 712                 providers: IndexVec<CrateNum, Providers<$tcx>>,
 713                 fallback_extern_providers: Providers<$tcx>,
 714                 on_disk_cache: OnDiskCache<'tcx>,
 715             ) -> Self {
 716                 Queries {
 717                     providers,
 718                     fallback_extern_providers: Box::new(fallback_extern_providers),
 719                     on_disk_cache,
 720                     $($name: Default::default()),*
 721                 }
 722             }
 723
 724             pub fn record_computed_queries(&self, sess: &Session) {
 725                 sess.profiler(|p| {
 726                     $(
 727                         p.record_computed_queries(
 728                             <queries::$name<'_> as QueryConfig<'_>>::NAME,
 729                             <queries::$name<'_> as QueryConfig<'_>>::CATEGORY,
 730                             self.$name.lock().results.len()
 731                         );
 732                     )*
 733                 });
 734             }
 735
 736             #[cfg(parallel_compiler)]
 737             pub fn collect_active_jobs(&self) -> Vec<Lrc<QueryJob<$tcx>>> {
 738                 let mut jobs = Vec::new();
 739
 740                 // We use try_lock here since we are only called from the
 741                 // deadlock handler, and this shouldn't be locked
 742                 $(
 743                     jobs.extend(
 744                         self.$name.try_lock().unwrap().active.values().filter_map(|v|
 745                             if let QueryResult::Started(ref job) = *v {
 746                                 Some(job.clone())
 747                             } else {
 748                                 None
 749                             }
 750                         )
 751                     );
 752                 )*
 753
 754                 jobs
 755             }
 756
 757             pub fn print_stats(&self) {
 758                 let mut queries = Vec::new();
 759
 760                 #[derive(Clone)]
 761                 struct QueryStats {
 762                     name: &'static str,
 763                     cache_hits: usize,
 764                     key_size: usize,
 765                     key_type: &'static str,
 766                     value_size: usize,
 767                     value_type: &'static str,
 768                     entry_count: usize,
 769                 }
 770
 771                 fn stats<'tcx, Q: QueryConfig<'tcx>>(
 772                     name: &'static str,
 773                     map: &QueryCache<'tcx, Q>
 774                 ) -> QueryStats {
 775                     QueryStats {
 776                         name,
 777                         #[cfg(debug_assertions)]
 778                         cache_hits: map.cache_hits,
 779                         #[cfg(not(debug_assertions))]
 780                         cache_hits: 0,
 781                         key_size: mem::size_of::<Q::Key>(),
 782                         key_type: unsafe { type_name::<Q::Key>() },
 783                         value_size: mem::size_of::<Q::Value>(),
 784                         value_type: unsafe { type_name::<Q::Value>() },
 785                         entry_count: map.results.len(),
 786                     }
 787                 }
 788
 789                 $(
 790                     queries.push(stats::<queries::$name<'_>>(
 791                         stringify!($name),
 792                         &*self.$name.lock()
 793                     ));
 794                 )*
 795
 796                 if cfg!(debug_assertions) {
 797                     let hits: usize = queries.iter().map(|s| s.cache_hits).sum();
 798                     let results: usize = queries.iter().map(|s| s.entry_count).sum();
 799                     println!("\nQuery cache hit rate: {}", hits as f64 / (hits + results) as f64);
 800                 }
 801
 802                 let mut query_key_sizes = queries.clone();
 803                 query_key_sizes.sort_by_key(|q| q.key_size);
 804                 println!("\nLarge query keys:");
 805                 for q in query_key_sizes.iter().rev()
 806                                         .filter(|q| q.key_size > 8) {
 807                     println!(
 808                         "   {} - {} x {} - {}",
 809                         q.name,
 810                         q.key_size,
 811                         q.entry_count,
 812                         q.key_type
 813                     );
 814                 }
 815
 816                 let mut query_value_sizes = queries.clone();
 817                 query_value_sizes.sort_by_key(|q| q.value_size);
 818                 println!("\nLarge query values:");
 819                 for q in query_value_sizes.iter().rev()
 820                                           .filter(|q| q.value_size > 8) {
 821                     println!(
 822                         "   {} - {} x {} - {}",
 823                         q.name,
 824                         q.value_size,
 825                         q.entry_count,
 826                         q.value_type
 827                     );
 828                 }
 829
 830                 if cfg!(debug_assertions) {
 831                     let mut query_cache_hits = queries.clone();
 832                     query_cache_hits.sort_by_key(|q| q.cache_hits);
 833                     println!("\nQuery cache hits:");
 834                     for q in query_cache_hits.iter().rev() {
 835                         println!(
 836                             "   {} - {} ({}%)",
 837                             q.name,
 838                             q.cache_hits,
 839                             q.cache_hits as f64 / (q.cache_hits + q.entry_count) as f64
 840                         );
 841                     }
 842                 }
 843
 844                 let mut query_value_count = queries.clone();
 845                 query_value_count.sort_by_key(|q| q.entry_count);
 846                 println!("\nQuery value count:");
 847                 for q in query_value_count.iter().rev() {
 848                     println!("   {} - {}", q.name, q.entry_count);
 849                 }
 850             }
 851         }
 852
 853         #[allow(nonstandard_style)]
 854         #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 855         pub enum Query<$tcx> {
 856             $($(#[$attr])* $name($K)),*
 857         }
 858
 859         impl<$tcx> Query<$tcx> {
 860             pub fn name(&self) -> &'static str {
 861                 match *self {
 862                     $(Query::$name(_) => stringify!($name),)*
 863                 }
 864             }
 865
 866             pub fn describe(&self, tcx: TyCtxt<'_, '_, '_>) -> Cow<'static, str> {
 867                 let (r, name) = match *self {
 868                     $(Query::$name(key) => {
 869                         (queries::$name::describe(tcx, key), stringify!($name))
 870                     })*
 871                 };
 872                 if tcx.sess.verbose() {
 873                     format!("{} [{}]", r, name).into()
 874                 } else {
 875                     r
 876                 }
 877             }
 878
 879             // FIXME(eddyb) Get more valid Span's on queries.
 880             pub fn default_span(&self, tcx: TyCtxt<'_, $tcx, '_>, span: Span) -> Span {
 881                 if !span.is_dummy() {
 882                     return span;
 883                 }
 884                 // The def_span query is used to calculate default_span,
 885                 // so exit to avoid infinite recursion
 886                 if let Query::def_span(..) = *self {
 887                     return span
 888                 }
 889                 match *self {
 890                     $(Query::$name(key) => key.default_span(tcx),)*
 891                 }
 892             }
 893         }
 894
 895         impl<'a, $tcx> HashStable<StableHashingContext<'a>> for Query<$tcx> {
 896             fn hash_stable<W: StableHasherResult>(&self,
 897                                                 hcx: &mut StableHashingContext<'a>,
 898                                                 hasher: &mut StableHasher<W>) {
 899                 mem::discriminant(self).hash_stable(hcx, hasher);
 900                 match *self {
 901                     $(Query::$name(key) => key.hash_stable(hcx, hasher),)*
 902                 }
 903             }
 904         }
 905
 906         pub mod queries {
 907             use std::marker::PhantomData;
 908
 909             $(#[allow(nonstandard_style)]
 910             pub struct $name<$tcx> {
 911                 data: PhantomData<&$tcx ()>
 912             })*
 913         }
 914
 915         // This module and the functions in it exist only to provide a
 916         // predictable symbol name prefix for query providers. This is helpful
 917         // for analyzing queries in profilers.
 918         pub(super) mod __query_compute {
 919             $(#[inline(never)]
 920             pub fn $name<F: FnOnce() -> R, R>(f: F) -> R {
 921                 f()
 922             })*
 923         }
 924
 925         $(impl<$tcx> QueryConfig<$tcx> for queries::$name<$tcx> {
 926             type Key = $K;
 927             type Value = $V;
 928
 929             const NAME: &'static str = stringify!($name);
 930             const CATEGORY: ProfileCategory = $category;
 931         }
 932
 933         impl<$tcx> QueryAccessors<$tcx> for queries::$name<$tcx> {
 934             #[inline(always)]
 935             fn query(key: Self::Key) -> Query<'tcx> {
 936                 Query::$name(key)
 937             }
 938
 939             #[inline(always)]
 940             fn query_cache<'a>(tcx: TyCtxt<'a, $tcx, '_>) -> &'a Lock<QueryCache<$tcx, Self>> {
 941                 &tcx.queries.$name
 942             }
 943
 944             #[allow(unused)]
 945             #[inline(always)]
 946             fn to_dep_node(tcx: TyCtxt<'_, $tcx, '_>, key: &Self::Key) -> DepNode {
 947                 use crate::dep_graph::DepConstructor::*;
 948
 949                 DepNode::new(tcx, $node(*key))
 950             }
 951
 952             #[inline]
 953             fn compute(tcx: TyCtxt<'_, 'tcx, '_>, key: Self::Key) -> Self::Value {
 954                 __query_compute::$name(move || {
 955                     let provider = tcx.queries.providers.get(key.query_crate())
 956                         // HACK(eddyb) it's possible crates may be loaded after
 957                         // the query engine is created, and because crate loading
 958                         // is not yet integrated with the query engine, such crates
 959                         // would be missing appropriate entries in `providers`.
 960                         .unwrap_or(&tcx.queries.fallback_extern_providers)
 961                         .$name;
 962                     provider(tcx.global_tcx(), key)
 963                 })
 964             }
 965
 966             fn hash_result(
 967                 _hcx: &mut StableHashingContext<'_>,
 968                 _result: &Self::Value
 969             ) -> Option<Fingerprint> {
 970                 hash_result!([$($modifiers)*][_hcx, _result])
 971             }
 972
 973             fn handle_cycle_error(
 974                 tcx: TyCtxt<'_, 'tcx, '_>,
 975                 error: CycleError<'tcx>
 976             ) -> Self::Value {
 977                 handle_cycle_error!([$($modifiers)*][tcx, error])
 978             }
 979         })*
 980
 981         #[derive(Copy, Clone)]
 982         pub struct TyCtxtEnsure<'a, 'gcx: 'a+'tcx, 'tcx: 'a> {
 983             pub tcx: TyCtxt<'a, 'gcx, 'tcx>,
 984         }
 985
 986         impl<'a, $tcx, 'lcx> TyCtxtEnsure<'a, $tcx, 'lcx> {
 987             $($(#[$attr])*
 988             #[inline(always)]
 989             pub fn $name(self, key: $K) {
 990                 self.tcx.ensure_query::<queries::$name<'_>>(key)
 991             })*
 992         }
 993
 994         #[derive(Copy, Clone)]
 995         pub struct TyCtxtAt<'a, 'gcx: 'a+'tcx, 'tcx: 'a> {
 996             pub tcx: TyCtxt<'a, 'gcx, 'tcx>,
 997             pub span: Span,
 998         }
 999
1000         impl<'a, 'gcx, 'tcx> Deref for TyCtxtAt<'a, 'gcx, 'tcx> {
1001             type Target = TyCtxt<'a, 'gcx, 'tcx>;
1002             #[inline(always)]
1003             fn deref(&self) -> &Self::Target {
1004                 &self.tcx
1005             }
1006         }
1007
1008         impl<'a, $tcx, 'lcx> TyCtxt<'a, $tcx, 'lcx> {
1009             /// Returns a transparent wrapper for `TyCtxt`, which ensures queries
1010             /// are executed instead of just returing their results.
1011             #[inline(always)]
1012             pub fn ensure(self) -> TyCtxtEnsure<'a, $tcx, 'lcx> {
1013                 TyCtxtEnsure {
1014                     tcx: self,
1015                 }
1016             }
1017
1018             /// Returns a transparent wrapper for `TyCtxt` which uses
1019             /// `span` as the location of queries performed through it.
1020             #[inline(always)]
1021             pub fn at(self, span: Span) -> TyCtxtAt<'a, $tcx, 'lcx> {
1022                 TyCtxtAt {
1023                     tcx: self,
1024                     span
1025                 }
1026             }
1027
1028             $($(#[$attr])*
1029             #[inline(always)]
1030             pub fn $name(self, key: $K) -> $V {
1031                 self.at(DUMMY_SP).$name(key)
1032             })*
1033         }
1034
1035         impl<'a, $tcx, 'lcx> TyCtxtAt<'a, $tcx, 'lcx> {
1036             $($(#[$attr])*
1037             #[inline(always)]
1038             pub fn $name(self, key: $K) -> $V {
1039                 self.tcx.get_query::<queries::$name<'_>>(self.span, key)
1040             })*
1041         }
1042
1043         define_provider_struct! {
1044             tcx: $tcx,
1045             input: ($(([$($modifiers)*] [$name] [$K] [$V]))*)
1046         }
1047
1048         impl<$tcx> Copy for Providers<$tcx> {}
1049         impl<$tcx> Clone for Providers<$tcx> {
1050             fn clone(&self) -> Self { *self }
1051         }
1052     }
1053 }
1054
1055 macro_rules! define_queries_struct {
1056     (tcx: $tcx:tt,
1057      input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
1058         pub struct Queries<$tcx> {
1059             /// This provides access to the incrimental comilation on-disk cache for query results.
1060             /// Do not access this directly. It is only meant to be used by
1061             /// `DepGraph::try_mark_green()` and the query infrastructure.
1062             pub(crate) on_disk_cache: OnDiskCache<'tcx>,
1063
1064             providers: IndexVec<CrateNum, Providers<$tcx>>,
1065             fallback_extern_providers: Box<Providers<$tcx>>,
1066
1067             $($(#[$attr])*  $name: Lock<QueryCache<$tcx, queries::$name<$tcx>>>,)*
1068         }
1069     };
1070 }
1071
1072 macro_rules! define_provider_struct {
1073     (tcx: $tcx:tt,
1074      input: ($(([$($modifiers:tt)*] [$name:ident] [$K:ty] [$R:ty]))*)) => {
1075         pub struct Providers<$tcx> {
1076             $(pub $name: for<'a> fn(TyCtxt<'a, $tcx, $tcx>, $K) -> $R,)*
1077         }
1078
1079         impl<$tcx> Default for Providers<$tcx> {
1080             fn default() -> Self {
1081                 $(fn $name<'a, $tcx>(_: TyCtxt<'a, $tcx, $tcx>, key: $K) -> $R {
1082                     bug!("tcx.{}({:?}) unsupported by its crate",
1083                          stringify!($name), key);
1084                 })*
1085                 Providers { $($name),* }
1086             }
1087         }
1088     };
1089 }
1090
1091
1092 /// The red/green evaluation system will try to mark a specific DepNode in the
1093 /// dependency graph as green by recursively trying to mark the dependencies of
1094 /// that DepNode as green. While doing so, it will sometimes encounter a DepNode
1095 /// where we don't know if it is red or green and we therefore actually have
1096 /// to recompute its value in order to find out. Since the only piece of
1097 /// information that we have at that point is the DepNode we are trying to
1098 /// re-evaluate, we need some way to re-run a query from just that. This is what
1099 /// `force_from_dep_node()` implements.
1100 ///
1101 /// In the general case, a DepNode consists of a DepKind and an opaque
1102 /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint
1103 /// is usually constructed by computing a stable hash of the query-key that the
1104 /// DepNode corresponds to. Consequently, it is not in general possible to go
1105 /// back from hash to query-key (since hash functions are not reversible). For
1106 /// this reason `force_from_dep_node()` is expected to fail from time to time
1107 /// because we just cannot find out, from the DepNode alone, what the
1108 /// corresponding query-key is and therefore cannot re-run the query.
1109 ///
1110 /// The system deals with this case letting `try_mark_green` fail which forces
1111 /// the root query to be re-evaluated.
1112 ///
1113 /// Now, if force_from_dep_node() would always fail, it would be pretty useless.
1114 /// Fortunately, we can use some contextual information that will allow us to
1115 /// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we
1116 /// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a
1117 /// valid `DefPathHash`. Since we also always build a huge table that maps every
1118 /// `DefPathHash` in the current codebase to the corresponding `DefId`, we have
1119 /// everything we need to re-run the query.
1120 ///
1121 /// Take the `mir_validated` query as an example. Like many other queries, it
1122 /// just has a single parameter: the `DefId` of the item it will compute the
1123 /// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode`
1124 /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode`
1125 /// is actually a `DefPathHash`, and can therefore just look up the corresponding
1126 /// `DefId` in `tcx.def_path_hash_to_def_id`.
1127 ///
1128 /// When you implement a new query, it will likely have a corresponding new
1129 /// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As
1130 /// a rule of thumb, if your query takes a `DefId` or `DefIndex` as sole parameter,
1131 /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
1132 /// add it to the "We don't have enough information to reconstruct..." group in
1133 /// the match below.
1134 pub fn force_from_dep_node<'tcx>(
1135     tcx: TyCtxt<'_, 'tcx, 'tcx>,
1136     dep_node: &DepNode
1137 ) -> bool {
1138     use crate::hir::def_id::LOCAL_CRATE;
1139     use crate::dep_graph::RecoverKey;
1140
1141     // We must avoid ever having to call force_from_dep_node() for a
1142     // DepNode::CodegenUnit:
1143     // Since we cannot reconstruct the query key of a DepNode::CodegenUnit, we
1144     // would always end up having to evaluate the first caller of the
1145     // `codegen_unit` query that *is* reconstructible. This might very well be
1146     // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
1147     // to re-trigger calling the `codegen_unit` query with the right key. At
1148     // that point we would already have re-done all the work we are trying to
1149     // avoid doing in the first place.
1150     // The solution is simple: Just explicitly call the `codegen_unit` query for
1151     // each CGU, right after partitioning. This way `try_mark_green` will always
1152     // hit the cache instead of having to go through `force_from_dep_node`.
1153     // This assertion makes sure, we actually keep applying the solution above.
1154     debug_assert!(dep_node.kind != DepKind::CodegenUnit,
1155                   "calling force_from_dep_node() on DepKind::CodegenUnit");
1156
1157     if !dep_node.kind.can_reconstruct_query_key() {
1158         return false
1159     }
1160
1161     macro_rules! def_id {
1162         () => {
1163             if let Some(def_id) = dep_node.extract_def_id(tcx) {
1164                 def_id
1165             } else {
1166                 // return from the whole function
1167                 return false
1168             }
1169         }
1170     };
1171
1172     macro_rules! krate {
1173         () => { (def_id!()).krate }
1174     };
1175
1176     macro_rules! force_ex {
1177         ($tcx:expr, $query:ident, $key:expr) => {
1178             {
1179                 $tcx.force_query::<crate::ty::query::queries::$query<'_>>(
1180                     $key,
1181                     DUMMY_SP,
1182                     *dep_node
1183                 );
1184             }
1185         }
1186     };
1187
1188     macro_rules! force {
1189         ($query:ident, $key:expr) => { force_ex!(tcx, $query, $key) }
1190     };
1191
1192     // FIXME(#45015): We should try move this boilerplate code into a macro
1193     //                somehow.
1194
1195     rustc_dep_node_force!([dep_node, tcx]
1196         // These are inputs that are expected to be pre-allocated and that
1197         // should therefore always be red or green already
1198         DepKind::AllLocalTraitImpls |
1199         DepKind::Krate |
1200         DepKind::CrateMetadata |
1201         DepKind::HirBody |
1202         DepKind::Hir |
1203
1204         // This are anonymous nodes
1205         DepKind::TraitSelect |
1206
1207         // We don't have enough information to reconstruct the query key of
1208         // these
1209         DepKind::IsCopy |
1210         DepKind::IsSized |
1211         DepKind::IsFreeze |
1212         DepKind::NeedsDrop |
1213         DepKind::Layout |
1214         DepKind::ConstEval |
1215         DepKind::ConstEvalRaw |
1216         DepKind::InstanceSymbolName |
1217         DepKind::MirShim |
1218         DepKind::BorrowCheckKrate |
1219         DepKind::Specializes |
1220         DepKind::ImplementationsOfTrait |
1221         DepKind::TypeParamPredicates |
1222         DepKind::CodegenUnit |
1223         DepKind::CompileCodegenUnit |
1224         DepKind::FulfillObligation |
1225         DepKind::VtableMethods |
1226         DepKind::EraseRegionsTy |
1227         DepKind::NormalizeProjectionTy |
1228         DepKind::NormalizeTyAfterErasingRegions |
1229         DepKind::ImpliedOutlivesBounds |
1230         DepKind::DropckOutlives |
1231         DepKind::EvaluateObligation |
1232         DepKind::EvaluateGoal |
1233         DepKind::TypeOpAscribeUserType |
1234         DepKind::TypeOpEq |
1235         DepKind::TypeOpSubtype |
1236         DepKind::TypeOpProvePredicate |
1237         DepKind::TypeOpNormalizeTy |
1238         DepKind::TypeOpNormalizePredicate |
1239         DepKind::TypeOpNormalizePolyFnSig |
1240         DepKind::TypeOpNormalizeFnSig |
1241         DepKind::SubstituteNormalizeAndTestPredicates |
1242         DepKind::MethodAutoderefSteps |
1243         DepKind::InstanceDefSizeEstimate |
1244         DepKind::ProgramClausesForEnv |
1245
1246         // This one should never occur in this context
1247         DepKind::Null => {
1248             bug!("force_from_dep_node() - Encountered {:?}", dep_node)
1249         }
1250
1251         // These are not queries
1252         DepKind::CoherenceCheckTrait |
1253         DepKind::ItemVarianceConstraints => {
1254             return false
1255         }
1256
1257         DepKind::RegionScopeTree => { force!(region_scope_tree, def_id!()); }
1258
1259         DepKind::Coherence => { force!(crate_inherent_impls, LOCAL_CRATE); }
1260         DepKind::CoherenceInherentImplOverlapCheck => {
1261             force!(crate_inherent_impls_overlap_check, LOCAL_CRATE)
1262         },
1263         DepKind::PrivacyAccessLevels => { force!(privacy_access_levels, LOCAL_CRATE); }
1264         DepKind::CheckPrivateInPublic => { force!(check_private_in_public, LOCAL_CRATE); }
1265         DepKind::MirBuilt => { force!(mir_built, def_id!()); }
1266         DepKind::MirConstQualif => { force!(mir_const_qualif, def_id!()); }
1267         DepKind::MirConst => { force!(mir_const, def_id!()); }
1268         DepKind::MirValidated => { force!(mir_validated, def_id!()); }
1269         DepKind::MirOptimized => { force!(optimized_mir, def_id!()); }
1270
1271         DepKind::BorrowCheck => { force!(borrowck, def_id!()); }
1272         DepKind::MirBorrowCheck => { force!(mir_borrowck, def_id!()); }
1273         DepKind::UnsafetyCheckResult => { force!(unsafety_check_result, def_id!()); }
1274         DepKind::UnsafeDeriveOnReprPacked => { force!(unsafe_derive_on_repr_packed, def_id!()); }
1275         DepKind::CheckModAttrs => { force!(check_mod_attrs, def_id!()); }
1276         DepKind::CheckModLoops => { force!(check_mod_loops, def_id!()); }
1277         DepKind::CheckModUnstableApiUsage => { force!(check_mod_unstable_api_usage, def_id!()); }
1278         DepKind::CheckModItemTypes => { force!(check_mod_item_types, def_id!()); }
1279         DepKind::CheckModPrivacy => { force!(check_mod_privacy, def_id!()); }
1280         DepKind::CheckModIntrinsics => { force!(check_mod_intrinsics, def_id!()); }
1281         DepKind::CheckModLiveness => { force!(check_mod_liveness, def_id!()); }
1282         DepKind::CheckModImplWf => { force!(check_mod_impl_wf, def_id!()); }
1283         DepKind::CollectModItemTypes => { force!(collect_mod_item_types, def_id!()); }
1284         DepKind::Reachability => { force!(reachable_set, LOCAL_CRATE); }
1285         DepKind::MirKeys => { force!(mir_keys, LOCAL_CRATE); }
1286         DepKind::CrateVariances => { force!(crate_variances, LOCAL_CRATE); }
1287         DepKind::AssociatedItems => { force!(associated_item, def_id!()); }
1288         DepKind::PredicatesDefinedOnItem => { force!(predicates_defined_on, def_id!()); }
1289         DepKind::ExplicitPredicatesOfItem => { force!(explicit_predicates_of, def_id!()); }
1290         DepKind::InferredOutlivesOf => { force!(inferred_outlives_of, def_id!()); }
1291         DepKind::InferredOutlivesCrate => { force!(inferred_outlives_crate, LOCAL_CRATE); }
1292         DepKind::SuperPredicatesOfItem => { force!(super_predicates_of, def_id!()); }
1293         DepKind::TraitDefOfItem => { force!(trait_def, def_id!()); }
1294         DepKind::AdtDefOfItem => { force!(adt_def, def_id!()); }
1295         DepKind::ImplTraitRef => { force!(impl_trait_ref, def_id!()); }
1296         DepKind::ImplPolarity => { force!(impl_polarity, def_id!()); }
1297         DepKind::Issue33140SelfTy => { force!(issue33140_self_ty, def_id!()); }
1298         DepKind::FnSignature => { force!(fn_sig, def_id!()); }
1299         DepKind::CoerceUnsizedInfo => { force!(coerce_unsized_info, def_id!()); }
1300         DepKind::ItemVariances => { force!(variances_of, def_id!()); }
1301         DepKind::IsConstFn => { force!(is_const_fn_raw, def_id!()); }
1302         DepKind::IsPromotableConstFn => { force!(is_promotable_const_fn, def_id!()); }
1303         DepKind::IsForeignItem => { force!(is_foreign_item, def_id!()); }
1304         DepKind::SizedConstraint => { force!(adt_sized_constraint, def_id!()); }
1305         DepKind::DtorckConstraint => { force!(adt_dtorck_constraint, def_id!()); }
1306         DepKind::AdtDestructor => { force!(adt_destructor, def_id!()); }
1307         DepKind::AssociatedItemDefIds => { force!(associated_item_def_ids, def_id!()); }
1308         DepKind::InherentImpls => { force!(inherent_impls, def_id!()); }
1309         DepKind::TypeckBodiesKrate => { force!(typeck_item_bodies, LOCAL_CRATE); }
1310         DepKind::TypeckTables => { force!(typeck_tables_of, def_id!()); }
1311         DepKind::UsedTraitImports => { force!(used_trait_imports, def_id!()); }
1312         DepKind::HasTypeckTables => { force!(has_typeck_tables, def_id!()); }
1313         DepKind::SymbolName => { force!(def_symbol_name, def_id!()); }
1314         DepKind::SpecializationGraph => { force!(specialization_graph_of, def_id!()); }
1315         DepKind::ObjectSafety => { force!(is_object_safe, def_id!()); }
1316         DepKind::TraitImpls => { force!(trait_impls_of, def_id!()); }
1317         DepKind::CheckMatch => { force!(check_match, def_id!()); }
1318
1319         DepKind::ParamEnv => { force!(param_env, def_id!()); }
1320         DepKind::Environment => { force!(environment, def_id!()); }
1321         DepKind::DescribeDef => { force!(describe_def, def_id!()); }
1322         DepKind::DefSpan => { force!(def_span, def_id!()); }
1323         DepKind::LookupStability => { force!(lookup_stability, def_id!()); }
1324         DepKind::LookupDeprecationEntry => {
1325             force!(lookup_deprecation_entry, def_id!());
1326         }
1327         DepKind::ConstIsRvaluePromotableToStatic => {
1328             force!(const_is_rvalue_promotable_to_static, def_id!());
1329         }
1330         DepKind::RvaluePromotableMap => { force!(rvalue_promotable_map, def_id!()); }
1331         DepKind::ImplParent => { force!(impl_parent, def_id!()); }
1332         DepKind::TraitOfItem => { force!(trait_of_item, def_id!()); }
1333         DepKind::IsReachableNonGeneric => { force!(is_reachable_non_generic, def_id!()); }
1334         DepKind::IsUnreachableLocalDefinition => {
1335             force!(is_unreachable_local_definition, def_id!());
1336         }
1337         DepKind::IsMirAvailable => { force!(is_mir_available, def_id!()); }
1338         DepKind::ItemAttrs => { force!(item_attrs, def_id!()); }
1339         DepKind::CodegenFnAttrs => { force!(codegen_fn_attrs, def_id!()); }
1340         DepKind::FnArgNames => { force!(fn_arg_names, def_id!()); }
1341         DepKind::RenderedConst => { force!(rendered_const, def_id!()); }
1342         DepKind::DylibDepFormats => { force!(dylib_dependency_formats, krate!()); }
1343         DepKind::IsCompilerBuiltins => { force!(is_compiler_builtins, krate!()); }
1344         DepKind::HasGlobalAllocator => { force!(has_global_allocator, krate!()); }
1345         DepKind::HasPanicHandler => { force!(has_panic_handler, krate!()); }
1346         DepKind::ExternCrate => { force!(extern_crate, def_id!()); }
1347         DepKind::LintLevels => { force!(lint_levels, LOCAL_CRATE); }
1348         DepKind::InScopeTraits => { force!(in_scope_traits_map, def_id!().index); }
1349         DepKind::ModuleExports => { force!(module_exports, def_id!()); }
1350         DepKind::IsSanitizerRuntime => { force!(is_sanitizer_runtime, krate!()); }
1351         DepKind::IsProfilerRuntime => { force!(is_profiler_runtime, krate!()); }
1352         DepKind::GetPanicStrategy => { force!(panic_strategy, krate!()); }
1353         DepKind::IsNoBuiltins => { force!(is_no_builtins, krate!()); }
1354         DepKind::ImplDefaultness => { force!(impl_defaultness, def_id!()); }
1355         DepKind::CheckItemWellFormed => { force!(check_item_well_formed, def_id!()); }
1356         DepKind::CheckTraitItemWellFormed => { force!(check_trait_item_well_formed, def_id!()); }
1357         DepKind::CheckImplItemWellFormed => { force!(check_impl_item_well_formed, def_id!()); }
1358         DepKind::ReachableNonGenerics => { force!(reachable_non_generics, krate!()); }
1359         DepKind::EntryFn => { force!(entry_fn, krate!()); }
1360         DepKind::PluginRegistrarFn => { force!(plugin_registrar_fn, krate!()); }
1361         DepKind::ProcMacroDeclsStatic => { force!(proc_macro_decls_static, krate!()); }
1362         DepKind::CrateDisambiguator => { force!(crate_disambiguator, krate!()); }
1363         DepKind::CrateHash => { force!(crate_hash, krate!()); }
1364         DepKind::OriginalCrateName => { force!(original_crate_name, krate!()); }
1365         DepKind::ExtraFileName => { force!(extra_filename, krate!()); }
1366         DepKind::Analysis => { force!(analysis, krate!()); }
1367
1368         DepKind::AllTraitImplementations => {
1369             force!(all_trait_implementations, krate!());
1370         }
1371
1372         DepKind::DllimportForeignItems => {
1373             force!(dllimport_foreign_items, krate!());
1374         }
1375         DepKind::IsDllimportForeignItem => {
1376             force!(is_dllimport_foreign_item, def_id!());
1377         }
1378         DepKind::IsStaticallyIncludedForeignItem => {
1379             force!(is_statically_included_foreign_item, def_id!());
1380         }
1381         DepKind::NativeLibraryKind => { force!(native_library_kind, def_id!()); }
1382         DepKind::LinkArgs => { force!(link_args, LOCAL_CRATE); }
1383
1384         DepKind::ResolveLifetimes => { force!(resolve_lifetimes, krate!()); }
1385         DepKind::NamedRegion => { force!(named_region_map, def_id!().index); }
1386         DepKind::IsLateBound => { force!(is_late_bound_map, def_id!().index); }
1387         DepKind::ObjectLifetimeDefaults => {
1388             force!(object_lifetime_defaults_map, def_id!().index);
1389         }
1390
1391         DepKind::Visibility => { force!(visibility, def_id!()); }
1392         DepKind::DepKind => { force!(dep_kind, krate!()); }
1393         DepKind::CrateName => { force!(crate_name, krate!()); }
1394         DepKind::ItemChildren => { force!(item_children, def_id!()); }
1395         DepKind::ExternModStmtCnum => { force!(extern_mod_stmt_cnum, def_id!()); }
1396         DepKind::GetLibFeatures => { force!(get_lib_features, LOCAL_CRATE); }
1397         DepKind::DefinedLibFeatures => { force!(defined_lib_features, krate!()); }
1398         DepKind::GetLangItems => { force!(get_lang_items, LOCAL_CRATE); }
1399         DepKind::DefinedLangItems => { force!(defined_lang_items, krate!()); }
1400         DepKind::MissingLangItems => { force!(missing_lang_items, krate!()); }
1401         DepKind::VisibleParentMap => { force!(visible_parent_map, LOCAL_CRATE); }
1402         DepKind::MissingExternCrateItem => {
1403             force!(missing_extern_crate_item, krate!());
1404         }
1405         DepKind::UsedCrateSource => { force!(used_crate_source, krate!()); }
1406         DepKind::PostorderCnums => { force!(postorder_cnums, LOCAL_CRATE); }
1407
1408         DepKind::Freevars => { force!(freevars, def_id!()); }
1409         DepKind::MaybeUnusedTraitImport => {
1410             force!(maybe_unused_trait_import, def_id!());
1411         }
1412         DepKind::NamesImportedByGlobUse => { force!(names_imported_by_glob_use, def_id!()); }
1413         DepKind::MaybeUnusedExternCrates => { force!(maybe_unused_extern_crates, LOCAL_CRATE); }
1414         DepKind::StabilityIndex => { force!(stability_index, LOCAL_CRATE); }
1415         DepKind::AllTraits => { force!(all_traits, LOCAL_CRATE); }
1416         DepKind::AllCrateNums => { force!(all_crate_nums, LOCAL_CRATE); }
1417         DepKind::ExportedSymbols => { force!(exported_symbols, krate!()); }
1418         DepKind::CollectAndPartitionMonoItems => {
1419             force!(collect_and_partition_mono_items, LOCAL_CRATE);
1420         }
1421         DepKind::IsCodegenedItem => { force!(is_codegened_item, def_id!()); }
1422         DepKind::OutputFilenames => { force!(output_filenames, LOCAL_CRATE); }
1423
1424         DepKind::TargetFeaturesWhitelist => { force!(target_features_whitelist, LOCAL_CRATE); }
1425
1426         DepKind::Features => { force!(features_query, LOCAL_CRATE); }
1427
1428         DepKind::ProgramClausesFor => { force!(program_clauses_for, def_id!()); }
1429         DepKind::WasmImportModuleMap => { force!(wasm_import_module_map, krate!()); }
1430         DepKind::ForeignModules => { force!(foreign_modules, krate!()); }
1431
1432         DepKind::UpstreamMonomorphizations => {
1433             force!(upstream_monomorphizations, krate!());
1434         }
1435         DepKind::UpstreamMonomorphizationsFor => {
1436             force!(upstream_monomorphizations_for, def_id!());
1437         }
1438         DepKind::BackendOptimizationLevel => {
1439             force!(backend_optimization_level, krate!());
1440         }
1441     );
1442
1443     true
1444 }
1445
1446
1447 // FIXME(#45015): Another piece of boilerplate code that could be generated in
1448 //                a combined define_dep_nodes!()/define_queries!() macro.
1449 macro_rules! impl_load_from_cache {
1450     ($($dep_kind:ident => $query_name:ident,)*) => {
1451         impl DepNode {
1452             // Check whether the query invocation corresponding to the given
1453             // DepNode is eligible for on-disk-caching.
1454             pub fn cache_on_disk(&self, tcx: TyCtxt<'_, '_, '_>) -> bool {
1455                 use crate::ty::query::queries;
1456                 use crate::ty::query::QueryDescription;
1457
1458                 match self.kind {
1459                     $(DepKind::$dep_kind => {
1460                         let def_id = self.extract_def_id(tcx).unwrap();
1461                         queries::$query_name::cache_on_disk(tcx.global_tcx(), def_id)
1462                     })*
1463                     _ => false
1464                 }
1465             }
1466
1467             // This is method will execute the query corresponding to the given
1468             // DepNode. It is only expected to work for DepNodes where the
1469             // above `cache_on_disk` methods returns true.
1470             // Also, as a sanity check, it expects that the corresponding query
1471             // invocation has been marked as green already.
1472             pub fn load_from_on_disk_cache(&self, tcx: TyCtxt<'_, '_, '_>) {
1473                 match self.kind {
1474                     $(DepKind::$dep_kind => {
1475                         debug_assert!(tcx.dep_graph
1476                                          .node_color(self)
1477                                          .map(|c| c.is_green())
1478                                          .unwrap_or(false));
1479
1480                         let def_id = self.extract_def_id(tcx).unwrap();
1481                         let _ = tcx.$query_name(def_id);
1482                     })*
1483                     _ => {
1484                         bug!()
1485                     }
1486                 }
1487             }
1488         }
1489     }
1490 }
1491
1492 impl_load_from_cache!(
1493     TypeckTables => typeck_tables_of,
1494     MirOptimized => optimized_mir,
1495     UnsafetyCheckResult => unsafety_check_result,
1496     BorrowCheck => borrowck,
1497     MirBorrowCheck => mir_borrowck,
1498     MirConstQualif => mir_const_qualif,
1499     SymbolName => def_symbol_name,
1500     ConstIsRvaluePromotableToStatic => const_is_rvalue_promotable_to_static,
1501     CheckMatch => check_match,
1502     type_of => type_of,
1503     generics_of => generics_of,
1504     predicates_of => predicates_of,
1505     UsedTraitImports => used_trait_imports,
1506     CodegenFnAttrs => codegen_fn_attrs,
1507     SpecializationGraph => specialization_graph_of,
1508 );