src/librustc/ty/query/plumbing.rs

   1 //! The implementation of the query system itself. Defines the macros
   2 //! that generate the actual methods on tcx which find and execute the
   3 //! provider, manage the caches, and so forth.
   4
   5 use crate::dep_graph::{DepNodeIndex, DepNode, DepKind, SerializedDepNodeIndex};
   6 use crate::errors::DiagnosticBuilder;
   7 use crate::errors::Level;
   8 use crate::errors::Diagnostic;
   9 use crate::errors::FatalError;
  10 use crate::ty::tls;
  11 use crate::ty::{TyCtxt};
  12 use crate::ty::query::Query;
  13 use crate::ty::query::config::{QueryConfig, QueryDescription};
  14 use crate::ty::query::job::{QueryJob, QueryResult, QueryInfo};
  15 use crate::ty::item_path;
  16
  17 use crate::util::common::{profq_msg, ProfileQueriesMsg, QueryMsg};
  18
  19 use rustc_data_structures::fx::{FxHashMap};
  20 use rustc_data_structures::sync::{Lrc, Lock};
  21 use rustc_data_structures::thin_vec::ThinVec;
  22 use std::mem;
  23 use std::ptr;
  24 use std::collections::hash_map::Entry;
  25 use syntax_pos::Span;
  26 use syntax::source_map::DUMMY_SP;
  27
  28 pub struct QueryCache<'tcx, D: QueryConfig<'tcx> + ?Sized> {
  29     pub(super) results: FxHashMap<D::Key, QueryValue<D::Value>>,
  30     pub(super) active: FxHashMap<D::Key, QueryResult<'tcx>>,
  31     #[cfg(debug_assertions)]
  32     pub(super) cache_hits: usize,
  33 }
  34
  35 pub(super) struct QueryValue<T> {
  36     pub(super) value: T,
  37     pub(super) index: DepNodeIndex,
  38 }
  39
  40 impl<T> QueryValue<T> {
  41     pub(super) fn new(value: T,
  42                       dep_node_index: DepNodeIndex)
  43                       -> QueryValue<T> {
  44         QueryValue {
  45             value,
  46             index: dep_node_index,
  47         }
  48     }
  49 }
  50
  51 impl<'tcx, M: QueryConfig<'tcx>> Default for QueryCache<'tcx, M> {
  52     fn default() -> QueryCache<'tcx, M> {
  53         QueryCache {
  54             results: FxHashMap::default(),
  55             active: FxHashMap::default(),
  56             #[cfg(debug_assertions)]
  57             cache_hits: 0,
  58         }
  59     }
  60 }
  61
  62 // If enabled, send a message to the profile-queries thread
  63 macro_rules! profq_msg {
  64     ($tcx:expr, $msg:expr) => {
  65         if cfg!(debug_assertions) {
  66             if $tcx.sess.profile_queries() {
  67                 profq_msg($tcx.sess, $msg)
  68             }
  69         }
  70     }
  71 }
  72
  73 // If enabled, format a key using its debug string, which can be
  74 // expensive to compute (in terms of time).
  75 macro_rules! profq_query_msg {
  76     ($query:expr, $tcx:expr, $key:expr) => {{
  77         let msg = if cfg!(debug_assertions) {
  78             if $tcx.sess.profile_queries_and_keys() {
  79                 Some(format!("{:?}", $key))
  80             } else { None }
  81         } else { None };
  82         QueryMsg {
  83             query: $query,
  84             msg,
  85         }
  86     }}
  87 }
  88
  89 /// A type representing the responsibility to execute the job in the `job` field.
  90 /// This will poison the relevant query if dropped.
  91 pub(super) struct JobOwner<'a, 'tcx: 'a, Q: QueryDescription<'tcx> + 'a> {
  92     cache: &'a Lock<QueryCache<'tcx, Q>>,
  93     key: Q::Key,
  94     job: Lrc<QueryJob<'tcx>>,
  95 }
  96
  97 impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> {
  98     /// Either gets a JobOwner corresponding the query, allowing us to
  99     /// start executing the query, or it returns with the result of the query.
 100     /// If the query is executing elsewhere, this will wait for it.
 101     /// If the query panicked, this will silently panic.
 102     ///
 103     /// This function is inlined because that results in a noticeable speedup
 104     /// for some compile-time benchmarks.
 105     #[inline(always)]
 106     pub(super) fn try_get(
 107         tcx: TyCtxt<'a, 'tcx, '_>,
 108         span: Span,
 109         key: &Q::Key,
 110     ) -> TryGetJob<'a, 'tcx, Q> {
 111         let cache = Q::query_cache(tcx);
 112         loop {
 113             let mut lock = cache.borrow_mut();
 114             if let Some(value) = lock.results.get(key) {
 115                 profq_msg!(tcx, ProfileQueriesMsg::CacheHit);
 116                 tcx.sess.profiler(|p| p.record_query_hit(Q::CATEGORY));
 117                 let result = Ok((value.value.clone(), value.index));
 118                 #[cfg(debug_assertions)]
 119                 {
 120                     lock.cache_hits += 1;
 121                 }
 122                 return TryGetJob::JobCompleted(result);
 123             }
 124             let job = match lock.active.entry((*key).clone()) {
 125                 Entry::Occupied(entry) => {
 126                     match *entry.get() {
 127                         QueryResult::Started(ref job) => job.clone(),
 128                         QueryResult::Poisoned => FatalError.raise(),
 129                     }
 130                 }
 131                 Entry::Vacant(entry) => {
 132                     // No job entry for this query. Return a new one to be started later
 133                     return tls::with_related_context(tcx, |icx| {
 134                         // Create the `parent` variable before `info`. This allows LLVM
 135                         // to elide the move of `info`
 136                         let parent = icx.query.clone();
 137                         let info = QueryInfo {
 138                             span,
 139                             query: Q::query(key.clone()),
 140                         };
 141                         let job = Lrc::new(QueryJob::new(info, parent));
 142                         let owner = JobOwner {
 143                             cache,
 144                             job: job.clone(),
 145                             key: (*key).clone(),
 146                         };
 147                         entry.insert(QueryResult::Started(job));
 148                         TryGetJob::NotYetStarted(owner)
 149                     })
 150                 }
 151             };
 152             mem::drop(lock);
 153
 154             // If we are single-threaded we know that we have cycle error,
 155             // so we just turn the errror
 156             #[cfg(not(parallel_compiler))]
 157             return job.cycle_error(tcx, span);
 158
 159             // With parallel queries we might just have to wait on some other
 160             // thread
 161             #[cfg(parallel_compiler)]
 162             {
 163                 if let Err(cycle) = job.r#await(tcx, span) {
 164                     return TryGetJob::JobCompleted(Err(cycle));
 165                 }
 166             }
 167         }
 168     }
 169
 170     /// Completes the query by updating the query cache with the `result`,
 171     /// signals the waiter and forgets the JobOwner, so it won't poison the query
 172     #[inline(always)]
 173     pub(super) fn complete(self, result: &Q::Value, dep_node_index: DepNodeIndex) {
 174         // We can move out of `self` here because we `mem::forget` it below
 175         let key = unsafe { ptr::read(&self.key) };
 176         let job = unsafe { ptr::read(&self.job) };
 177         let cache = self.cache;
 178
 179         // Forget ourself so our destructor won't poison the query
 180         mem::forget(self);
 181
 182         let value = QueryValue::new(result.clone(), dep_node_index);
 183         {
 184             let mut lock = cache.borrow_mut();
 185             lock.active.remove(&key);
 186             lock.results.insert(key, value);
 187         }
 188
 189         job.signal_complete();
 190     }
 191
 192     /// Executes a job by changing the ImplicitCtxt to point to the
 193     /// new query job while it executes. It returns the diagnostics
 194     /// captured during execution and the actual result.
 195     #[inline(always)]
 196     pub(super) fn start<'lcx, F, R>(
 197         &self,
 198         tcx: TyCtxt<'_, 'tcx, 'lcx>,
 199         diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
 200         compute: F)
 201     -> R
 202     where
 203         F: for<'b> FnOnce(TyCtxt<'b, 'tcx, 'lcx>) -> R
 204     {
 205         // The TyCtxt stored in TLS has the same global interner lifetime
 206         // as `tcx`, so we use `with_related_context` to relate the 'gcx lifetimes
 207         // when accessing the ImplicitCtxt
 208         tls::with_related_context(tcx, move |current_icx| {
 209             // Update the ImplicitCtxt to point to our new query job
 210             let new_icx = tls::ImplicitCtxt {
 211                 tcx: tcx.global_tcx(),
 212                 query: Some(self.job.clone()),
 213                 diagnostics,
 214                 layout_depth: current_icx.layout_depth,
 215                 task_deps: current_icx.task_deps,
 216             };
 217
 218             // Use the ImplicitCtxt while we execute the query
 219             tls::enter_context(&new_icx, |_| {
 220                 compute(tcx)
 221             })
 222         })
 223     }
 224
 225 }
 226
 227 #[inline(always)]
 228 fn with_diagnostics<F, R>(f: F) -> (R, ThinVec<Diagnostic>)
 229 where
 230     F: FnOnce(Option<&Lock<ThinVec<Diagnostic>>>) -> R
 231 {
 232     let diagnostics = Lock::new(ThinVec::new());
 233     let result = f(Some(&diagnostics));
 234     (result, diagnostics.into_inner())
 235 }
 236
 237 impl<'a, 'tcx, Q: QueryDescription<'tcx>> Drop for JobOwner<'a, 'tcx, Q> {
 238     #[inline(never)]
 239     #[cold]
 240     fn drop(&mut self) {
 241         // Poison the query so jobs waiting on it panic
 242         self.cache.borrow_mut().active.insert(self.key.clone(), QueryResult::Poisoned);
 243         // Also signal the completion of the job, so waiters
 244         // will continue execution
 245         self.job.signal_complete();
 246     }
 247 }
 248
 249 #[derive(Clone)]
 250 pub struct CycleError<'tcx> {
 251     /// The query and related span which uses the cycle
 252     pub(super) usage: Option<(Span, Query<'tcx>)>,
 253     pub(super) cycle: Vec<QueryInfo<'tcx>>,
 254 }
 255
 256 /// The result of `try_get_lock`
 257 pub(super) enum TryGetJob<'a, 'tcx: 'a, D: QueryDescription<'tcx> + 'a> {
 258     /// The query is not yet started. Contains a guard to the cache eventually used to start it.
 259     NotYetStarted(JobOwner<'a, 'tcx, D>),
 260
 261     /// The query was already completed.
 262     /// Returns the result of the query and its dep node index
 263     /// if it succeeded or a cycle error if it failed
 264     JobCompleted(Result<(D::Value, DepNodeIndex), Box<CycleError<'tcx>>>),
 265 }
 266
 267 impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
 268     #[inline(never)]
 269     #[cold]
 270     pub(super) fn report_cycle(
 271         self,
 272         box CycleError { usage, cycle: stack }: Box<CycleError<'gcx>>
 273     ) -> Box<DiagnosticBuilder<'a>>
 274     {
 275         assert!(!stack.is_empty());
 276
 277         let fix_span = |span: Span, query: &Query<'gcx>| {
 278             self.sess.source_map().def_span(query.default_span(self, span))
 279         };
 280
 281         // Disable naming impls with types in this path, since that
 282         // sometimes cycles itself, leading to extra cycle errors.
 283         // (And cycle errors around impls tend to occur during the
 284         // collect/coherence phases anyhow.)
 285         item_path::with_forced_impl_filename_line(|| {
 286             let span = fix_span(stack[1 % stack.len()].span, &stack[0].query);
 287             let mut err = struct_span_err!(self.sess,
 288                                            span,
 289                                            E0391,
 290                                            "cycle detected when {}",
 291                                            stack[0].query.describe(self));
 292
 293             for i in 1..stack.len() {
 294                 let query = &stack[i].query;
 295                 let span = fix_span(stack[(i + 1) % stack.len()].span, query);
 296                 err.span_note(span, &format!("...which requires {}...", query.describe(self)));
 297             }
 298
 299             err.note(&format!("...which again requires {}, completing the cycle",
 300                               stack[0].query.describe(self)));
 301
 302             if let Some((span, query)) = usage {
 303                 err.span_note(fix_span(span, &query),
 304                               &format!("cycle used when {}", query.describe(self)));
 305             }
 306
 307             return Box::new(err)
 308         })
 309     }
 310
 311     pub fn try_print_query_stack() {
 312         eprintln!("query stack during panic:");
 313
 314         tls::with_context_opt(|icx| {
 315             if let Some(icx) = icx {
 316                 let mut current_query = icx.query.clone();
 317                 let mut i = 0;
 318
 319                 while let Some(query) = current_query {
 320                     let mut db = DiagnosticBuilder::new(icx.tcx.sess.diagnostic(),
 321                         Level::FailureNote,
 322                         &format!("#{} [{}] {}",
 323                                  i,
 324                                  query.info.query.name(),
 325                                  query.info.query.describe(icx.tcx)));
 326                     db.set_span(icx.tcx.sess.source_map().def_span(query.info.span));
 327                     icx.tcx.sess.diagnostic().force_print_db(db);
 328
 329                     current_query = query.parent.clone();
 330                     i += 1;
 331                 }
 332             }
 333         });
 334
 335         eprintln!("end of query stack");
 336     }
 337
 338     #[inline(never)]
 339     fn try_get_with<Q: QueryDescription<'gcx>>(
 340         self,
 341         span: Span,
 342         key: Q::Key)
 343     -> Result<Q::Value, Box<CycleError<'gcx>>>
 344     {
 345         debug!("ty::queries::{}::try_get_with(key={:?}, span={:?})",
 346                Q::NAME,
 347                key,
 348                span);
 349
 350         profq_msg!(self,
 351             ProfileQueriesMsg::QueryBegin(
 352                 span.data(),
 353                 profq_query_msg!(Q::NAME, self, key),
 354             )
 355         );
 356
 357         let job = match JobOwner::try_get(self, span, &key) {
 358             TryGetJob::NotYetStarted(job) => job,
 359             TryGetJob::JobCompleted(result) => {
 360                 return result.map(|(v, index)| {
 361                     self.dep_graph.read_index(index);
 362                     v
 363                 })
 364             }
 365         };
 366
 367         // Fast path for when incr. comp. is off. `to_dep_node` is
 368         // expensive for some DepKinds.
 369         if !self.dep_graph.is_fully_enabled() {
 370             let null_dep_node = DepNode::new_no_params(crate::dep_graph::DepKind::Null);
 371             return Ok(self.force_query_with_job::<Q>(key, job, null_dep_node).0);
 372         }
 373
 374         let dep_node = Q::to_dep_node(self, &key);
 375
 376         if dep_node.kind.is_anon() {
 377             profq_msg!(self, ProfileQueriesMsg::ProviderBegin);
 378             self.sess.profiler(|p| p.start_activity(Q::CATEGORY));
 379
 380             let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
 381                 job.start(self, diagnostics, |tcx| {
 382                     tcx.dep_graph.with_anon_task(dep_node.kind, || {
 383                         Q::compute(tcx.global_tcx(), key)
 384                     })
 385                 })
 386             });
 387
 388             self.sess.profiler(|p| p.end_activity(Q::CATEGORY));
 389             profq_msg!(self, ProfileQueriesMsg::ProviderEnd);
 390
 391             self.dep_graph.read_index(dep_node_index);
 392
 393             if unlikely!(!diagnostics.is_empty()) {
 394                 self.queries.on_disk_cache
 395                     .store_diagnostics_for_anon_node(dep_node_index, diagnostics);
 396             }
 397
 398             job.complete(&result, dep_node_index);
 399
 400             return Ok(result);
 401         }
 402
 403         if !dep_node.kind.is_input() {
 404             if let Some((prev_dep_node_index,
 405                          dep_node_index)) = self.dep_graph.try_mark_green_and_read(self,
 406                                                                                    &dep_node) {
 407                 return Ok(self.load_from_disk_and_cache_in_memory::<Q>(
 408                     key,
 409                     job,
 410                     prev_dep_node_index,
 411                     dep_node_index,
 412                     &dep_node
 413                 ))
 414             }
 415         }
 416
 417         let (result, dep_node_index) = self.force_query_with_job::<Q>(key, job, dep_node);
 418         self.dep_graph.read_index(dep_node_index);
 419         Ok(result)
 420     }
 421
 422     fn load_from_disk_and_cache_in_memory<Q: QueryDescription<'gcx>>(
 423         self,
 424         key: Q::Key,
 425         job: JobOwner<'a, 'gcx, Q>,
 426         prev_dep_node_index: SerializedDepNodeIndex,
 427         dep_node_index: DepNodeIndex,
 428         dep_node: &DepNode
 429     ) -> Q::Value
 430     {
 431         // Note this function can be called concurrently from the same query
 432         // We must ensure that this is handled correctly
 433
 434         debug_assert!(self.dep_graph.is_green(dep_node));
 435
 436         // First we try to load the result from the on-disk cache
 437         let result = if Q::cache_on_disk(self.global_tcx(), key.clone()) &&
 438                         self.sess.opts.debugging_opts.incremental_queries {
 439             let result = Q::try_load_from_disk(self.global_tcx(), prev_dep_node_index);
 440
 441             // We always expect to find a cached result for things that
 442             // can be forced from DepNode.
 443             debug_assert!(!dep_node.kind.can_reconstruct_query_key() ||
 444                           result.is_some(),
 445                           "Missing on-disk cache entry for {:?}",
 446                           dep_node);
 447             result
 448         } else {
 449             // Some things are never cached on disk.
 450             None
 451         };
 452
 453         let result = if let Some(result) = result {
 454             profq_msg!(self, ProfileQueriesMsg::CacheHit);
 455             self.sess.profiler(|p| p.record_query_hit(Q::CATEGORY));
 456
 457             result
 458         } else {
 459             // We could not load a result from the on-disk cache, so
 460             // recompute.
 461
 462             self.sess.profiler(|p| p.start_activity(Q::CATEGORY));
 463
 464             // The diagnostics for this query have already been
 465             // promoted to the current session during
 466             // try_mark_green(), so we can ignore them here.
 467             let result = job.start(self, None, |tcx| {
 468                 // The dep-graph for this computation is already in
 469                 // place
 470                 tcx.dep_graph.with_ignore(|| {
 471                     Q::compute(tcx, key)
 472                 })
 473             });
 474
 475             self.sess.profiler(|p| p.end_activity(Q::CATEGORY));
 476             result
 477         };
 478
 479         // If -Zincremental-verify-ich is specified, re-hash results from
 480         // the cache and make sure that they have the expected fingerprint.
 481         if unlikely!(self.sess.opts.debugging_opts.incremental_verify_ich) {
 482             self.incremental_verify_ich::<Q>(&result, dep_node, dep_node_index);
 483         }
 484
 485         if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) {
 486             self.dep_graph.mark_loaded_from_cache(dep_node_index, true);
 487         }
 488
 489         job.complete(&result, dep_node_index);
 490
 491         result
 492     }
 493
 494     #[inline(never)]
 495     #[cold]
 496     fn incremental_verify_ich<Q: QueryDescription<'gcx>>(
 497         self,
 498         result: &Q::Value,
 499         dep_node: &DepNode,
 500         dep_node_index: DepNodeIndex,
 501     ) {
 502         use crate::ich::Fingerprint;
 503
 504         assert!(Some(self.dep_graph.fingerprint_of(dep_node_index)) ==
 505                 self.dep_graph.prev_fingerprint_of(dep_node),
 506                 "Fingerprint for green query instance not loaded \
 507                     from cache: {:?}", dep_node);
 508
 509         debug!("BEGIN verify_ich({:?})", dep_node);
 510         let mut hcx = self.create_stable_hashing_context();
 511
 512         let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
 513         debug!("END verify_ich({:?})", dep_node);
 514
 515         let old_hash = self.dep_graph.fingerprint_of(dep_node_index);
 516
 517         assert!(new_hash == old_hash, "Found unstable fingerprints \
 518             for {:?}", dep_node);
 519     }
 520
 521     #[inline(always)]
 522     fn force_query_with_job<Q: QueryDescription<'gcx>>(
 523         self,
 524         key: Q::Key,
 525         job: JobOwner<'_, 'gcx, Q>,
 526         dep_node: DepNode)
 527     -> (Q::Value, DepNodeIndex) {
 528         // If the following assertion triggers, it can have two reasons:
 529         // 1. Something is wrong with DepNode creation, either here or
 530         //    in DepGraph::try_mark_green()
 531         // 2. Two distinct query keys get mapped to the same DepNode
 532         //    (see for example #48923)
 533         assert!(!self.dep_graph.dep_node_exists(&dep_node),
 534                 "Forcing query with already existing DepNode.\n\
 535                  - query-key: {:?}\n\
 536                  - dep-node: {:?}",
 537                 key, dep_node);
 538
 539         profq_msg!(self, ProfileQueriesMsg::ProviderBegin);
 540         self.sess.profiler(|p| p.start_activity(Q::CATEGORY));
 541
 542         let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
 543             job.start(self, diagnostics, |tcx| {
 544                 if dep_node.kind.is_eval_always() {
 545                     tcx.dep_graph.with_eval_always_task(dep_node,
 546                                                         tcx,
 547                                                         key,
 548                                                         Q::compute,
 549                                                         Q::hash_result)
 550                 } else {
 551                     tcx.dep_graph.with_task(dep_node,
 552                                             tcx,
 553                                             key,
 554                                             Q::compute,
 555                                             Q::hash_result)
 556                 }
 557             })
 558         });
 559
 560         self.sess.profiler(|p| p.end_activity(Q::CATEGORY));
 561         profq_msg!(self, ProfileQueriesMsg::ProviderEnd);
 562
 563         if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) {
 564             self.dep_graph.mark_loaded_from_cache(dep_node_index, false);
 565         }
 566
 567         if dep_node.kind != crate::dep_graph::DepKind::Null {
 568             if unlikely!(!diagnostics.is_empty()) {
 569                 self.queries.on_disk_cache
 570                     .store_diagnostics(dep_node_index, diagnostics);
 571             }
 572         }
 573
 574         job.complete(&result, dep_node_index);
 575
 576         (result, dep_node_index)
 577     }
 578
 579     /// Ensure that either this query has all green inputs or been executed.
 580     /// Executing query::ensure(D) is considered a read of the dep-node D.
 581     ///
 582     /// This function is particularly useful when executing passes for their
 583     /// side-effects -- e.g., in order to report errors for erroneous programs.
 584     ///
 585     /// Note: The optimization is only available during incr. comp.
 586     pub(super) fn ensure_query<Q: QueryDescription<'gcx>>(self, key: Q::Key) -> () {
 587         let dep_node = Q::to_dep_node(self, &key);
 588
 589         // Ensuring an "input" or anonymous query makes no sense
 590         assert!(!dep_node.kind.is_anon());
 591         assert!(!dep_node.kind.is_input());
 592         if self.dep_graph.try_mark_green_and_read(self, &dep_node).is_none() {
 593             // A None return from `try_mark_green_and_read` means that this is either
 594             // a new dep node or that the dep node has already been marked red.
 595             // Either way, we can't call `dep_graph.read()` as we don't have the
 596             // DepNodeIndex. We must invoke the query itself. The performance cost
 597             // this introduces should be negligible as we'll immediately hit the
 598             // in-memory cache, or another query down the line will.
 599
 600             let _ = self.get_query::<Q>(DUMMY_SP, key);
 601         } else {
 602             profq_msg!(self, ProfileQueriesMsg::CacheHit);
 603             self.sess.profiler(|p| p.record_query_hit(Q::CATEGORY));
 604         }
 605     }
 606
 607     #[allow(dead_code)]
 608     fn force_query<Q: QueryDescription<'gcx>>(
 609         self,
 610         key: Q::Key,
 611         span: Span,
 612         dep_node: DepNode
 613     ) {
 614         profq_msg!(
 615             self,
 616             ProfileQueriesMsg::QueryBegin(span.data(), profq_query_msg!(Q::NAME, self, key))
 617         );
 618
 619         // We may be concurrently trying both execute and force a query
 620         // Ensure that only one of them runs the query
 621         let job = match JobOwner::try_get(self, span, &key) {
 622             TryGetJob::NotYetStarted(job) => job,
 623             TryGetJob::JobCompleted(result) => {
 624                 if let Err(e) = result {
 625                     self.report_cycle(e).emit();
 626                 }
 627                 return
 628             }
 629         };
 630         self.force_query_with_job::<Q>(key, job, dep_node);
 631     }
 632
 633     pub(super) fn try_get_query<Q: QueryDescription<'gcx>>(
 634         self,
 635         span: Span,
 636         key: Q::Key,
 637     ) -> Result<Q::Value, Box<DiagnosticBuilder<'a>>> {
 638         match self.try_get_with::<Q>(span, key) {
 639             Ok(e) => Ok(e),
 640             Err(e) => Err(self.report_cycle(e)),
 641         }
 642     }
 643
 644     // FIXME: Try uninlining this
 645     #[inline(always)]
 646     pub(super) fn get_query<Q: QueryDescription<'gcx>>(
 647         self,
 648         span: Span,
 649         key: Q::Key,
 650     ) -> Q::Value {
 651         self.try_get_with::<Q>(span, key).unwrap_or_else(|e| {
 652             self.emit_error::<Q>(e)
 653         })
 654     }
 655
 656     #[inline(never)]
 657     #[cold]
 658     fn emit_error<Q: QueryDescription<'gcx>>(
 659         self,
 660         e: Box<CycleError<'gcx>>,
 661     ) -> Q::Value {
 662         self.report_cycle(e).emit();
 663         Q::handle_cycle_error(self)
 664     }
 665 }
 666
 667 macro_rules! handle_cycle_error {
 668     ([][$this: expr]) => {{
 669         Value::from_cycle_error($this.global_tcx())
 670     }};
 671     ([fatal_cycle$(, $modifiers:ident)*][$this:expr]) => {{
 672         $this.sess.abort_if_errors();
 673         unreachable!();
 674     }};
 675     ([$other:ident$(, $modifiers:ident)*][$($args:tt)*]) => {
 676         handle_cycle_error!([$($modifiers),*][$($args)*])
 677     };
 678 }
 679
 680 macro_rules! hash_result {
 681     ([][$hcx:expr, $result:expr]) => {{
 682         dep_graph::hash_result($hcx, &$result)
 683     }};
 684     ([no_hash$(, $modifiers:ident)*][$hcx:expr, $result:expr]) => {{
 685         None
 686     }};
 687     ([$other:ident$(, $modifiers:ident)*][$($args:tt)*]) => {
 688         hash_result!([$($modifiers),*][$($args)*])
 689     };
 690 }
 691
 692 macro_rules! define_queries {
 693     (<$tcx:tt> $($category:tt {
 694         $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*
 695     },)*) => {
 696         define_queries_inner! { <$tcx>
 697             $($( $(#[$attr])* category<$category> [$($modifiers)*] fn $name: $node($K) -> $V,)*)*
 698         }
 699     }
 700 }
 701
 702 macro_rules! define_queries_inner {
 703     (<$tcx:tt>
 704      $($(#[$attr:meta])* category<$category:tt>
 705         [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*) => {
 706
 707         use std::mem;
 708         #[cfg(parallel_compiler)]
 709         use ty::query::job::QueryResult;
 710         use rustc_data_structures::sync::Lock;
 711         use crate::{
 712             rustc_data_structures::stable_hasher::HashStable,
 713             rustc_data_structures::stable_hasher::StableHasherResult,
 714             rustc_data_structures::stable_hasher::StableHasher,
 715             ich::StableHashingContext
 716         };
 717         use crate::util::profiling::ProfileCategory;
 718
 719         define_queries_struct! {
 720             tcx: $tcx,
 721             input: ($(([$($modifiers)*] [$($attr)*] [$name]))*)
 722         }
 723
 724         impl<$tcx> Queries<$tcx> {
 725             pub fn new(
 726                 providers: IndexVec<CrateNum, Providers<$tcx>>,
 727                 fallback_extern_providers: Providers<$tcx>,
 728                 on_disk_cache: OnDiskCache<'tcx>,
 729             ) -> Self {
 730                 Queries {
 731                     providers,
 732                     fallback_extern_providers: Box::new(fallback_extern_providers),
 733                     on_disk_cache,
 734                     $($name: Default::default()),*
 735                 }
 736             }
 737
 738             pub fn record_computed_queries(&self, sess: &Session) {
 739                 sess.profiler(|p| {
 740                     $(
 741                         p.record_computed_queries(
 742                             <queries::$name<'_> as QueryConfig<'_>>::CATEGORY,
 743                             self.$name.lock().results.len()
 744                         );
 745                     )*
 746                 });
 747             }
 748
 749             #[cfg(parallel_compiler)]
 750             pub fn collect_active_jobs(&self) -> Vec<Lrc<QueryJob<$tcx>>> {
 751                 let mut jobs = Vec::new();
 752
 753                 // We use try_lock here since we are only called from the
 754                 // deadlock handler, and this shouldn't be locked
 755                 $(
 756                     jobs.extend(
 757                         self.$name.try_lock().unwrap().active.values().filter_map(|v|
 758                             if let QueryResult::Started(ref job) = *v {
 759                                 Some(job.clone())
 760                             } else {
 761                                 None
 762                             }
 763                         )
 764                     );
 765                 )*
 766
 767                 jobs
 768             }
 769
 770             pub fn print_stats(&self) {
 771                 let mut queries = Vec::new();
 772
 773                 #[derive(Clone)]
 774                 struct QueryStats {
 775                     name: &'static str,
 776                     cache_hits: usize,
 777                     key_size: usize,
 778                     key_type: &'static str,
 779                     value_size: usize,
 780                     value_type: &'static str,
 781                     entry_count: usize,
 782                 }
 783
 784                 fn stats<'tcx, Q: QueryConfig<'tcx>>(
 785                     name: &'static str,
 786                     map: &QueryCache<'tcx, Q>
 787                 ) -> QueryStats {
 788                     QueryStats {
 789                         name,
 790                         #[cfg(debug_assertions)]
 791                         cache_hits: map.cache_hits,
 792                         #[cfg(not(debug_assertions))]
 793                         cache_hits: 0,
 794                         key_size: mem::size_of::<Q::Key>(),
 795                         key_type: unsafe { type_name::<Q::Key>() },
 796                         value_size: mem::size_of::<Q::Value>(),
 797                         value_type: unsafe { type_name::<Q::Value>() },
 798                         entry_count: map.results.len(),
 799                     }
 800                 }
 801
 802                 $(
 803                     queries.push(stats::<queries::$name<'_>>(
 804                         stringify!($name),
 805                         &*self.$name.lock()
 806                     ));
 807                 )*
 808
 809                 if cfg!(debug_assertions) {
 810                     let hits: usize = queries.iter().map(|s| s.cache_hits).sum();
 811                     let results: usize = queries.iter().map(|s| s.entry_count).sum();
 812                     println!("\nQuery cache hit rate: {}", hits as f64 / (hits + results) as f64);
 813                 }
 814
 815                 let mut query_key_sizes = queries.clone();
 816                 query_key_sizes.sort_by_key(|q| q.key_size);
 817                 println!("\nLarge query keys:");
 818                 for q in query_key_sizes.iter().rev()
 819                                         .filter(|q| q.key_size > 8) {
 820                     println!(
 821                         "   {} - {} x {} - {}",
 822                         q.name,
 823                         q.key_size,
 824                         q.entry_count,
 825                         q.key_type
 826                     );
 827                 }
 828
 829                 let mut query_value_sizes = queries.clone();
 830                 query_value_sizes.sort_by_key(|q| q.value_size);
 831                 println!("\nLarge query values:");
 832                 for q in query_value_sizes.iter().rev()
 833                                           .filter(|q| q.value_size > 8) {
 834                     println!(
 835                         "   {} - {} x {} - {}",
 836                         q.name,
 837                         q.value_size,
 838                         q.entry_count,
 839                         q.value_type
 840                     );
 841                 }
 842
 843                 if cfg!(debug_assertions) {
 844                     let mut query_cache_hits = queries.clone();
 845                     query_cache_hits.sort_by_key(|q| q.cache_hits);
 846                     println!("\nQuery cache hits:");
 847                     for q in query_cache_hits.iter().rev() {
 848                         println!(
 849                             "   {} - {} ({}%)",
 850                             q.name,
 851                             q.cache_hits,
 852                             q.cache_hits as f64 / (q.cache_hits + q.entry_count) as f64
 853                         );
 854                     }
 855                 }
 856
 857                 let mut query_value_count = queries.clone();
 858                 query_value_count.sort_by_key(|q| q.entry_count);
 859                 println!("\nQuery value count:");
 860                 for q in query_value_count.iter().rev() {
 861                     println!("   {} - {}", q.name, q.entry_count);
 862                 }
 863             }
 864         }
 865
 866         #[allow(nonstandard_style)]
 867         #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 868         pub enum Query<$tcx> {
 869             $($(#[$attr])* $name($K)),*
 870         }
 871
 872         impl<$tcx> Query<$tcx> {
 873             pub fn name(&self) -> &'static str {
 874                 match *self {
 875                     $(Query::$name(_) => stringify!($name),)*
 876                 }
 877             }
 878
 879             pub fn describe(&self, tcx: TyCtxt<'_, '_, '_>) -> Cow<'static, str> {
 880                 let (r, name) = match *self {
 881                     $(Query::$name(key) => {
 882                         (queries::$name::describe(tcx, key), stringify!($name))
 883                     })*
 884                 };
 885                 if tcx.sess.verbose() {
 886                     format!("{} [{}]", r, name).into()
 887                 } else {
 888                     r
 889                 }
 890             }
 891
 892             // FIXME(eddyb) Get more valid Span's on queries.
 893             pub fn default_span(&self, tcx: TyCtxt<'_, $tcx, '_>, span: Span) -> Span {
 894                 if !span.is_dummy() {
 895                     return span;
 896                 }
 897                 // The def_span query is used to calculate default_span,
 898                 // so exit to avoid infinite recursion
 899                 if let Query::def_span(..) = *self {
 900                     return span
 901                 }
 902                 match *self {
 903                     $(Query::$name(key) => key.default_span(tcx),)*
 904                 }
 905             }
 906         }
 907
 908         impl<'a, $tcx> HashStable<StableHashingContext<'a>> for Query<$tcx> {
 909             fn hash_stable<W: StableHasherResult>(&self,
 910                                                 hcx: &mut StableHashingContext<'a>,
 911                                                 hasher: &mut StableHasher<W>) {
 912                 mem::discriminant(self).hash_stable(hcx, hasher);
 913                 match *self {
 914                     $(Query::$name(key) => key.hash_stable(hcx, hasher),)*
 915                 }
 916             }
 917         }
 918
 919         pub mod queries {
 920             use std::marker::PhantomData;
 921
 922             $(#[allow(nonstandard_style)]
 923             pub struct $name<$tcx> {
 924                 data: PhantomData<&$tcx ()>
 925             })*
 926         }
 927
 928         // This module and the functions in it exist only to provide a
 929         // predictable symbol name prefix for query providers. This is helpful
 930         // for analyzing queries in profilers.
 931         pub(super) mod __query_compute {
 932             $(#[inline(never)]
 933             pub fn $name<F: FnOnce() -> R, R>(f: F) -> R {
 934                 f()
 935             })*
 936         }
 937
 938         $(impl<$tcx> QueryConfig<$tcx> for queries::$name<$tcx> {
 939             type Key = $K;
 940             type Value = $V;
 941
 942             const NAME: &'static str = stringify!($name);
 943             const CATEGORY: ProfileCategory = $category;
 944         }
 945
 946         impl<$tcx> QueryAccessors<$tcx> for queries::$name<$tcx> {
 947             #[inline(always)]
 948             fn query(key: Self::Key) -> Query<'tcx> {
 949                 Query::$name(key)
 950             }
 951
 952             #[inline(always)]
 953             fn query_cache<'a>(tcx: TyCtxt<'a, $tcx, '_>) -> &'a Lock<QueryCache<$tcx, Self>> {
 954                 &tcx.queries.$name
 955             }
 956
 957             #[allow(unused)]
 958             #[inline(always)]
 959             fn to_dep_node(tcx: TyCtxt<'_, $tcx, '_>, key: &Self::Key) -> DepNode {
 960                 use crate::dep_graph::DepConstructor::*;
 961
 962                 DepNode::new(tcx, $node(*key))
 963             }
 964
 965             #[inline]
 966             fn compute(tcx: TyCtxt<'_, 'tcx, '_>, key: Self::Key) -> Self::Value {
 967                 __query_compute::$name(move || {
 968                     let provider = tcx.queries.providers.get(key.query_crate())
 969                         // HACK(eddyb) it's possible crates may be loaded after
 970                         // the query engine is created, and because crate loading
 971                         // is not yet integrated with the query engine, such crates
 972                         // would be missing appropriate entries in `providers`.
 973                         .unwrap_or(&tcx.queries.fallback_extern_providers)
 974                         .$name;
 975                     provider(tcx.global_tcx(), key)
 976                 })
 977             }
 978
 979             fn hash_result(
 980                 _hcx: &mut StableHashingContext<'_>,
 981                 _result: &Self::Value
 982             ) -> Option<Fingerprint> {
 983                 hash_result!([$($modifiers)*][_hcx, _result])
 984             }
 985
 986             fn handle_cycle_error(tcx: TyCtxt<'_, 'tcx, '_>) -> Self::Value {
 987                 handle_cycle_error!([$($modifiers)*][tcx])
 988             }
 989         })*
 990
 991         #[derive(Copy, Clone)]
 992         pub struct TyCtxtEnsure<'a, 'gcx: 'a+'tcx, 'tcx: 'a> {
 993             pub tcx: TyCtxt<'a, 'gcx, 'tcx>,
 994         }
 995
 996         impl<'a, $tcx, 'lcx> TyCtxtEnsure<'a, $tcx, 'lcx> {
 997             $($(#[$attr])*
 998             #[inline(always)]
 999             pub fn $name(self, key: $K) {
1000                 self.tcx.ensure_query::<queries::$name<'_>>(key)
1001             })*
1002         }
1003
1004         #[derive(Copy, Clone)]
1005         pub struct TyCtxtAt<'a, 'gcx: 'a+'tcx, 'tcx: 'a> {
1006             pub tcx: TyCtxt<'a, 'gcx, 'tcx>,
1007             pub span: Span,
1008         }
1009
1010         impl<'a, 'gcx, 'tcx> Deref for TyCtxtAt<'a, 'gcx, 'tcx> {
1011             type Target = TyCtxt<'a, 'gcx, 'tcx>;
1012             #[inline(always)]
1013             fn deref(&self) -> &Self::Target {
1014                 &self.tcx
1015             }
1016         }
1017
1018         impl<'a, $tcx, 'lcx> TyCtxt<'a, $tcx, 'lcx> {
1019             /// Return a transparent wrapper for `TyCtxt` which ensures queries
1020             /// are executed instead of returing their result
1021             #[inline(always)]
1022             pub fn ensure(self) -> TyCtxtEnsure<'a, $tcx, 'lcx> {
1023                 TyCtxtEnsure {
1024                     tcx: self,
1025                 }
1026             }
1027
1028             /// Return a transparent wrapper for `TyCtxt` which uses
1029             /// `span` as the location of queries performed through it.
1030             #[inline(always)]
1031             pub fn at(self, span: Span) -> TyCtxtAt<'a, $tcx, 'lcx> {
1032                 TyCtxtAt {
1033                     tcx: self,
1034                     span
1035                 }
1036             }
1037
1038             $($(#[$attr])*
1039             #[inline(always)]
1040             pub fn $name(self, key: $K) -> $V {
1041                 self.at(DUMMY_SP).$name(key)
1042             })*
1043         }
1044
1045         impl<'a, $tcx, 'lcx> TyCtxtAt<'a, $tcx, 'lcx> {
1046             $($(#[$attr])*
1047             #[inline(always)]
1048             pub fn $name(self, key: $K) -> $V {
1049                 self.tcx.get_query::<queries::$name<'_>>(self.span, key)
1050             })*
1051         }
1052
1053         define_provider_struct! {
1054             tcx: $tcx,
1055             input: ($(([$($modifiers)*] [$name] [$K] [$V]))*)
1056         }
1057
1058         impl<$tcx> Copy for Providers<$tcx> {}
1059         impl<$tcx> Clone for Providers<$tcx> {
1060             fn clone(&self) -> Self { *self }
1061         }
1062     }
1063 }
1064
1065 macro_rules! define_queries_struct {
1066     (tcx: $tcx:tt,
1067      input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
1068         pub struct Queries<$tcx> {
1069             /// This provides access to the incr. comp. on-disk cache for query results.
1070             /// Do not access this directly. It is only meant to be used by
1071             /// `DepGraph::try_mark_green()` and the query infrastructure.
1072             pub(crate) on_disk_cache: OnDiskCache<'tcx>,
1073
1074             providers: IndexVec<CrateNum, Providers<$tcx>>,
1075             fallback_extern_providers: Box<Providers<$tcx>>,
1076
1077             $($(#[$attr])*  $name: Lock<QueryCache<$tcx, queries::$name<$tcx>>>,)*
1078         }
1079     };
1080 }
1081
1082 macro_rules! define_provider_struct {
1083     (tcx: $tcx:tt,
1084      input: ($(([$($modifiers:tt)*] [$name:ident] [$K:ty] [$R:ty]))*)) => {
1085         pub struct Providers<$tcx> {
1086             $(pub $name: for<'a> fn(TyCtxt<'a, $tcx, $tcx>, $K) -> $R,)*
1087         }
1088
1089         impl<$tcx> Default for Providers<$tcx> {
1090             fn default() -> Self {
1091                 $(fn $name<'a, $tcx>(_: TyCtxt<'a, $tcx, $tcx>, key: $K) -> $R {
1092                     bug!("tcx.{}({:?}) unsupported by its crate",
1093                          stringify!($name), key);
1094                 })*
1095                 Providers { $($name),* }
1096             }
1097         }
1098     };
1099 }
1100
1101
1102 /// The red/green evaluation system will try to mark a specific DepNode in the
1103 /// dependency graph as green by recursively trying to mark the dependencies of
1104 /// that DepNode as green. While doing so, it will sometimes encounter a DepNode
1105 /// where we don't know if it is red or green and we therefore actually have
1106 /// to recompute its value in order to find out. Since the only piece of
1107 /// information that we have at that point is the DepNode we are trying to
1108 /// re-evaluate, we need some way to re-run a query from just that. This is what
1109 /// `force_from_dep_node()` implements.
1110 ///
1111 /// In the general case, a DepNode consists of a DepKind and an opaque
1112 /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint
1113 /// is usually constructed by computing a stable hash of the query-key that the
1114 /// DepNode corresponds to. Consequently, it is not in general possible to go
1115 /// back from hash to query-key (since hash functions are not reversible). For
1116 /// this reason `force_from_dep_node()` is expected to fail from time to time
1117 /// because we just cannot find out, from the DepNode alone, what the
1118 /// corresponding query-key is and therefore cannot re-run the query.
1119 ///
1120 /// The system deals with this case letting `try_mark_green` fail which forces
1121 /// the root query to be re-evaluated.
1122 ///
1123 /// Now, if force_from_dep_node() would always fail, it would be pretty useless.
1124 /// Fortunately, we can use some contextual information that will allow us to
1125 /// reconstruct query-keys for certain kinds of DepNodes. In particular, we
1126 /// enforce by construction that the GUID/fingerprint of certain DepNodes is a
1127 /// valid DefPathHash. Since we also always build a huge table that maps every
1128 /// DefPathHash in the current codebase to the corresponding DefId, we have
1129 /// everything we need to re-run the query.
1130 ///
1131 /// Take the `mir_validated` query as an example. Like many other queries, it
1132 /// just has a single parameter: the DefId of the item it will compute the
1133 /// validated MIR for. Now, when we call `force_from_dep_node()` on a dep-node
1134 /// with kind `MirValidated`, we know that the GUID/fingerprint of the dep-node
1135 /// is actually a DefPathHash, and can therefore just look up the corresponding
1136 /// DefId in `tcx.def_path_hash_to_def_id`.
1137 ///
1138 /// When you implement a new query, it will likely have a corresponding new
1139 /// DepKind, and you'll have to support it here in `force_from_dep_node()`. As
1140 /// a rule of thumb, if your query takes a DefId or DefIndex as sole parameter,
1141 /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
1142 /// add it to the "We don't have enough information to reconstruct..." group in
1143 /// the match below.
1144 pub fn force_from_dep_node<'a, 'gcx, 'lcx>(tcx: TyCtxt<'a, 'gcx, 'lcx>,
1145                                            dep_node: &DepNode)
1146                                            -> bool {
1147     use crate::hir::def_id::LOCAL_CRATE;
1148
1149     // We must avoid ever having to call force_from_dep_node() for a
1150     // DepNode::CodegenUnit:
1151     // Since we cannot reconstruct the query key of a DepNode::CodegenUnit, we
1152     // would always end up having to evaluate the first caller of the
1153     // `codegen_unit` query that *is* reconstructible. This might very well be
1154     // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
1155     // to re-trigger calling the `codegen_unit` query with the right key. At
1156     // that point we would already have re-done all the work we are trying to
1157     // avoid doing in the first place.
1158     // The solution is simple: Just explicitly call the `codegen_unit` query for
1159     // each CGU, right after partitioning. This way `try_mark_green` will always
1160     // hit the cache instead of having to go through `force_from_dep_node`.
1161     // This assertion makes sure, we actually keep applying the solution above.
1162     debug_assert!(dep_node.kind != DepKind::CodegenUnit,
1163                   "calling force_from_dep_node() on DepKind::CodegenUnit");
1164
1165     if !dep_node.kind.can_reconstruct_query_key() {
1166         return false
1167     }
1168
1169     macro_rules! def_id {
1170         () => {
1171             if let Some(def_id) = dep_node.extract_def_id(tcx) {
1172                 def_id
1173             } else {
1174                 // return from the whole function
1175                 return false
1176             }
1177         }
1178     };
1179
1180     macro_rules! krate {
1181         () => { (def_id!()).krate }
1182     };
1183
1184     macro_rules! force {
1185         ($query:ident, $key:expr) => {
1186             {
1187                 tcx.force_query::<crate::ty::query::queries::$query<'_>>($key, DUMMY_SP, *dep_node);
1188             }
1189         }
1190     };
1191
1192     // FIXME(#45015): We should try move this boilerplate code into a macro
1193     //                somehow.
1194     match dep_node.kind {
1195         // These are inputs that are expected to be pre-allocated and that
1196         // should therefore always be red or green already
1197         DepKind::AllLocalTraitImpls |
1198         DepKind::Krate |
1199         DepKind::CrateMetadata |
1200         DepKind::HirBody |
1201         DepKind::Hir |
1202
1203         // This are anonymous nodes
1204         DepKind::TraitSelect |
1205
1206         // We don't have enough information to reconstruct the query key of
1207         // these
1208         DepKind::IsCopy |
1209         DepKind::IsSized |
1210         DepKind::IsFreeze |
1211         DepKind::NeedsDrop |
1212         DepKind::Layout |
1213         DepKind::ConstEval |
1214         DepKind::ConstEvalRaw |
1215         DepKind::InstanceSymbolName |
1216         DepKind::MirShim |
1217         DepKind::BorrowCheckKrate |
1218         DepKind::Specializes |
1219         DepKind::ImplementationsOfTrait |
1220         DepKind::TypeParamPredicates |
1221         DepKind::CodegenUnit |
1222         DepKind::CompileCodegenUnit |
1223         DepKind::FulfillObligation |
1224         DepKind::VtableMethods |
1225         DepKind::EraseRegionsTy |
1226         DepKind::NormalizeProjectionTy |
1227         DepKind::NormalizeTyAfterErasingRegions |
1228         DepKind::ImpliedOutlivesBounds |
1229         DepKind::DropckOutlives |
1230         DepKind::EvaluateObligation |
1231         DepKind::EvaluateGoal |
1232         DepKind::TypeOpAscribeUserType |
1233         DepKind::TypeOpEq |
1234         DepKind::TypeOpSubtype |
1235         DepKind::TypeOpProvePredicate |
1236         DepKind::TypeOpNormalizeTy |
1237         DepKind::TypeOpNormalizePredicate |
1238         DepKind::TypeOpNormalizePolyFnSig |
1239         DepKind::TypeOpNormalizeFnSig |
1240         DepKind::SubstituteNormalizeAndTestPredicates |
1241         DepKind::MethodAutoderefSteps |
1242         DepKind::InstanceDefSizeEstimate |
1243         DepKind::ProgramClausesForEnv |
1244
1245         // This one should never occur in this context
1246         DepKind::Null => {
1247             bug!("force_from_dep_node() - Encountered {:?}", dep_node)
1248         }
1249
1250         // These are not queries
1251         DepKind::CoherenceCheckTrait |
1252         DepKind::ItemVarianceConstraints => {
1253             return false
1254         }
1255
1256         DepKind::RegionScopeTree => { force!(region_scope_tree, def_id!()); }
1257
1258         DepKind::Coherence => { force!(crate_inherent_impls, LOCAL_CRATE); }
1259         DepKind::CoherenceInherentImplOverlapCheck => {
1260             force!(crate_inherent_impls_overlap_check, LOCAL_CRATE)
1261         },
1262         DepKind::PrivacyAccessLevels => { force!(privacy_access_levels, LOCAL_CRATE); }
1263         DepKind::MirBuilt => { force!(mir_built, def_id!()); }
1264         DepKind::MirConstQualif => { force!(mir_const_qualif, def_id!()); }
1265         DepKind::MirConst => { force!(mir_const, def_id!()); }
1266         DepKind::MirValidated => { force!(mir_validated, def_id!()); }
1267         DepKind::MirOptimized => { force!(optimized_mir, def_id!()); }
1268
1269         DepKind::BorrowCheck => { force!(borrowck, def_id!()); }
1270         DepKind::MirBorrowCheck => { force!(mir_borrowck, def_id!()); }
1271         DepKind::UnsafetyCheckResult => { force!(unsafety_check_result, def_id!()); }
1272         DepKind::UnsafeDeriveOnReprPacked => { force!(unsafe_derive_on_repr_packed, def_id!()); }
1273         DepKind::CheckModAttrs => { force!(check_mod_attrs, def_id!()); }
1274         DepKind::CheckModLoops => { force!(check_mod_loops, def_id!()); }
1275         DepKind::CheckModUnstableApiUsage => { force!(check_mod_unstable_api_usage, def_id!()); }
1276         DepKind::CheckModItemTypes => { force!(check_mod_item_types, def_id!()); }
1277         DepKind::CheckModPrivacy => { force!(check_mod_privacy, def_id!()); }
1278         DepKind::CheckModIntrinsics => { force!(check_mod_intrinsics, def_id!()); }
1279         DepKind::CheckModLiveness => { force!(check_mod_liveness, def_id!()); }
1280         DepKind::CheckModImplWf => { force!(check_mod_impl_wf, def_id!()); }
1281         DepKind::CollectModItemTypes => { force!(collect_mod_item_types, def_id!()); }
1282         DepKind::Reachability => { force!(reachable_set, LOCAL_CRATE); }
1283         DepKind::MirKeys => { force!(mir_keys, LOCAL_CRATE); }
1284         DepKind::CrateVariances => { force!(crate_variances, LOCAL_CRATE); }
1285         DepKind::AssociatedItems => { force!(associated_item, def_id!()); }
1286         DepKind::TypeOfItem => { force!(type_of, def_id!()); }
1287         DepKind::GenericsOfItem => { force!(generics_of, def_id!()); }
1288         DepKind::PredicatesOfItem => { force!(predicates_of, def_id!()); }
1289         DepKind::PredicatesDefinedOnItem => { force!(predicates_defined_on, def_id!()); }
1290         DepKind::ExplicitPredicatesOfItem => { force!(explicit_predicates_of, def_id!()); }
1291         DepKind::InferredOutlivesOf => { force!(inferred_outlives_of, def_id!()); }
1292         DepKind::InferredOutlivesCrate => { force!(inferred_outlives_crate, LOCAL_CRATE); }
1293         DepKind::SuperPredicatesOfItem => { force!(super_predicates_of, def_id!()); }
1294         DepKind::TraitDefOfItem => { force!(trait_def, def_id!()); }
1295         DepKind::AdtDefOfItem => { force!(adt_def, def_id!()); }
1296         DepKind::ImplTraitRef => { force!(impl_trait_ref, def_id!()); }
1297         DepKind::ImplPolarity => { force!(impl_polarity, def_id!()); }
1298         DepKind::Issue33140SelfTy => { force!(issue33140_self_ty, def_id!()); }
1299         DepKind::FnSignature => { force!(fn_sig, def_id!()); }
1300         DepKind::CoerceUnsizedInfo => { force!(coerce_unsized_info, def_id!()); }
1301         DepKind::ItemVariances => { force!(variances_of, def_id!()); }
1302         DepKind::IsConstFn => { force!(is_const_fn_raw, def_id!()); }
1303         DepKind::IsPromotableConstFn => { force!(is_promotable_const_fn, def_id!()); }
1304         DepKind::IsForeignItem => { force!(is_foreign_item, def_id!()); }
1305         DepKind::SizedConstraint => { force!(adt_sized_constraint, def_id!()); }
1306         DepKind::DtorckConstraint => { force!(adt_dtorck_constraint, def_id!()); }
1307         DepKind::AdtDestructor => { force!(adt_destructor, def_id!()); }
1308         DepKind::AssociatedItemDefIds => { force!(associated_item_def_ids, def_id!()); }
1309         DepKind::InherentImpls => { force!(inherent_impls, def_id!()); }
1310         DepKind::TypeckBodiesKrate => { force!(typeck_item_bodies, LOCAL_CRATE); }
1311         DepKind::TypeckTables => { force!(typeck_tables_of, def_id!()); }
1312         DepKind::UsedTraitImports => { force!(used_trait_imports, def_id!()); }
1313         DepKind::HasTypeckTables => { force!(has_typeck_tables, def_id!()); }
1314         DepKind::SymbolName => { force!(def_symbol_name, def_id!()); }
1315         DepKind::SpecializationGraph => { force!(specialization_graph_of, def_id!()); }
1316         DepKind::ObjectSafety => { force!(is_object_safe, def_id!()); }
1317         DepKind::TraitImpls => { force!(trait_impls_of, def_id!()); }
1318         DepKind::CheckMatch => { force!(check_match, def_id!()); }
1319
1320         DepKind::ParamEnv => { force!(param_env, def_id!()); }
1321         DepKind::Environment => { force!(environment, def_id!()); }
1322         DepKind::DescribeDef => { force!(describe_def, def_id!()); }
1323         DepKind::DefSpan => { force!(def_span, def_id!()); }
1324         DepKind::LookupStability => { force!(lookup_stability, def_id!()); }
1325         DepKind::LookupDeprecationEntry => {
1326             force!(lookup_deprecation_entry, def_id!());
1327         }
1328         DepKind::ConstIsRvaluePromotableToStatic => {
1329             force!(const_is_rvalue_promotable_to_static, def_id!());
1330         }
1331         DepKind::RvaluePromotableMap => { force!(rvalue_promotable_map, def_id!()); }
1332         DepKind::ImplParent => { force!(impl_parent, def_id!()); }
1333         DepKind::TraitOfItem => { force!(trait_of_item, def_id!()); }
1334         DepKind::IsReachableNonGeneric => { force!(is_reachable_non_generic, def_id!()); }
1335         DepKind::IsUnreachableLocalDefinition => {
1336             force!(is_unreachable_local_definition, def_id!());
1337         }
1338         DepKind::IsMirAvailable => { force!(is_mir_available, def_id!()); }
1339         DepKind::ItemAttrs => { force!(item_attrs, def_id!()); }
1340         DepKind::CodegenFnAttrs => { force!(codegen_fn_attrs, def_id!()); }
1341         DepKind::FnArgNames => { force!(fn_arg_names, def_id!()); }
1342         DepKind::RenderedConst => { force!(rendered_const, def_id!()); }
1343         DepKind::DylibDepFormats => { force!(dylib_dependency_formats, krate!()); }
1344         DepKind::IsPanicRuntime => { force!(is_panic_runtime, krate!()); }
1345         DepKind::IsCompilerBuiltins => { force!(is_compiler_builtins, krate!()); }
1346         DepKind::HasGlobalAllocator => { force!(has_global_allocator, krate!()); }
1347         DepKind::HasPanicHandler => { force!(has_panic_handler, krate!()); }
1348         DepKind::ExternCrate => { force!(extern_crate, def_id!()); }
1349         DepKind::LintLevels => { force!(lint_levels, LOCAL_CRATE); }
1350         DepKind::InScopeTraits => { force!(in_scope_traits_map, def_id!().index); }
1351         DepKind::ModuleExports => { force!(module_exports, def_id!()); }
1352         DepKind::IsSanitizerRuntime => { force!(is_sanitizer_runtime, krate!()); }
1353         DepKind::IsProfilerRuntime => { force!(is_profiler_runtime, krate!()); }
1354         DepKind::GetPanicStrategy => { force!(panic_strategy, krate!()); }
1355         DepKind::IsNoBuiltins => { force!(is_no_builtins, krate!()); }
1356         DepKind::ImplDefaultness => { force!(impl_defaultness, def_id!()); }
1357         DepKind::CheckItemWellFormed => { force!(check_item_well_formed, def_id!()); }
1358         DepKind::CheckTraitItemWellFormed => { force!(check_trait_item_well_formed, def_id!()); }
1359         DepKind::CheckImplItemWellFormed => { force!(check_impl_item_well_formed, def_id!()); }
1360         DepKind::ReachableNonGenerics => { force!(reachable_non_generics, krate!()); }
1361         DepKind::NativeLibraries => { force!(native_libraries, krate!()); }
1362         DepKind::EntryFn => { force!(entry_fn, krate!()); }
1363         DepKind::PluginRegistrarFn => { force!(plugin_registrar_fn, krate!()); }
1364         DepKind::ProcMacroDeclsStatic => { force!(proc_macro_decls_static, krate!()); }
1365         DepKind::CrateDisambiguator => { force!(crate_disambiguator, krate!()); }
1366         DepKind::CrateHash => { force!(crate_hash, krate!()); }
1367         DepKind::OriginalCrateName => { force!(original_crate_name, krate!()); }
1368         DepKind::ExtraFileName => { force!(extra_filename, krate!()); }
1369
1370         DepKind::AllTraitImplementations => {
1371             force!(all_trait_implementations, krate!());
1372         }
1373
1374         DepKind::DllimportForeignItems => {
1375             force!(dllimport_foreign_items, krate!());
1376         }
1377         DepKind::IsDllimportForeignItem => {
1378             force!(is_dllimport_foreign_item, def_id!());
1379         }
1380         DepKind::IsStaticallyIncludedForeignItem => {
1381             force!(is_statically_included_foreign_item, def_id!());
1382         }
1383         DepKind::NativeLibraryKind => { force!(native_library_kind, def_id!()); }
1384         DepKind::LinkArgs => { force!(link_args, LOCAL_CRATE); }
1385
1386         DepKind::ResolveLifetimes => { force!(resolve_lifetimes, krate!()); }
1387         DepKind::NamedRegion => { force!(named_region_map, def_id!().index); }
1388         DepKind::IsLateBound => { force!(is_late_bound_map, def_id!().index); }
1389         DepKind::ObjectLifetimeDefaults => {
1390             force!(object_lifetime_defaults_map, def_id!().index);
1391         }
1392
1393         DepKind::Visibility => { force!(visibility, def_id!()); }
1394         DepKind::DepKind => { force!(dep_kind, krate!()); }
1395         DepKind::CrateName => { force!(crate_name, krate!()); }
1396         DepKind::ItemChildren => { force!(item_children, def_id!()); }
1397         DepKind::ExternModStmtCnum => { force!(extern_mod_stmt_cnum, def_id!()); }
1398         DepKind::GetLibFeatures => { force!(get_lib_features, LOCAL_CRATE); }
1399         DepKind::DefinedLibFeatures => { force!(defined_lib_features, krate!()); }
1400         DepKind::GetLangItems => { force!(get_lang_items, LOCAL_CRATE); }
1401         DepKind::DefinedLangItems => { force!(defined_lang_items, krate!()); }
1402         DepKind::MissingLangItems => { force!(missing_lang_items, krate!()); }
1403         DepKind::VisibleParentMap => { force!(visible_parent_map, LOCAL_CRATE); }
1404         DepKind::MissingExternCrateItem => {
1405             force!(missing_extern_crate_item, krate!());
1406         }
1407         DepKind::UsedCrateSource => { force!(used_crate_source, krate!()); }
1408         DepKind::PostorderCnums => { force!(postorder_cnums, LOCAL_CRATE); }
1409
1410         DepKind::Freevars => { force!(freevars, def_id!()); }
1411         DepKind::MaybeUnusedTraitImport => {
1412             force!(maybe_unused_trait_import, def_id!());
1413         }
1414         DepKind::NamesImportedByGlobUse => { force!(names_imported_by_glob_use, def_id!()); }
1415         DepKind::MaybeUnusedExternCrates => { force!(maybe_unused_extern_crates, LOCAL_CRATE); }
1416         DepKind::StabilityIndex => { force!(stability_index, LOCAL_CRATE); }
1417         DepKind::AllTraits => { force!(all_traits, LOCAL_CRATE); }
1418         DepKind::AllCrateNums => { force!(all_crate_nums, LOCAL_CRATE); }
1419         DepKind::ExportedSymbols => { force!(exported_symbols, krate!()); }
1420         DepKind::CollectAndPartitionMonoItems => {
1421             force!(collect_and_partition_mono_items, LOCAL_CRATE);
1422         }
1423         DepKind::IsCodegenedItem => { force!(is_codegened_item, def_id!()); }
1424         DepKind::OutputFilenames => { force!(output_filenames, LOCAL_CRATE); }
1425
1426         DepKind::TargetFeaturesWhitelist => { force!(target_features_whitelist, LOCAL_CRATE); }
1427
1428         DepKind::Features => { force!(features_query, LOCAL_CRATE); }
1429
1430         DepKind::ProgramClausesFor => { force!(program_clauses_for, def_id!()); }
1431         DepKind::WasmImportModuleMap => { force!(wasm_import_module_map, krate!()); }
1432         DepKind::ForeignModules => { force!(foreign_modules, krate!()); }
1433
1434         DepKind::UpstreamMonomorphizations => {
1435             force!(upstream_monomorphizations, krate!());
1436         }
1437         DepKind::UpstreamMonomorphizationsFor => {
1438             force!(upstream_monomorphizations_for, def_id!());
1439         }
1440         DepKind::BackendOptimizationLevel => {
1441             force!(backend_optimization_level, krate!());
1442         }
1443     }
1444
1445     true
1446 }
1447
1448
1449 // FIXME(#45015): Another piece of boilerplate code that could be generated in
1450 //                a combined define_dep_nodes!()/define_queries!() macro.
1451 macro_rules! impl_load_from_cache {
1452     ($($dep_kind:ident => $query_name:ident,)*) => {
1453         impl DepNode {
1454             // Check whether the query invocation corresponding to the given
1455             // DepNode is eligible for on-disk-caching.
1456             pub fn cache_on_disk(&self, tcx: TyCtxt<'_, '_, '_>) -> bool {
1457                 use crate::ty::query::queries;
1458                 use crate::ty::query::QueryDescription;
1459
1460                 match self.kind {
1461                     $(DepKind::$dep_kind => {
1462                         let def_id = self.extract_def_id(tcx).unwrap();
1463                         queries::$query_name::cache_on_disk(tcx.global_tcx(), def_id)
1464                     })*
1465                     _ => false
1466                 }
1467             }
1468
1469             // This is method will execute the query corresponding to the given
1470             // DepNode. It is only expected to work for DepNodes where the
1471             // above `cache_on_disk` methods returns true.
1472             // Also, as a sanity check, it expects that the corresponding query
1473             // invocation has been marked as green already.
1474             pub fn load_from_on_disk_cache(&self, tcx: TyCtxt<'_, '_, '_>) {
1475                 match self.kind {
1476                     $(DepKind::$dep_kind => {
1477                         debug_assert!(tcx.dep_graph
1478                                          .node_color(self)
1479                                          .map(|c| c.is_green())
1480                                          .unwrap_or(false));
1481
1482                         let def_id = self.extract_def_id(tcx).unwrap();
1483                         let _ = tcx.$query_name(def_id);
1484                     })*
1485                     _ => {
1486                         bug!()
1487                     }
1488                 }
1489             }
1490         }
1491     }
1492 }
1493
1494 impl_load_from_cache!(
1495     TypeckTables => typeck_tables_of,
1496     MirOptimized => optimized_mir,
1497     UnsafetyCheckResult => unsafety_check_result,
1498     BorrowCheck => borrowck,
1499     MirBorrowCheck => mir_borrowck,
1500     MirConstQualif => mir_const_qualif,
1501     SymbolName => def_symbol_name,
1502     ConstIsRvaluePromotableToStatic => const_is_rvalue_promotable_to_static,
1503     CheckMatch => check_match,
1504     TypeOfItem => type_of,
1505     GenericsOfItem => generics_of,
1506     PredicatesOfItem => predicates_of,
1507     UsedTraitImports => used_trait_imports,
1508     CodegenFnAttrs => codegen_fn_attrs,
1509     SpecializationGraph => specialization_graph_of,
1510 );