src/librustc/ty/query/plumbing.rs

   1 //! The implementation of the query system itself. This defines the macros that
   2 //! generate the actual methods on tcx which find and execute the provider,
   3 //! manage the caches, and so forth.
   4
   5 use crate::dep_graph::{DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex};
   6 use crate::ty::query::caches::QueryCache;
   7 use crate::ty::query::config::QueryDescription;
   8 use crate::ty::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId};
   9 use crate::ty::query::Query;
  10 use crate::ty::tls;
  11 use crate::ty::{self, TyCtxt};
  12
  13 #[cfg(not(parallel_compiler))]
  14 use rustc_data_structures::cold_path;
  15 use rustc_data_structures::fx::{FxHashMap, FxHasher};
  16 use rustc_data_structures::sharded::Sharded;
  17 use rustc_data_structures::sync::{Lock, LockGuard};
  18 use rustc_data_structures::thin_vec::ThinVec;
  19 use rustc_errors::{struct_span_err, Diagnostic, DiagnosticBuilder, FatalError, Handler, Level};
  20 use rustc_span::source_map::DUMMY_SP;
  21 use rustc_span::Span;
  22 use std::collections::hash_map::Entry;
  23 use std::convert::TryFrom;
  24 use std::fmt::Debug;
  25 use std::hash::{Hash, Hasher};
  26 use std::mem;
  27 use std::num::NonZeroU32;
  28 use std::ptr;
  29 #[cfg(debug_assertions)]
  30 use std::sync::atomic::{AtomicUsize, Ordering};
  31
  32 pub(crate) struct QueryStateShard<'tcx, K, C> {
  33     pub(super) cache: C,
  34     pub(super) active: FxHashMap<K, QueryResult<'tcx>>,
  35
  36     /// Used to generate unique ids for active jobs.
  37     pub(super) jobs: u32,
  38 }
  39
  40 impl<'tcx, K, C> QueryStateShard<'tcx, K, C> {
  41     fn get_cache(&mut self) -> &mut C {
  42         &mut self.cache
  43     }
  44 }
  45
  46 impl<'tcx, K, C: Default> Default for QueryStateShard<'tcx, K, C> {
  47     fn default() -> QueryStateShard<'tcx, K, C> {
  48         QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 }
  49     }
  50 }
  51
  52 pub(crate) struct QueryState<'tcx, C: QueryCache> {
  53     pub(super) cache: C,
  54     pub(super) shards: Sharded<QueryStateShard<'tcx, C::Key, C::Sharded>>,
  55     #[cfg(debug_assertions)]
  56     pub(super) cache_hits: AtomicUsize,
  57 }
  58
  59 impl<'tcx, C: QueryCache> QueryState<'tcx, C> {
  60     pub(super) fn get_lookup<K2: Hash>(
  61         &'tcx self,
  62         key: &K2,
  63     ) -> QueryLookup<'tcx, C::Key, C::Sharded> {
  64         // We compute the key's hash once and then use it for both the
  65         // shard lookup and the hashmap lookup. This relies on the fact
  66         // that both of them use `FxHasher`.
  67         let mut hasher = FxHasher::default();
  68         key.hash(&mut hasher);
  69         let key_hash = hasher.finish();
  70
  71         let shard = self.shards.get_shard_index_by_hash(key_hash);
  72         let lock = self.shards.get_shard_by_index(shard).lock();
  73         QueryLookup { key_hash, shard, lock }
  74     }
  75 }
  76
  77 /// Indicates the state of a query for a given key in a query map.
  78 pub(super) enum QueryResult<'tcx> {
  79     /// An already executing query. The query job can be used to await for its completion.
  80     Started(QueryJob<'tcx>),
  81
  82     /// The query panicked. Queries trying to wait on this will raise a fatal error which will
  83     /// silently panic.
  84     Poisoned,
  85 }
  86
  87 impl<'tcx, C: QueryCache> QueryState<'tcx, C> {
  88     pub fn iter_results<R>(
  89         &self,
  90         f: impl for<'a> FnOnce(
  91             Box<dyn Iterator<Item = (&'a C::Key, &'a C::Value, DepNodeIndex)> + 'a>,
  92         ) -> R,
  93     ) -> R {
  94         self.cache.iter(&self.shards, |shard| &mut shard.cache, f)
  95     }
  96     pub fn all_inactive(&self) -> bool {
  97         let shards = self.shards.lock_shards();
  98         shards.iter().all(|shard| shard.active.is_empty())
  99     }
 100
 101     pub(super) fn try_collect_active_jobs(
 102         &self,
 103         kind: DepKind,
 104         make_query: fn(C::Key) -> Query<'tcx>,
 105         jobs: &mut FxHashMap<QueryJobId, QueryJobInfo<'tcx>>,
 106     ) -> Option<()>
 107     where
 108         C::Key: Clone,
 109     {
 110         // We use try_lock_shards here since we are called from the
 111         // deadlock handler, and this shouldn't be locked.
 112         let shards = self.shards.try_lock_shards()?;
 113         let shards = shards.iter().enumerate();
 114         jobs.extend(shards.flat_map(|(shard_id, shard)| {
 115             shard.active.iter().filter_map(move |(k, v)| {
 116                 if let QueryResult::Started(ref job) = *v {
 117                     let id =
 118                         QueryJobId { job: job.id, shard: u16::try_from(shard_id).unwrap(), kind };
 119                     let info = QueryInfo { span: job.span, query: make_query(k.clone()) };
 120                     Some((id, QueryJobInfo { info, job: job.clone() }))
 121                 } else {
 122                     None
 123                 }
 124             })
 125         }));
 126
 127         Some(())
 128     }
 129 }
 130
 131 impl<'tcx, C: QueryCache> Default for QueryState<'tcx, C> {
 132     fn default() -> QueryState<'tcx, C> {
 133         QueryState {
 134             cache: C::default(),
 135             shards: Default::default(),
 136             #[cfg(debug_assertions)]
 137             cache_hits: AtomicUsize::new(0),
 138         }
 139     }
 140 }
 141
 142 /// Values used when checking a query cache which can be reused on a cache-miss to execute the query.
 143 pub(crate) struct QueryLookup<'tcx, K, C> {
 144     pub(super) key_hash: u64,
 145     pub(super) shard: usize,
 146     pub(super) lock: LockGuard<'tcx, QueryStateShard<'tcx, K, C>>,
 147 }
 148
 149 /// A type representing the responsibility to execute the job in the `job` field.
 150 /// This will poison the relevant query if dropped.
 151 pub(super) struct JobOwner<'tcx, C>
 152 where
 153     C: QueryCache,
 154     C::Key: Eq + Hash + Clone + Debug,
 155     C::Value: Clone,
 156 {
 157     state: &'tcx QueryState<'tcx, C>,
 158     key: C::Key,
 159     id: QueryJobId,
 160 }
 161
 162 impl<'tcx, C: QueryCache> JobOwner<'tcx, C>
 163 where
 164     C: QueryCache,
 165     C::Key: Eq + Hash + Clone + Debug,
 166     C::Value: Clone,
 167 {
 168     /// Either gets a `JobOwner` corresponding the query, allowing us to
 169     /// start executing the query, or returns with the result of the query.
 170     /// This function assumes that `try_get_cached` is already called and returned `lookup`.
 171     /// If the query is executing elsewhere, this will wait for it and return the result.
 172     /// If the query panicked, this will silently panic.
 173     ///
 174     /// This function is inlined because that results in a noticeable speed-up
 175     /// for some compile-time benchmarks.
 176     #[inline(always)]
 177     pub(super) fn try_start<Q>(
 178         tcx: TyCtxt<'tcx>,
 179         span: Span,
 180         key: &C::Key,
 181         mut lookup: QueryLookup<'tcx, C::Key, C::Sharded>,
 182     ) -> TryGetJob<'tcx, C>
 183     where
 184         Q: QueryDescription<'tcx, Key = C::Key, Value = C::Value, Cache = C>,
 185     {
 186         let lock = &mut *lookup.lock;
 187
 188         let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) {
 189             Entry::Occupied(mut entry) => {
 190                 match entry.get_mut() {
 191                     QueryResult::Started(job) => {
 192                         // For parallel queries, we'll block and wait until the query running
 193                         // in another thread has completed. Record how long we wait in the
 194                         // self-profiler.
 195                         let _query_blocked_prof_timer = if cfg!(parallel_compiler) {
 196                             Some(tcx.prof.query_blocked())
 197                         } else {
 198                             None
 199                         };
 200
 201                         // Create the id of the job we're waiting for
 202                         let id = QueryJobId::new(job.id, lookup.shard, Q::DEP_KIND);
 203
 204                         (job.latch(id), _query_blocked_prof_timer)
 205                     }
 206                     QueryResult::Poisoned => FatalError.raise(),
 207                 }
 208             }
 209             Entry::Vacant(entry) => {
 210                 // No job entry for this query. Return a new one to be started later.
 211
 212                 // Generate an id unique within this shard.
 213                 let id = lock.jobs.checked_add(1).unwrap();
 214                 lock.jobs = id;
 215                 let id = QueryShardJobId(NonZeroU32::new(id).unwrap());
 216
 217                 let global_id = QueryJobId::new(id, lookup.shard, Q::DEP_KIND);
 218
 219                 let job = tls::with_related_context(tcx, |icx| QueryJob::new(id, span, icx.query));
 220
 221                 entry.insert(QueryResult::Started(job));
 222
 223                 let owner =
 224                     JobOwner { state: Q::query_state(tcx), id: global_id, key: (*key).clone() };
 225                 return TryGetJob::NotYetStarted(owner);
 226             }
 227         };
 228         mem::drop(lookup.lock);
 229
 230         // If we are single-threaded we know that we have cycle error,
 231         // so we just return the error.
 232         #[cfg(not(parallel_compiler))]
 233         return TryGetJob::Cycle(cold_path(|| {
 234             Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span))
 235         }));
 236
 237         // With parallel queries we might just have to wait on some other
 238         // thread.
 239         #[cfg(parallel_compiler)]
 240         {
 241             let result = latch.wait_on(tcx, span);
 242
 243             if let Err(cycle) = result {
 244                 return TryGetJob::Cycle(Q::handle_cycle_error(tcx, cycle));
 245             }
 246
 247             let cached = tcx.try_get_cached(
 248                 Q::query_state(tcx),
 249                 (*key).clone(),
 250                 |value, index| (value.clone(), index),
 251                 |_, _| panic!("value must be in cache after waiting"),
 252             );
 253
 254             if let Some(prof_timer) = _query_blocked_prof_timer.take() {
 255                 prof_timer.finish_with_query_invocation_id(cached.1.into());
 256             }
 257
 258             return TryGetJob::JobCompleted(cached);
 259         }
 260     }
 261
 262     /// Completes the query by updating the query cache with the `result`,
 263     /// signals the waiter and forgets the JobOwner, so it won't poison the query
 264     #[inline(always)]
 265     pub(super) fn complete(
 266         self,
 267         tcx: TyCtxt<'tcx>,
 268         result: &C::Value,
 269         dep_node_index: DepNodeIndex,
 270     ) {
 271         // We can move out of `self` here because we `mem::forget` it below
 272         let key = unsafe { ptr::read(&self.key) };
 273         let state = self.state;
 274
 275         // Forget ourself so our destructor won't poison the query
 276         mem::forget(self);
 277
 278         let job = {
 279             let result = result.clone();
 280             let mut lock = state.shards.get_shard_by_value(&key).lock();
 281             let job = match lock.active.remove(&key).unwrap() {
 282                 QueryResult::Started(job) => job,
 283                 QueryResult::Poisoned => panic!(),
 284             };
 285             state.cache.complete(tcx, &mut lock.cache, key, result, dep_node_index);
 286             job
 287         };
 288
 289         job.signal_complete();
 290     }
 291 }
 292
 293 #[inline(always)]
 294 fn with_diagnostics<F, R>(f: F) -> (R, ThinVec<Diagnostic>)
 295 where
 296     F: FnOnce(Option<&Lock<ThinVec<Diagnostic>>>) -> R,
 297 {
 298     let diagnostics = Lock::new(ThinVec::new());
 299     let result = f(Some(&diagnostics));
 300     (result, diagnostics.into_inner())
 301 }
 302
 303 impl<'tcx, C: QueryCache> Drop for JobOwner<'tcx, C>
 304 where
 305     C::Key: Eq + Hash + Clone + Debug,
 306     C::Value: Clone,
 307 {
 308     #[inline(never)]
 309     #[cold]
 310     fn drop(&mut self) {
 311         // Poison the query so jobs waiting on it panic.
 312         let state = self.state;
 313         let shard = state.shards.get_shard_by_value(&self.key);
 314         let job = {
 315             let mut shard = shard.lock();
 316             let job = match shard.active.remove(&self.key).unwrap() {
 317                 QueryResult::Started(job) => job,
 318                 QueryResult::Poisoned => panic!(),
 319             };
 320             shard.active.insert(self.key.clone(), QueryResult::Poisoned);
 321             job
 322         };
 323         // Also signal the completion of the job, so waiters
 324         // will continue execution.
 325         job.signal_complete();
 326     }
 327 }
 328
 329 #[derive(Clone)]
 330 pub struct CycleError<'tcx> {
 331     /// The query and related span that uses the cycle.
 332     pub(super) usage: Option<(Span, Query<'tcx>)>,
 333     pub(super) cycle: Vec<QueryInfo<'tcx>>,
 334 }
 335
 336 /// The result of `try_start`.
 337 pub(super) enum TryGetJob<'tcx, C: QueryCache>
 338 where
 339     C::Key: Eq + Hash + Clone + Debug,
 340     C::Value: Clone,
 341 {
 342     /// The query is not yet started. Contains a guard to the cache eventually used to start it.
 343     NotYetStarted(JobOwner<'tcx, C>),
 344
 345     /// The query was already completed.
 346     /// Returns the result of the query and its dep-node index
 347     /// if it succeeded or a cycle error if it failed.
 348     #[cfg(parallel_compiler)]
 349     JobCompleted((C::Value, DepNodeIndex)),
 350
 351     /// Trying to execute the query resulted in a cycle.
 352     Cycle(C::Value),
 353 }
 354
 355 impl<'tcx> TyCtxt<'tcx> {
 356     /// Executes a job by changing the `ImplicitCtxt` to point to the
 357     /// new query job while it executes. It returns the diagnostics
 358     /// captured during execution and the actual result.
 359     #[inline(always)]
 360     pub(super) fn start_query<F, R>(
 361         self,
 362         token: QueryJobId,
 363         diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
 364         compute: F,
 365     ) -> R
 366     where
 367         F: FnOnce(TyCtxt<'tcx>) -> R,
 368     {
 369         // The `TyCtxt` stored in TLS has the same global interner lifetime
 370         // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
 371         // when accessing the `ImplicitCtxt`.
 372         tls::with_related_context(self, move |current_icx| {
 373             // Update the `ImplicitCtxt` to point to our new query job.
 374             let new_icx = tls::ImplicitCtxt {
 375                 tcx: self,
 376                 query: Some(token),
 377                 diagnostics,
 378                 layout_depth: current_icx.layout_depth,
 379                 task_deps: current_icx.task_deps,
 380             };
 381
 382             // Use the `ImplicitCtxt` while we execute the query.
 383             tls::enter_context(&new_icx, |_| compute(self))
 384         })
 385     }
 386
 387     #[inline(never)]
 388     #[cold]
 389     pub(super) fn report_cycle(
 390         self,
 391         CycleError { usage, cycle: stack }: CycleError<'tcx>,
 392     ) -> DiagnosticBuilder<'tcx> {
 393         assert!(!stack.is_empty());
 394
 395         let fix_span = |span: Span, query: &Query<'tcx>| {
 396             self.sess.source_map().def_span(query.default_span(self, span))
 397         };
 398
 399         // Disable naming impls with types in this path, since that
 400         // sometimes cycles itself, leading to extra cycle errors.
 401         // (And cycle errors around impls tend to occur during the
 402         // collect/coherence phases anyhow.)
 403         ty::print::with_forced_impl_filename_line(|| {
 404             let span = fix_span(stack[1 % stack.len()].span, &stack[0].query);
 405             let mut err = struct_span_err!(
 406                 self.sess,
 407                 span,
 408                 E0391,
 409                 "cycle detected when {}",
 410                 stack[0].query.describe(self)
 411             );
 412
 413             for i in 1..stack.len() {
 414                 let query = &stack[i].query;
 415                 let span = fix_span(stack[(i + 1) % stack.len()].span, query);
 416                 err.span_note(span, &format!("...which requires {}...", query.describe(self)));
 417             }
 418
 419             err.note(&format!(
 420                 "...which again requires {}, completing the cycle",
 421                 stack[0].query.describe(self)
 422             ));
 423
 424             if let Some((span, query)) = usage {
 425                 err.span_note(
 426                     fix_span(span, &query),
 427                     &format!("cycle used when {}", query.describe(self)),
 428                 );
 429             }
 430
 431             err
 432         })
 433     }
 434
 435     pub fn try_print_query_stack(handler: &Handler) {
 436         eprintln!("query stack during panic:");
 437
 438         // Be careful reyling on global state here: this code is called from
 439         // a panic hook, which means that the global `Handler` may be in a weird
 440         // state if it was responsible for triggering the panic.
 441         tls::with_context_opt(|icx| {
 442             if let Some(icx) = icx {
 443                 let query_map = icx.tcx.queries.try_collect_active_jobs();
 444
 445                 let mut current_query = icx.query;
 446                 let mut i = 0;
 447
 448                 while let Some(query) = current_query {
 449                     let query_info =
 450                         if let Some(info) = query_map.as_ref().and_then(|map| map.get(&query)) {
 451                             info
 452                         } else {
 453                             break;
 454                         };
 455                     let mut diag = Diagnostic::new(
 456                         Level::FailureNote,
 457                         &format!(
 458                             "#{} [{}] {}",
 459                             i,
 460                             query_info.info.query.name(),
 461                             query_info.info.query.describe(icx.tcx)
 462                         ),
 463                     );
 464                     diag.span = icx.tcx.sess.source_map().def_span(query_info.info.span).into();
 465                     handler.force_print_diagnostic(diag);
 466
 467                     current_query = query_info.job.parent;
 468                     i += 1;
 469                 }
 470             }
 471         });
 472
 473         eprintln!("end of query stack");
 474     }
 475
 476     /// Checks if the query is already computed and in the cache.
 477     /// It returns the shard index and a lock guard to the shard,
 478     /// which will be used if the query is not in the cache and we need
 479     /// to compute it.
 480     #[inline(always)]
 481     fn try_get_cached<C, R, OnHit, OnMiss>(
 482         self,
 483         state: &'tcx QueryState<'tcx, C>,
 484         key: C::Key,
 485         // `on_hit` can be called while holding a lock to the query cache
 486         on_hit: OnHit,
 487         on_miss: OnMiss,
 488     ) -> R
 489     where
 490         C: QueryCache,
 491         OnHit: FnOnce(&C::Value, DepNodeIndex) -> R,
 492         OnMiss: FnOnce(C::Key, QueryLookup<'tcx, C::Key, C::Sharded>) -> R,
 493     {
 494         state.cache.lookup(
 495             state,
 496             QueryStateShard::<C::Key, C::Sharded>::get_cache,
 497             key,
 498             |value, index| {
 499                 if unlikely!(self.prof.enabled()) {
 500                     self.prof.query_cache_hit(index.into());
 501                 }
 502                 #[cfg(debug_assertions)]
 503                 {
 504                     state.cache_hits.fetch_add(1, Ordering::Relaxed);
 505                 }
 506                 on_hit(value, index)
 507             },
 508             on_miss,
 509         )
 510     }
 511
 512     #[inline(never)]
 513     pub(super) fn get_query<Q: QueryDescription<'tcx> + 'tcx>(
 514         self,
 515         span: Span,
 516         key: Q::Key,
 517     ) -> Q::Value {
 518         debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
 519
 520         self.try_get_cached(
 521             Q::query_state(self),
 522             key,
 523             |value, index| {
 524                 self.dep_graph.read_index(index);
 525                 value.clone()
 526             },
 527             |key, lookup| self.try_execute_query::<Q>(span, key, lookup),
 528         )
 529     }
 530
 531     #[inline(always)]
 532     pub(super) fn try_execute_query<Q: QueryDescription<'tcx> + 'tcx>(
 533         self,
 534         span: Span,
 535         key: Q::Key,
 536         lookup: QueryLookup<'tcx, Q::Key, <Q::Cache as QueryCache>::Sharded>,
 537     ) -> Q::Value {
 538         let job = match JobOwner::try_start::<Q>(self, span, &key, lookup) {
 539             TryGetJob::NotYetStarted(job) => job,
 540             TryGetJob::Cycle(result) => return result,
 541             #[cfg(parallel_compiler)]
 542             TryGetJob::JobCompleted((v, index)) => {
 543                 self.dep_graph.read_index(index);
 544                 return v;
 545             }
 546         };
 547
 548         // Fast path for when incr. comp. is off. `to_dep_node` is
 549         // expensive for some `DepKind`s.
 550         if !self.dep_graph.is_fully_enabled() {
 551             let null_dep_node = DepNode::new_no_params(crate::dep_graph::DepKind::Null);
 552             return self.force_query_with_job::<Q>(key, job, null_dep_node).0;
 553         }
 554
 555         if Q::ANON {
 556             let prof_timer = self.prof.query_provider();
 557
 558             let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
 559                 self.start_query(job.id, diagnostics, |tcx| {
 560                     tcx.dep_graph.with_anon_task(Q::DEP_KIND, || Q::compute(tcx, key))
 561                 })
 562             });
 563
 564             prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 565
 566             self.dep_graph.read_index(dep_node_index);
 567
 568             if unlikely!(!diagnostics.is_empty()) {
 569                 self.queries
 570                     .on_disk_cache
 571                     .store_diagnostics_for_anon_node(dep_node_index, diagnostics);
 572             }
 573
 574             job.complete(self, &result, dep_node_index);
 575
 576             return result;
 577         }
 578
 579         let dep_node = Q::to_dep_node(self, &key);
 580
 581         if !Q::EVAL_ALWAYS {
 582             // The diagnostics for this query will be
 583             // promoted to the current session during
 584             // `try_mark_green()`, so we can ignore them here.
 585             let loaded = self.start_query(job.id, None, |tcx| {
 586                 let marked = tcx.dep_graph.try_mark_green_and_read(tcx, &dep_node);
 587                 marked.map(|(prev_dep_node_index, dep_node_index)| {
 588                     (
 589                         tcx.load_from_disk_and_cache_in_memory::<Q>(
 590                             key.clone(),
 591                             prev_dep_node_index,
 592                             dep_node_index,
 593                             &dep_node,
 594                         ),
 595                         dep_node_index,
 596                     )
 597                 })
 598             });
 599             if let Some((result, dep_node_index)) = loaded {
 600                 job.complete(self, &result, dep_node_index);
 601                 return result;
 602             }
 603         }
 604
 605         let (result, dep_node_index) = self.force_query_with_job::<Q>(key, job, dep_node);
 606         self.dep_graph.read_index(dep_node_index);
 607         result
 608     }
 609
 610     fn load_from_disk_and_cache_in_memory<Q: QueryDescription<'tcx>>(
 611         self,
 612         key: Q::Key,
 613         prev_dep_node_index: SerializedDepNodeIndex,
 614         dep_node_index: DepNodeIndex,
 615         dep_node: &DepNode,
 616     ) -> Q::Value {
 617         // Note this function can be called concurrently from the same query
 618         // We must ensure that this is handled correctly.
 619
 620         debug_assert!(self.dep_graph.is_green(dep_node));
 621
 622         // First we try to load the result from the on-disk cache.
 623         let result = if Q::cache_on_disk(self, key.clone(), None)
 624             && self.sess.opts.debugging_opts.incremental_queries
 625         {
 626             let prof_timer = self.prof.incr_cache_loading();
 627             let result = Q::try_load_from_disk(self, prev_dep_node_index);
 628             prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 629
 630             // We always expect to find a cached result for things that
 631             // can be forced from `DepNode`.
 632             debug_assert!(
 633                 !dep_node.kind.can_reconstruct_query_key() || result.is_some(),
 634                 "missing on-disk cache entry for {:?}",
 635                 dep_node
 636             );
 637             result
 638         } else {
 639             // Some things are never cached on disk.
 640             None
 641         };
 642
 643         let result = if let Some(result) = result {
 644             result
 645         } else {
 646             // We could not load a result from the on-disk cache, so
 647             // recompute.
 648             let prof_timer = self.prof.query_provider();
 649
 650             // The dep-graph for this computation is already in-place.
 651             let result = self.dep_graph.with_ignore(|| Q::compute(self, key));
 652
 653             prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 654
 655             result
 656         };
 657
 658         // If `-Zincremental-verify-ich` is specified, re-hash results from
 659         // the cache and make sure that they have the expected fingerprint.
 660         if unlikely!(self.sess.opts.debugging_opts.incremental_verify_ich) {
 661             self.incremental_verify_ich::<Q>(&result, dep_node, dep_node_index);
 662         }
 663
 664         result
 665     }
 666
 667     #[inline(never)]
 668     #[cold]
 669     fn incremental_verify_ich<Q: QueryDescription<'tcx>>(
 670         self,
 671         result: &Q::Value,
 672         dep_node: &DepNode,
 673         dep_node_index: DepNodeIndex,
 674     ) {
 675         use crate::ich::Fingerprint;
 676
 677         assert!(
 678             Some(self.dep_graph.fingerprint_of(dep_node_index))
 679                 == self.dep_graph.prev_fingerprint_of(dep_node),
 680             "fingerprint for green query instance not loaded from cache: {:?}",
 681             dep_node,
 682         );
 683
 684         debug!("BEGIN verify_ich({:?})", dep_node);
 685         let mut hcx = self.create_stable_hashing_context();
 686
 687         let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
 688         debug!("END verify_ich({:?})", dep_node);
 689
 690         let old_hash = self.dep_graph.fingerprint_of(dep_node_index);
 691
 692         assert!(new_hash == old_hash, "found unstable fingerprints for {:?}", dep_node,);
 693     }
 694
 695     #[inline(always)]
 696     fn force_query_with_job<Q: QueryDescription<'tcx> + 'tcx>(
 697         self,
 698         key: Q::Key,
 699         job: JobOwner<'tcx, Q::Cache>,
 700         dep_node: DepNode,
 701     ) -> (Q::Value, DepNodeIndex) {
 702         // If the following assertion triggers, it can have two reasons:
 703         // 1. Something is wrong with DepNode creation, either here or
 704         //    in `DepGraph::try_mark_green()`.
 705         // 2. Two distinct query keys get mapped to the same `DepNode`
 706         //    (see for example #48923).
 707         assert!(
 708             !self.dep_graph.dep_node_exists(&dep_node),
 709             "forcing query with already existing `DepNode`\n\
 710                  - query-key: {:?}\n\
 711                  - dep-node: {:?}",
 712             key,
 713             dep_node
 714         );
 715
 716         let prof_timer = self.prof.query_provider();
 717
 718         let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
 719             self.start_query(job.id, diagnostics, |tcx| {
 720                 if Q::EVAL_ALWAYS {
 721                     tcx.dep_graph.with_eval_always_task(
 722                         dep_node,
 723                         tcx,
 724                         key,
 725                         Q::compute,
 726                         Q::hash_result,
 727                     )
 728                 } else {
 729                     tcx.dep_graph.with_task(dep_node, tcx, key, Q::compute, Q::hash_result)
 730                 }
 731             })
 732         });
 733
 734         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 735
 736         if unlikely!(!diagnostics.is_empty()) {
 737             if dep_node.kind != crate::dep_graph::DepKind::Null {
 738                 self.queries.on_disk_cache.store_diagnostics(dep_node_index, diagnostics);
 739             }
 740         }
 741
 742         job.complete(self, &result, dep_node_index);
 743
 744         (result, dep_node_index)
 745     }
 746
 747     /// Ensure that either this query has all green inputs or been executed.
 748     /// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
 749     ///
 750     /// This function is particularly useful when executing passes for their
 751     /// side-effects -- e.g., in order to report errors for erroneous programs.
 752     ///
 753     /// Note: The optimization is only available during incr. comp.
 754     pub(super) fn ensure_query<Q: QueryDescription<'tcx> + 'tcx>(self, key: Q::Key) -> () {
 755         if Q::EVAL_ALWAYS {
 756             let _ = self.get_query::<Q>(DUMMY_SP, key);
 757             return;
 758         }
 759
 760         // Ensuring an anonymous query makes no sense
 761         assert!(!Q::ANON);
 762
 763         let dep_node = Q::to_dep_node(self, &key);
 764
 765         match self.dep_graph.try_mark_green_and_read(self, &dep_node) {
 766             None => {
 767                 // A None return from `try_mark_green_and_read` means that this is either
 768                 // a new dep node or that the dep node has already been marked red.
 769                 // Either way, we can't call `dep_graph.read()` as we don't have the
 770                 // DepNodeIndex. We must invoke the query itself. The performance cost
 771                 // this introduces should be negligible as we'll immediately hit the
 772                 // in-memory cache, or another query down the line will.
 773                 let _ = self.get_query::<Q>(DUMMY_SP, key);
 774             }
 775             Some((_, dep_node_index)) => {
 776                 self.prof.query_cache_hit(dep_node_index.into());
 777             }
 778         }
 779     }
 780
 781     #[allow(dead_code)]
 782     pub(super) fn force_query<Q: QueryDescription<'tcx> + 'tcx>(
 783         self,
 784         key: Q::Key,
 785         span: Span,
 786         dep_node: DepNode,
 787     ) {
 788         // We may be concurrently trying both execute and force a query.
 789         // Ensure that only one of them runs the query.
 790
 791         self.try_get_cached(
 792             Q::query_state(self),
 793             key,
 794             |_, _| {
 795                 // Cache hit, do nothing
 796             },
 797             |key, lookup| {
 798                 let job = match JobOwner::try_start::<Q>(self, span, &key, lookup) {
 799                     TryGetJob::NotYetStarted(job) => job,
 800                     TryGetJob::Cycle(_) => return,
 801                     #[cfg(parallel_compiler)]
 802                     TryGetJob::JobCompleted(_) => return,
 803                 };
 804                 self.force_query_with_job::<Q>(key, job, dep_node);
 805             },
 806         );
 807     }
 808 }
 809
 810 macro_rules! handle_cycle_error {
 811     ([][$tcx: expr, $error:expr]) => {{
 812         $tcx.report_cycle($error).emit();
 813         Value::from_cycle_error($tcx)
 814     }};
 815     ([fatal_cycle $($rest:tt)*][$tcx:expr, $error:expr]) => {{
 816         $tcx.report_cycle($error).emit();
 817         $tcx.sess.abort_if_errors();
 818         unreachable!()
 819     }};
 820     ([cycle_delay_bug $($rest:tt)*][$tcx:expr, $error:expr]) => {{
 821         $tcx.report_cycle($error).delay_as_bug();
 822         Value::from_cycle_error($tcx)
 823     }};
 824     ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => {
 825         handle_cycle_error!([$($($modifiers)*)*][$($args)*])
 826     };
 827 }
 828
 829 macro_rules! is_anon {
 830     ([]) => {{
 831         false
 832     }};
 833     ([anon $($rest:tt)*]) => {{
 834         true
 835     }};
 836     ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*]) => {
 837         is_anon!([$($($modifiers)*)*])
 838     };
 839 }
 840
 841 macro_rules! is_eval_always {
 842     ([]) => {{
 843         false
 844     }};
 845     ([eval_always $($rest:tt)*]) => {{
 846         true
 847     }};
 848     ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*]) => {
 849         is_eval_always!([$($($modifiers)*)*])
 850     };
 851 }
 852
 853 macro_rules! query_storage {
 854     ([][$K:ty, $V:ty]) => {
 855         <<$K as Key>::CacheSelector as CacheSelector<$K, $V>>::Cache
 856     };
 857     ([storage($ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => {
 858         $ty
 859     };
 860     ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => {
 861         query_storage!([$($($modifiers)*)*][$($args)*])
 862     };
 863 }
 864
 865 macro_rules! hash_result {
 866     ([][$hcx:expr, $result:expr]) => {{
 867         dep_graph::hash_result($hcx, &$result)
 868     }};
 869     ([no_hash $($rest:tt)*][$hcx:expr, $result:expr]) => {{
 870         None
 871     }};
 872     ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => {
 873         hash_result!([$($($modifiers)*)*][$($args)*])
 874     };
 875 }
 876
 877 macro_rules! define_queries {
 878     (<$tcx:tt> $($category:tt {
 879         $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*
 880     },)*) => {
 881         define_queries_inner! { <$tcx>
 882             $($( $(#[$attr])* category<$category> [$($modifiers)*] fn $name: $node($K) -> $V,)*)*
 883         }
 884     }
 885 }
 886
 887 macro_rules! define_queries_inner {
 888     (<$tcx:tt>
 889      $($(#[$attr:meta])* category<$category:tt>
 890         [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*) => {
 891
 892         use std::mem;
 893         use crate::{
 894             rustc_data_structures::stable_hasher::HashStable,
 895             rustc_data_structures::stable_hasher::StableHasher,
 896             ich::StableHashingContext
 897         };
 898         use rustc_data_structures::profiling::ProfileCategory;
 899
 900         define_queries_struct! {
 901             tcx: $tcx,
 902             input: ($(([$($modifiers)*] [$($attr)*] [$name]))*)
 903         }
 904
 905         #[allow(nonstandard_style)]
 906         #[derive(Clone, Debug)]
 907         pub enum Query<$tcx> {
 908             $($(#[$attr])* $name($K)),*
 909         }
 910
 911         impl<$tcx> Query<$tcx> {
 912             pub fn name(&self) -> &'static str {
 913                 match *self {
 914                     $(Query::$name(_) => stringify!($name),)*
 915                 }
 916             }
 917
 918             pub fn describe(&self, tcx: TyCtxt<'_>) -> Cow<'static, str> {
 919                 let (r, name) = match *self {
 920                     $(Query::$name(key) => {
 921                         (queries::$name::describe(tcx, key), stringify!($name))
 922                     })*
 923                 };
 924                 if tcx.sess.verbose() {
 925                     format!("{} [{}]", r, name).into()
 926                 } else {
 927                     r
 928                 }
 929             }
 930
 931             // FIXME(eddyb) Get more valid `Span`s on queries.
 932             pub fn default_span(&self, tcx: TyCtxt<$tcx>, span: Span) -> Span {
 933                 if !span.is_dummy() {
 934                     return span;
 935                 }
 936                 // The `def_span` query is used to calculate `default_span`,
 937                 // so exit to avoid infinite recursion.
 938                 if let Query::def_span(..) = *self {
 939                     return span
 940                 }
 941                 match *self {
 942                     $(Query::$name(key) => key.default_span(tcx),)*
 943                 }
 944             }
 945         }
 946
 947         impl<'a, $tcx> HashStable<StableHashingContext<'a>> for Query<$tcx> {
 948             fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
 949                 mem::discriminant(self).hash_stable(hcx, hasher);
 950                 match *self {
 951                     $(Query::$name(key) => key.hash_stable(hcx, hasher),)*
 952                 }
 953             }
 954         }
 955
 956         pub mod queries {
 957             use std::marker::PhantomData;
 958
 959             $(#[allow(nonstandard_style)]
 960             pub struct $name<$tcx> {
 961                 data: PhantomData<&$tcx ()>
 962             })*
 963         }
 964
 965         $(impl<$tcx> QueryConfig<$tcx> for queries::$name<$tcx> {
 966             type Key = $K;
 967             type Value = $V;
 968             const NAME: &'static str = stringify!($name);
 969             const CATEGORY: ProfileCategory = $category;
 970         }
 971
 972         impl<$tcx> QueryAccessors<$tcx> for queries::$name<$tcx> {
 973             const ANON: bool = is_anon!([$($modifiers)*]);
 974             const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]);
 975             const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::$node;
 976
 977             type Cache = query_storage!([$($modifiers)*][$K, $V]);
 978
 979             #[inline(always)]
 980             fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<$tcx, Self::Cache> {
 981                 &tcx.queries.$name
 982             }
 983
 984             #[allow(unused)]
 985             #[inline(always)]
 986             fn to_dep_node(tcx: TyCtxt<$tcx>, key: &Self::Key) -> DepNode {
 987                 DepConstructor::$node(tcx, *key)
 988             }
 989
 990             #[inline]
 991             fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
 992                 let provider = tcx.queries.providers.get(key.query_crate())
 993                     // HACK(eddyb) it's possible crates may be loaded after
 994                     // the query engine is created, and because crate loading
 995                     // is not yet integrated with the query engine, such crates
 996                     // would be missing appropriate entries in `providers`.
 997                     .unwrap_or(&tcx.queries.fallback_extern_providers)
 998                     .$name;
 999                 provider(tcx, key)
1000             }
1001
1002             fn hash_result(
1003                 _hcx: &mut StableHashingContext<'_>,
1004                 _result: &Self::Value
1005             ) -> Option<Fingerprint> {
1006                 hash_result!([$($modifiers)*][_hcx, _result])
1007             }
1008
1009             fn handle_cycle_error(
1010                 tcx: TyCtxt<'tcx>,
1011                 error: CycleError<'tcx>
1012             ) -> Self::Value {
1013                 handle_cycle_error!([$($modifiers)*][tcx, error])
1014             }
1015         })*
1016
1017         #[derive(Copy, Clone)]
1018         pub struct TyCtxtEnsure<'tcx> {
1019             pub tcx: TyCtxt<'tcx>,
1020         }
1021
1022         impl TyCtxtEnsure<$tcx> {
1023             $($(#[$attr])*
1024             #[inline(always)]
1025             pub fn $name(self, key: $K) {
1026                 self.tcx.ensure_query::<queries::$name<'_>>(key)
1027             })*
1028         }
1029
1030         #[derive(Copy, Clone)]
1031         pub struct TyCtxtAt<'tcx> {
1032             pub tcx: TyCtxt<'tcx>,
1033             pub span: Span,
1034         }
1035
1036         impl Deref for TyCtxtAt<'tcx> {
1037             type Target = TyCtxt<'tcx>;
1038             #[inline(always)]
1039             fn deref(&self) -> &Self::Target {
1040                 &self.tcx
1041             }
1042         }
1043
1044         impl TyCtxt<$tcx> {
1045             /// Returns a transparent wrapper for `TyCtxt`, which ensures queries
1046             /// are executed instead of just returning their results.
1047             #[inline(always)]
1048             pub fn ensure(self) -> TyCtxtEnsure<$tcx> {
1049                 TyCtxtEnsure {
1050                     tcx: self,
1051                 }
1052             }
1053
1054             /// Returns a transparent wrapper for `TyCtxt` which uses
1055             /// `span` as the location of queries performed through it.
1056             #[inline(always)]
1057             pub fn at(self, span: Span) -> TyCtxtAt<$tcx> {
1058                 TyCtxtAt {
1059                     tcx: self,
1060                     span
1061                 }
1062             }
1063
1064             $($(#[$attr])*
1065             #[inline(always)]
1066             pub fn $name(self, key: $K) -> $V {
1067                 self.at(DUMMY_SP).$name(key)
1068             })*
1069
1070             /// All self-profiling events generated by the query engine use
1071             /// virtual `StringId`s for their `event_id`. This method makes all
1072             /// those virtual `StringId`s point to actual strings.
1073             ///
1074             /// If we are recording only summary data, the ids will point to
1075             /// just the query names. If we are recording query keys too, we
1076             /// allocate the corresponding strings here.
1077             pub fn alloc_self_profile_query_strings(self) {
1078                 use crate::ty::query::profiling_support::{
1079                     alloc_self_profile_query_strings_for_query_cache,
1080                     QueryKeyStringCache,
1081                 };
1082
1083                 if !self.prof.enabled() {
1084                     return;
1085                 }
1086
1087                 let mut string_cache = QueryKeyStringCache::new();
1088
1089                 $({
1090                     alloc_self_profile_query_strings_for_query_cache(
1091                         self,
1092                         stringify!($name),
1093                         &self.queries.$name,
1094                         &mut string_cache,
1095                     );
1096                 })*
1097             }
1098         }
1099
1100         impl TyCtxtAt<$tcx> {
1101             $($(#[$attr])*
1102             #[inline(always)]
1103             pub fn $name(self, key: $K) -> $V {
1104                 self.tcx.get_query::<queries::$name<'_>>(self.span, key)
1105             })*
1106         }
1107
1108         define_provider_struct! {
1109             tcx: $tcx,
1110             input: ($(([$($modifiers)*] [$name] [$K] [$V]))*)
1111         }
1112
1113         impl<$tcx> Copy for Providers<$tcx> {}
1114         impl<$tcx> Clone for Providers<$tcx> {
1115             fn clone(&self) -> Self { *self }
1116         }
1117     }
1118 }
1119
1120 macro_rules! define_queries_struct {
1121     (tcx: $tcx:tt,
1122      input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
1123         pub struct Queries<$tcx> {
1124             /// This provides access to the incrimental comilation on-disk cache for query results.
1125             /// Do not access this directly. It is only meant to be used by
1126             /// `DepGraph::try_mark_green()` and the query infrastructure.
1127             pub(crate) on_disk_cache: OnDiskCache<'tcx>,
1128
1129             providers: IndexVec<CrateNum, Providers<$tcx>>,
1130             fallback_extern_providers: Box<Providers<$tcx>>,
1131
1132             $($(#[$attr])*  $name: QueryState<
1133                 $tcx,
1134                 <queries::$name<$tcx> as QueryAccessors<'tcx>>::Cache,
1135             >,)*
1136         }
1137
1138         impl<$tcx> Queries<$tcx> {
1139             pub fn new(
1140                 providers: IndexVec<CrateNum, Providers<$tcx>>,
1141                 fallback_extern_providers: Providers<$tcx>,
1142                 on_disk_cache: OnDiskCache<'tcx>,
1143             ) -> Self {
1144                 Queries {
1145                     providers,
1146                     fallback_extern_providers: Box::new(fallback_extern_providers),
1147                     on_disk_cache,
1148                     $($name: Default::default()),*
1149                 }
1150             }
1151
1152             pub fn try_collect_active_jobs(
1153                 &self
1154             ) -> Option<FxHashMap<QueryJobId, QueryJobInfo<'tcx>>> {
1155                 let mut jobs = FxHashMap::default();
1156
1157                 $(
1158                     self.$name.try_collect_active_jobs(
1159                         <queries::$name<'tcx> as QueryAccessors<'tcx>>::DEP_KIND,
1160                         Query::$name,
1161                         &mut jobs,
1162                     )?;
1163                 )*
1164
1165                 Some(jobs)
1166             }
1167         }
1168     };
1169 }
1170
1171 macro_rules! define_provider_struct {
1172     (tcx: $tcx:tt,
1173      input: ($(([$($modifiers:tt)*] [$name:ident] [$K:ty] [$R:ty]))*)) => {
1174         pub struct Providers<$tcx> {
1175             $(pub $name: fn(TyCtxt<$tcx>, $K) -> $R,)*
1176         }
1177
1178         impl<$tcx> Default for Providers<$tcx> {
1179             fn default() -> Self {
1180                 $(fn $name<$tcx>(_: TyCtxt<$tcx>, key: $K) -> $R {
1181                     bug!("`tcx.{}({:?})` unsupported by its crate",
1182                          stringify!($name), key);
1183                 })*
1184                 Providers { $($name),* }
1185             }
1186         }
1187     };
1188 }