+++ /dev/null
-For more information about how the query system works, see the [rustc dev guide].
-
-[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/query.html
+++ /dev/null
-use crate::dep_graph::DepNodeIndex;
-use crate::ty::query::config::QueryContext;
-use crate::ty::query::plumbing::{QueryLookup, QueryState, QueryStateShard};
-
-use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::sharded::Sharded;
-use std::default::Default;
-use std::hash::Hash;
-use std::marker::PhantomData;
-
-pub(crate) trait CacheSelector<CTX: QueryContext, K, V> {
- type Cache: QueryCache<CTX, Key = K, Value = V>;
-}
-
-pub(crate) trait QueryCache<CTX: QueryContext>: Default {
- type Key;
- type Value;
- type Sharded: Default;
-
- /// Checks if the query is already computed and in the cache.
- /// It returns the shard index and a lock guard to the shard,
- /// which will be used if the query is not in the cache and we need
- /// to compute it.
- fn lookup<R, GetCache, OnHit, OnMiss>(
- &self,
- state: &QueryState<CTX, Self>,
- get_cache: GetCache,
- key: Self::Key,
- // `on_hit` can be called while holding a lock to the query state shard.
- on_hit: OnHit,
- on_miss: OnMiss,
- ) -> R
- where
- GetCache: for<'a> Fn(
- &'a mut QueryStateShard<CTX, Self::Key, Self::Sharded>,
- ) -> &'a mut Self::Sharded,
- OnHit: FnOnce(&Self::Value, DepNodeIndex) -> R,
- OnMiss: FnOnce(Self::Key, QueryLookup<'_, CTX, Self::Key, Self::Sharded>) -> R;
-
- fn complete(
- &self,
- tcx: CTX,
- lock_sharded_storage: &mut Self::Sharded,
- key: Self::Key,
- value: Self::Value,
- index: DepNodeIndex,
- );
-
- fn iter<R, L>(
- &self,
- shards: &Sharded<L>,
- get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
- f: impl for<'a> FnOnce(
- Box<dyn Iterator<Item = (&'a Self::Key, &'a Self::Value, DepNodeIndex)> + 'a>,
- ) -> R,
- ) -> R;
-}
-
-pub struct DefaultCacheSelector;
-
-impl<CTX: QueryContext, K: Eq + Hash, V: Clone> CacheSelector<CTX, K, V> for DefaultCacheSelector {
- type Cache = DefaultCache<K, V>;
-}
-
-pub struct DefaultCache<K, V>(PhantomData<(K, V)>);
-
-impl<K, V> Default for DefaultCache<K, V> {
- fn default() -> Self {
- DefaultCache(PhantomData)
- }
-}
-
-impl<CTX: QueryContext, K: Eq + Hash, V: Clone> QueryCache<CTX> for DefaultCache<K, V> {
- type Key = K;
- type Value = V;
- type Sharded = FxHashMap<K, (V, DepNodeIndex)>;
-
- #[inline(always)]
- fn lookup<R, GetCache, OnHit, OnMiss>(
- &self,
- state: &QueryState<CTX, Self>,
- get_cache: GetCache,
- key: K,
- on_hit: OnHit,
- on_miss: OnMiss,
- ) -> R
- where
- GetCache:
- for<'a> Fn(&'a mut QueryStateShard<CTX, K, Self::Sharded>) -> &'a mut Self::Sharded,
- OnHit: FnOnce(&V, DepNodeIndex) -> R,
- OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R,
- {
- let mut lookup = state.get_lookup(&key);
- let lock = &mut *lookup.lock;
-
- let result = get_cache(lock).raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
-
- if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) }
- }
-
- #[inline]
- fn complete(
- &self,
- _: CTX,
- lock_sharded_storage: &mut Self::Sharded,
- key: K,
- value: V,
- index: DepNodeIndex,
- ) {
- lock_sharded_storage.insert(key, (value, index));
- }
-
- fn iter<R, L>(
- &self,
- shards: &Sharded<L>,
- get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
- f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
- ) -> R {
- let mut shards = shards.lock_shards();
- let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
- let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
- f(Box::new(results))
- }
-}
+++ /dev/null
-//! Query configuration and description traits.
-
-use crate::dep_graph::SerializedDepNodeIndex;
-use crate::ty::query::caches::QueryCache;
-use crate::ty::query::job::{QueryJobId, QueryJobInfo};
-use crate::ty::query::plumbing::CycleError;
-use crate::ty::query::QueryState;
-use rustc_data_structures::profiling::ProfileCategory;
-use rustc_hir::def_id::DefId;
-
-use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::stable_hasher::HashStable;
-use rustc_data_structures::sync::Lock;
-use rustc_data_structures::thin_vec::ThinVec;
-use rustc_errors::Diagnostic;
-use rustc_query_system::dep_graph::{DepContext, DepGraph, DepNode};
-use rustc_session::Session;
-use std::borrow::Cow;
-use std::fmt::Debug;
-use std::hash::Hash;
-
-pub trait QueryConfig<CTX> {
- const NAME: &'static str;
- const CATEGORY: ProfileCategory;
-
- type Key: Eq + Hash + Clone + Debug;
- type Value: Clone;
-}
-
-pub trait QueryContext: DepContext {
- type Query: Clone + HashStable<Self::StableHashingContext>;
-
- /// Access the session.
- fn session(&self) -> &Session;
-
- /// Get string representation from DefPath.
- fn def_path_str(&self, def_id: DefId) -> String;
-
- /// Access the DepGraph.
- fn dep_graph(&self) -> &DepGraph<Self::DepKind>;
-
- /// Get the query information from the TLS context.
- fn read_query_job<R>(&self, op: impl FnOnce(Option<QueryJobId<Self::DepKind>>) -> R) -> R;
-
- fn try_collect_active_jobs(
- &self,
- ) -> Option<FxHashMap<QueryJobId<Self::DepKind>, QueryJobInfo<Self>>>;
-
- /// Executes a job by changing the `ImplicitCtxt` to point to the
- /// new query job while it executes. It returns the diagnostics
- /// captured during execution and the actual result.
- fn start_query<R>(
- &self,
- token: QueryJobId<Self::DepKind>,
- diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
- compute: impl FnOnce(Self) -> R,
- ) -> R;
-}
-
-pub(crate) trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> {
- const ANON: bool;
- const EVAL_ALWAYS: bool;
- const DEP_KIND: CTX::DepKind;
-
- type Cache: QueryCache<CTX, Key = Self::Key, Value = Self::Value>;
-
- // Don't use this method to access query results, instead use the methods on TyCtxt
- fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX, Self::Cache>;
-
- fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind>;
-
- // Don't use this method to compute query results, instead use the methods on TyCtxt
- fn compute(tcx: CTX, key: Self::Key) -> Self::Value;
-
- fn hash_result(
- hcx: &mut CTX::StableHashingContext,
- result: &Self::Value,
- ) -> Option<Fingerprint>;
-
- fn handle_cycle_error(tcx: CTX, error: CycleError<CTX::Query>) -> Self::Value;
-}
-
-pub(crate) trait QueryDescription<CTX: QueryContext>: QueryAccessors<CTX> {
- fn describe(tcx: CTX, key: Self::Key) -> Cow<'static, str>;
-
- #[inline]
- fn cache_on_disk(_: CTX, _: Self::Key, _: Option<&Self::Value>) -> bool {
- false
- }
-
- fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option<Self::Value> {
- bug!("QueryDescription::load_from_disk() called for an unsupported query.")
- }
-}
-
-impl<CTX: QueryContext, M> QueryDescription<CTX> for M
-where
- M: QueryAccessors<CTX, Key = DefId>,
-{
- default fn describe(tcx: CTX, def_id: DefId) -> Cow<'static, str> {
- if !tcx.session().verbose() {
- format!("processing `{}`", tcx.def_path_str(def_id)).into()
- } else {
- let name = ::std::any::type_name::<M>();
- format!("processing {:?} with query `{}`", def_id, name).into()
- }
- }
-
- default fn cache_on_disk(_: CTX, _: Self::Key, _: Option<&Self::Value>) -> bool {
- false
- }
-
- default fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option<Self::Value> {
- bug!("QueryDescription::load_from_disk() called for an unsupported query.")
- }
-}
+++ /dev/null
-use crate::ty::query::config::QueryContext;
-use crate::ty::query::plumbing::CycleError;
-#[cfg(parallel_compiler)]
-use crate::ty::tls;
-
-use rustc_data_structures::fx::FxHashMap;
-use rustc_query_system::dep_graph::DepContext;
-use rustc_span::Span;
-
-use std::convert::TryFrom;
-use std::marker::PhantomData;
-use std::num::NonZeroU32;
-
-#[cfg(parallel_compiler)]
-use {
- parking_lot::{Condvar, Mutex},
- rustc_data_structures::fx::FxHashSet,
- rustc_data_structures::stable_hasher::{HashStable, StableHasher},
- rustc_data_structures::sync::Lock,
- rustc_data_structures::sync::Lrc,
- rustc_data_structures::{jobserver, OnDrop},
- rustc_rayon_core as rayon_core,
- rustc_span::DUMMY_SP,
- std::iter::FromIterator,
- std::{mem, process, thread},
-};
-
-/// Represents a span and a query key.
-#[derive(Clone, Debug)]
-pub struct QueryInfo<Q> {
- /// The span corresponding to the reason for which this query was required.
- pub span: Span,
- pub query: Q,
-}
-
-type QueryMap<CTX> = FxHashMap<QueryJobId<<CTX as DepContext>::DepKind>, QueryJobInfo<CTX>>;
-
-/// A value uniquely identifiying an active query job within a shard in the query cache.
-#[derive(Copy, Clone, Eq, PartialEq, Hash)]
-pub struct QueryShardJobId(pub NonZeroU32);
-
-/// A value uniquely identifiying an active query job.
-#[derive(Copy, Clone, Eq, PartialEq, Hash)]
-pub struct QueryJobId<K> {
- /// Which job within a shard is this
- pub job: QueryShardJobId,
-
- /// In which shard is this job
- pub shard: u16,
-
- /// What kind of query this job is
- pub kind: K,
-}
-
-impl<K: rustc_query_system::dep_graph::DepKind> QueryJobId<K> {
- pub fn new(job: QueryShardJobId, shard: usize, kind: K) -> Self {
- QueryJobId { job, shard: u16::try_from(shard).unwrap(), kind }
- }
-
- fn query<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> CTX::Query {
- map.get(&self).unwrap().info.query.clone()
- }
-
- #[cfg(parallel_compiler)]
- fn span<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> Span {
- map.get(&self).unwrap().job.span
- }
-
- #[cfg(parallel_compiler)]
- fn parent<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> Option<QueryJobId<K>> {
- map.get(&self).unwrap().job.parent
- }
-
- #[cfg(parallel_compiler)]
- fn latch<'a, CTX: QueryContext<DepKind = K>>(
- self,
- map: &'a QueryMap<CTX>,
- ) -> Option<&'a QueryLatch<CTX>> {
- map.get(&self).unwrap().job.latch.as_ref()
- }
-}
-
-pub struct QueryJobInfo<CTX: QueryContext> {
- pub info: QueryInfo<CTX::Query>,
- pub job: QueryJob<CTX>,
-}
-
-/// Represents an active query job.
-#[derive(Clone)]
-pub struct QueryJob<CTX: QueryContext> {
- pub id: QueryShardJobId,
-
- /// The span corresponding to the reason for which this query was required.
- pub span: Span,
-
- /// The parent query job which created this job and is implicitly waiting on it.
- pub parent: Option<QueryJobId<CTX::DepKind>>,
-
- /// The latch that is used to wait on this job.
- #[cfg(parallel_compiler)]
- latch: Option<QueryLatch<CTX>>,
-
- dummy: PhantomData<QueryLatch<CTX>>,
-}
-
-impl<CTX: QueryContext> QueryJob<CTX> {
- /// Creates a new query job.
- pub fn new(id: QueryShardJobId, span: Span, parent: Option<QueryJobId<CTX::DepKind>>) -> Self {
- QueryJob {
- id,
- span,
- parent,
- #[cfg(parallel_compiler)]
- latch: None,
- dummy: PhantomData,
- }
- }
-
- #[cfg(parallel_compiler)]
- pub(super) fn latch(&mut self, _id: QueryJobId<CTX::DepKind>) -> QueryLatch<CTX> {
- if self.latch.is_none() {
- self.latch = Some(QueryLatch::new());
- }
- self.latch.as_ref().unwrap().clone()
- }
-
- #[cfg(not(parallel_compiler))]
- pub(super) fn latch(&mut self, id: QueryJobId<CTX::DepKind>) -> QueryLatch<CTX> {
- QueryLatch { id, dummy: PhantomData }
- }
-
- /// Signals to waiters that the query is complete.
- ///
- /// This does nothing for single threaded rustc,
- /// as there are no concurrent jobs which could be waiting on us
- pub fn signal_complete(self) {
- #[cfg(parallel_compiler)]
- self.latch.map(|latch| latch.set());
- }
-}
-
-#[cfg(not(parallel_compiler))]
-#[derive(Clone)]
-pub(super) struct QueryLatch<CTX: QueryContext> {
- id: QueryJobId<CTX::DepKind>,
- dummy: PhantomData<CTX>,
-}
-
-#[cfg(not(parallel_compiler))]
-impl<CTX: QueryContext> QueryLatch<CTX> {
- pub(super) fn find_cycle_in_stack(&self, tcx: CTX, span: Span) -> CycleError<CTX::Query> {
- let query_map = tcx.try_collect_active_jobs().unwrap();
-
- // Get the current executing query (waiter) and find the waitee amongst its parents
- let mut current_job = tcx.read_query_job(|query| query);
- let mut cycle = Vec::new();
-
- while let Some(job) = current_job {
- let info = query_map.get(&job).unwrap();
- cycle.push(info.info.clone());
-
- if job == self.id {
- cycle.reverse();
-
- // This is the end of the cycle
- // The span entry we included was for the usage
- // of the cycle itself, and not part of the cycle
- // Replace it with the span which caused the cycle to form
- cycle[0].span = span;
- // Find out why the cycle itself was used
- let usage = info
- .job
- .parent
- .as_ref()
- .map(|parent| (info.info.span, parent.query(&query_map)));
- return CycleError { usage, cycle };
- }
-
- current_job = info.job.parent;
- }
-
- panic!("did not find a cycle")
- }
-}
-
-#[cfg(parallel_compiler)]
-struct QueryWaiter<CTX: QueryContext> {
- query: Option<QueryJobId<CTX::DepKind>>,
- condvar: Condvar,
- span: Span,
- cycle: Lock<Option<CycleError<CTX::Query>>>,
-}
-
-#[cfg(parallel_compiler)]
-impl<CTX: QueryContext> QueryWaiter<CTX> {
- fn notify(&self, registry: &rayon_core::Registry) {
- rayon_core::mark_unblocked(registry);
- self.condvar.notify_one();
- }
-}
-
-#[cfg(parallel_compiler)]
-struct QueryLatchInfo<CTX: QueryContext> {
- complete: bool,
- waiters: Vec<Lrc<QueryWaiter<CTX>>>,
-}
-
-#[cfg(parallel_compiler)]
-#[derive(Clone)]
-pub(super) struct QueryLatch<CTX: QueryContext> {
- info: Lrc<Mutex<QueryLatchInfo<CTX>>>,
-}
-
-#[cfg(parallel_compiler)]
-impl<CTX: QueryContext> QueryLatch<CTX> {
- fn new() -> Self {
- QueryLatch {
- info: Lrc::new(Mutex::new(QueryLatchInfo { complete: false, waiters: Vec::new() })),
- }
- }
-}
-
-#[cfg(parallel_compiler)]
-impl<CTX: QueryContext> QueryLatch<CTX> {
- /// Awaits for the query job to complete.
- pub(super) fn wait_on(&self, tcx: CTX, span: Span) -> Result<(), CycleError<CTX::Query>> {
- tcx.read_query_job(move |query| {
- let waiter = Lrc::new(QueryWaiter {
- query,
- span,
- cycle: Lock::new(None),
- condvar: Condvar::new(),
- });
- self.wait_on_inner(&waiter);
- // FIXME: Get rid of this lock. We have ownership of the QueryWaiter
- // although another thread may still have a Lrc reference so we cannot
- // use Lrc::get_mut
- let mut cycle = waiter.cycle.lock();
- match cycle.take() {
- None => Ok(()),
- Some(cycle) => Err(cycle),
- }
- })
- }
-}
-
-#[cfg(parallel_compiler)]
-impl<CTX: QueryContext> QueryLatch<CTX> {
- /// Awaits the caller on this latch by blocking the current thread.
- fn wait_on_inner(&self, waiter: &Lrc<QueryWaiter<CTX>>) {
- let mut info = self.info.lock();
- if !info.complete {
- // We push the waiter on to the `waiters` list. It can be accessed inside
- // the `wait` call below, by 1) the `set` method or 2) by deadlock detection.
- // Both of these will remove it from the `waiters` list before resuming
- // this thread.
- info.waiters.push(waiter.clone());
-
- // If this detects a deadlock and the deadlock handler wants to resume this thread
- // we have to be in the `wait` call. This is ensured by the deadlock handler
- // getting the self.info lock.
- rayon_core::mark_blocked();
- jobserver::release_thread();
- waiter.condvar.wait(&mut info);
- // Release the lock before we potentially block in `acquire_thread`
- mem::drop(info);
- jobserver::acquire_thread();
- }
- }
-
- /// Sets the latch and resumes all waiters on it
- fn set(&self) {
- let mut info = self.info.lock();
- debug_assert!(!info.complete);
- info.complete = true;
- let registry = rayon_core::Registry::current();
- for waiter in info.waiters.drain(..) {
- waiter.notify(®istry);
- }
- }
-
- /// Removes a single waiter from the list of waiters.
- /// This is used to break query cycles.
- fn extract_waiter(&self, waiter: usize) -> Lrc<QueryWaiter<CTX>> {
- let mut info = self.info.lock();
- debug_assert!(!info.complete);
- // Remove the waiter from the list of waiters
- info.waiters.remove(waiter)
- }
-}
-
-/// A resumable waiter of a query. The usize is the index into waiters in the query's latch
-#[cfg(parallel_compiler)]
-type Waiter<K> = (QueryJobId<K>, usize);
-
-/// Visits all the non-resumable and resumable waiters of a query.
-/// Only waiters in a query are visited.
-/// `visit` is called for every waiter and is passed a query waiting on `query_ref`
-/// and a span indicating the reason the query waited on `query_ref`.
-/// If `visit` returns Some, this function returns.
-/// For visits of non-resumable waiters it returns the return value of `visit`.
-/// For visits of resumable waiters it returns Some(Some(Waiter)) which has the
-/// required information to resume the waiter.
-/// If all `visit` calls returns None, this function also returns None.
-#[cfg(parallel_compiler)]
-fn visit_waiters<CTX: QueryContext, F>(
- query_map: &QueryMap<CTX>,
- query: QueryJobId<CTX::DepKind>,
- mut visit: F,
-) -> Option<Option<Waiter<CTX::DepKind>>>
-where
- F: FnMut(Span, QueryJobId<CTX::DepKind>) -> Option<Option<Waiter<CTX::DepKind>>>,
-{
- // Visit the parent query which is a non-resumable waiter since it's on the same stack
- if let Some(parent) = query.parent(query_map) {
- if let Some(cycle) = visit(query.span(query_map), parent) {
- return Some(cycle);
- }
- }
-
- // Visit the explicit waiters which use condvars and are resumable
- if let Some(latch) = query.latch(query_map) {
- for (i, waiter) in latch.info.lock().waiters.iter().enumerate() {
- if let Some(waiter_query) = waiter.query {
- if visit(waiter.span, waiter_query).is_some() {
- // Return a value which indicates that this waiter can be resumed
- return Some(Some((query, i)));
- }
- }
- }
- }
-
- None
-}
-
-/// Look for query cycles by doing a depth first search starting at `query`.
-/// `span` is the reason for the `query` to execute. This is initially DUMMY_SP.
-/// If a cycle is detected, this initial value is replaced with the span causing
-/// the cycle.
-#[cfg(parallel_compiler)]
-fn cycle_check<CTX: QueryContext>(
- query_map: &QueryMap<CTX>,
- query: QueryJobId<CTX::DepKind>,
- span: Span,
- stack: &mut Vec<(Span, QueryJobId<CTX::DepKind>)>,
- visited: &mut FxHashSet<QueryJobId<CTX::DepKind>>,
-) -> Option<Option<Waiter<CTX::DepKind>>> {
- if !visited.insert(query) {
- return if let Some(p) = stack.iter().position(|q| q.1 == query) {
- // We detected a query cycle, fix up the initial span and return Some
-
- // Remove previous stack entries
- stack.drain(0..p);
- // Replace the span for the first query with the cycle cause
- stack[0].0 = span;
- Some(None)
- } else {
- None
- };
- }
-
- // Query marked as visited is added it to the stack
- stack.push((span, query));
-
- // Visit all the waiters
- let r = visit_waiters(query_map, query, |span, successor| {
- cycle_check(query_map, successor, span, stack, visited)
- });
-
- // Remove the entry in our stack if we didn't find a cycle
- if r.is_none() {
- stack.pop();
- }
-
- r
-}
-
-/// Finds out if there's a path to the compiler root (aka. code which isn't in a query)
-/// from `query` without going through any of the queries in `visited`.
-/// This is achieved with a depth first search.
-#[cfg(parallel_compiler)]
-fn connected_to_root<CTX: QueryContext>(
- query_map: &QueryMap<CTX>,
- query: QueryJobId<CTX::DepKind>,
- visited: &mut FxHashSet<QueryJobId<CTX::DepKind>>,
-) -> bool {
- // We already visited this or we're deliberately ignoring it
- if !visited.insert(query) {
- return false;
- }
-
- // This query is connected to the root (it has no query parent), return true
- if query.parent(query_map).is_none() {
- return true;
- }
-
- visit_waiters(query_map, query, |_, successor| {
- connected_to_root(query_map, successor, visited).then_some(None)
- })
- .is_some()
-}
-
-// Deterministically pick an query from a list
-#[cfg(parallel_compiler)]
-fn pick_query<'a, CTX, T, F>(query_map: &QueryMap<CTX>, tcx: CTX, queries: &'a [T], f: F) -> &'a T
-where
- CTX: QueryContext,
- F: Fn(&T) -> (Span, QueryJobId<CTX::DepKind>),
-{
- // Deterministically pick an entry point
- // FIXME: Sort this instead
- let mut hcx = tcx.create_stable_hashing_context();
- queries
- .iter()
- .min_by_key(|v| {
- let (span, query) = f(v);
- let mut stable_hasher = StableHasher::new();
- query.query(query_map).hash_stable(&mut hcx, &mut stable_hasher);
- // Prefer entry points which have valid spans for nicer error messages
- // We add an integer to the tuple ensuring that entry points
- // with valid spans are picked first
- let span_cmp = if span == DUMMY_SP { 1 } else { 0 };
- (span_cmp, stable_hasher.finish::<u64>())
- })
- .unwrap()
-}
-
-/// Looks for query cycles starting from the last query in `jobs`.
-/// If a cycle is found, all queries in the cycle is removed from `jobs` and
-/// the function return true.
-/// If a cycle was not found, the starting query is removed from `jobs` and
-/// the function returns false.
-#[cfg(parallel_compiler)]
-fn remove_cycle<CTX: QueryContext>(
- query_map: &QueryMap<CTX>,
- jobs: &mut Vec<QueryJobId<CTX::DepKind>>,
- wakelist: &mut Vec<Lrc<QueryWaiter<CTX>>>,
- tcx: CTX,
-) -> bool {
- let mut visited = FxHashSet::default();
- let mut stack = Vec::new();
- // Look for a cycle starting with the last query in `jobs`
- if let Some(waiter) =
- cycle_check(query_map, jobs.pop().unwrap(), DUMMY_SP, &mut stack, &mut visited)
- {
- // The stack is a vector of pairs of spans and queries; reverse it so that
- // the earlier entries require later entries
- let (mut spans, queries): (Vec<_>, Vec<_>) = stack.into_iter().rev().unzip();
-
- // Shift the spans so that queries are matched with the span for their waitee
- spans.rotate_right(1);
-
- // Zip them back together
- let mut stack: Vec<_> = spans.into_iter().zip(queries).collect();
-
- // Remove the queries in our cycle from the list of jobs to look at
- for r in &stack {
- jobs.remove_item(&r.1);
- }
-
- // Find the queries in the cycle which are
- // connected to queries outside the cycle
- let entry_points = stack
- .iter()
- .filter_map(|&(span, query)| {
- if query.parent(query_map).is_none() {
- // This query is connected to the root (it has no query parent)
- Some((span, query, None))
- } else {
- let mut waiters = Vec::new();
- // Find all the direct waiters who lead to the root
- visit_waiters(query_map, query, |span, waiter| {
- // Mark all the other queries in the cycle as already visited
- let mut visited = FxHashSet::from_iter(stack.iter().map(|q| q.1));
-
- if connected_to_root(query_map, waiter, &mut visited) {
- waiters.push((span, waiter));
- }
-
- None
- });
- if waiters.is_empty() {
- None
- } else {
- // Deterministically pick one of the waiters to show to the user
- let waiter = *pick_query(query_map, tcx, &waiters, |s| *s);
- Some((span, query, Some(waiter)))
- }
- }
- })
- .collect::<Vec<(Span, QueryJobId<CTX::DepKind>, Option<(Span, QueryJobId<CTX::DepKind>)>)>>();
-
- // Deterministically pick an entry point
- let (_, entry_point, usage) = pick_query(query_map, tcx, &entry_points, |e| (e.0, e.1));
-
- // Shift the stack so that our entry point is first
- let entry_point_pos = stack.iter().position(|(_, query)| query == entry_point);
- if let Some(pos) = entry_point_pos {
- stack.rotate_left(pos);
- }
-
- let usage = usage.as_ref().map(|(span, query)| (*span, query.query(query_map)));
-
- // Create the cycle error
- let error = CycleError {
- usage,
- cycle: stack
- .iter()
- .map(|&(s, ref q)| QueryInfo { span: s, query: q.query(query_map) })
- .collect(),
- };
-
- // We unwrap `waiter` here since there must always be one
- // edge which is resumeable / waited using a query latch
- let (waitee_query, waiter_idx) = waiter.unwrap();
-
- // Extract the waiter we want to resume
- let waiter = waitee_query.latch(query_map).unwrap().extract_waiter(waiter_idx);
-
- // Set the cycle error so it will be picked up when resumed
- *waiter.cycle.lock() = Some(error);
-
- // Put the waiter on the list of things to resume
- wakelist.push(waiter);
-
- true
- } else {
- false
- }
-}
-
-/// Creates a new thread and forwards information in thread locals to it.
-/// The new thread runs the deadlock handler.
-/// Must only be called when a deadlock is about to happen.
-#[cfg(parallel_compiler)]
-pub unsafe fn handle_deadlock() {
- let registry = rayon_core::Registry::current();
-
- let gcx_ptr = tls::GCX_PTR.with(|gcx_ptr| gcx_ptr as *const _);
- let gcx_ptr = &*gcx_ptr;
-
- let rustc_span_globals =
- rustc_span::GLOBALS.with(|rustc_span_globals| rustc_span_globals as *const _);
- let rustc_span_globals = &*rustc_span_globals;
- let syntax_globals = rustc_ast::attr::GLOBALS.with(|syntax_globals| syntax_globals as *const _);
- let syntax_globals = &*syntax_globals;
- thread::spawn(move || {
- tls::GCX_PTR.set(gcx_ptr, || {
- rustc_ast::attr::GLOBALS.set(syntax_globals, || {
- rustc_span::GLOBALS
- .set(rustc_span_globals, || tls::with_global(|tcx| deadlock(tcx, ®istry)))
- });
- })
- });
-}
-
-/// Detects query cycles by using depth first search over all active query jobs.
-/// If a query cycle is found it will break the cycle by finding an edge which
-/// uses a query latch and then resuming that waiter.
-/// There may be multiple cycles involved in a deadlock, so this searches
-/// all active queries for cycles before finally resuming all the waiters at once.
-#[cfg(parallel_compiler)]
-fn deadlock<CTX: QueryContext>(tcx: CTX, registry: &rayon_core::Registry) {
- let on_panic = OnDrop(|| {
- eprintln!("deadlock handler panicked, aborting process");
- process::abort();
- });
-
- let mut wakelist = Vec::new();
- let query_map = tcx.try_collect_active_jobs().unwrap();
- let mut jobs: Vec<QueryJobId<CTX::DepKind>> = query_map.keys().cloned().collect();
-
- let mut found_cycle = false;
-
- while jobs.len() > 0 {
- if remove_cycle(&query_map, &mut jobs, &mut wakelist, tcx) {
- found_cycle = true;
- }
- }
-
- // Check that a cycle was found. It is possible for a deadlock to occur without
- // a query cycle if a query which can be waited on uses Rayon to do multithreading
- // internally. Such a query (X) may be executing on 2 threads (A and B) and A may
- // wait using Rayon on B. Rayon may then switch to executing another query (Y)
- // which in turn will wait on X causing a deadlock. We have a false dependency from
- // X to Y due to Rayon waiting and a true dependency from Y to X. The algorithm here
- // only considers the true dependency and won't detect a cycle.
- assert!(found_cycle);
-
- // FIXME: Ensure this won't cause a deadlock before we return
- for waiter in wakelist.into_iter() {
- waiter.notify(registry);
- }
-
- on_panic.disable();
-}
+++ /dev/null
-use crate::dep_graph::{self, DepConstructor, DepNode, DepNodeParams};
-use crate::hir::exports::Export;
-use crate::hir::map;
-use crate::infer::canonical::{self, Canonical};
-use crate::lint::LintLevelMap;
-use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
-use crate::middle::cstore::{CrateSource, DepKind, NativeLibraryKind};
-use crate::middle::cstore::{ExternCrate, ForeignModule, LinkagePreference, NativeLibrary};
-use crate::middle::exported_symbols::{ExportedSymbol, SymbolExportLevel};
-use crate::middle::lang_items::{LangItem, LanguageItems};
-use crate::middle::lib_features::LibFeatures;
-use crate::middle::privacy::AccessLevels;
-use crate::middle::region;
-use crate::middle::resolve_lifetime::{ObjectLifetimeDefault, Region, ResolveLifetimes};
-use crate::middle::stability::{self, DeprecationEntry};
-use crate::mir;
-use crate::mir::interpret::GlobalId;
-use crate::mir::interpret::{ConstEvalRawResult, ConstEvalResult, ConstValue};
-use crate::mir::interpret::{LitToConstError, LitToConstInput};
-use crate::mir::mono::CodegenUnit;
-use crate::traits::query::{
- CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal,
- CanonicalTypeOpAscribeUserTypeGoal, CanonicalTypeOpEqGoal, CanonicalTypeOpNormalizeGoal,
- CanonicalTypeOpProvePredicateGoal, CanonicalTypeOpSubtypeGoal, NoSolution,
-};
-use crate::traits::query::{
- DropckOutlivesResult, DtorckConstraint, MethodAutoderefStepsResult, NormalizationResult,
- OutlivesBound,
-};
-use crate::traits::specialization_graph;
-use crate::traits::Clauses;
-use crate::traits::{self, Vtable};
-use crate::ty::steal::Steal;
-use crate::ty::subst::{GenericArg, SubstsRef};
-use crate::ty::util::AlwaysRequiresDrop;
-use crate::ty::{self, AdtSizedConstraint, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt};
-use crate::util::common::ErrorReported;
-use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
-use rustc_data_structures::profiling::ProfileCategory::*;
-use rustc_data_structures::stable_hasher::StableVec;
-use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::Lrc;
-use rustc_hir as hir;
-use rustc_hir::def::DefKind;
-use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId};
-use rustc_hir::{Crate, HirIdSet, ItemLocalId, TraitCandidate};
-use rustc_index::vec::IndexVec;
-use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion};
-use rustc_session::CrateDisambiguator;
-use rustc_target::spec::PanicStrategy;
-
-use rustc_ast::ast;
-use rustc_attr as attr;
-use rustc_span::symbol::Symbol;
-use rustc_span::{Span, DUMMY_SP};
-use std::borrow::Cow;
-use std::collections::BTreeMap;
-use std::ops::Deref;
-use std::sync::Arc;
-
-#[macro_use]
-mod plumbing;
-pub(crate) use self::plumbing::CycleError;
-use self::plumbing::*;
-
-mod stats;
-pub use self::stats::print_stats;
-
-mod job;
-#[cfg(parallel_compiler)]
-pub use self::job::handle_deadlock;
-use self::job::QueryJobInfo;
-pub use self::job::{QueryInfo, QueryJob, QueryJobId};
-
-mod keys;
-use self::keys::Key;
-
-mod values;
-use self::values::Value;
-
-mod caches;
-use self::caches::CacheSelector;
-
-mod config;
-use self::config::QueryAccessors;
-pub use self::config::QueryConfig;
-pub(crate) use self::config::QueryDescription;
-
-mod on_disk_cache;
-pub use self::on_disk_cache::OnDiskCache;
-
-mod profiling_support;
-pub use self::profiling_support::{IntoSelfProfilingString, QueryKeyStringBuilder};
-
-// Each of these queries corresponds to a function pointer field in the
-// `Providers` struct for requesting a value of that type, and a method
-// on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
-// which memoizes and does dep-graph tracking, wrapping around the actual
-// `Providers` that the driver creates (using several `rustc_*` crates).
-//
-// The result type of each query must implement `Clone`, and additionally
-// `ty::query::values::Value`, which produces an appropriate placeholder
-// (error) value if the query resulted in a query cycle.
-// Queries marked with `fatal_cycle` do not need the latter implementation,
-// as they will raise an fatal error on query cycles instead.
-
-rustc_query_append! { [define_queries!][<'tcx>] }
-
-/// The red/green evaluation system will try to mark a specific DepNode in the
-/// dependency graph as green by recursively trying to mark the dependencies of
-/// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode`
-/// where we don't know if it is red or green and we therefore actually have
-/// to recompute its value in order to find out. Since the only piece of
-/// information that we have at that point is the `DepNode` we are trying to
-/// re-evaluate, we need some way to re-run a query from just that. This is what
-/// `force_from_dep_node()` implements.
-///
-/// In the general case, a `DepNode` consists of a `DepKind` and an opaque
-/// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint
-/// is usually constructed by computing a stable hash of the query-key that the
-/// `DepNode` corresponds to. Consequently, it is not in general possible to go
-/// back from hash to query-key (since hash functions are not reversible). For
-/// this reason `force_from_dep_node()` is expected to fail from time to time
-/// because we just cannot find out, from the `DepNode` alone, what the
-/// corresponding query-key is and therefore cannot re-run the query.
-///
-/// The system deals with this case letting `try_mark_green` fail which forces
-/// the root query to be re-evaluated.
-///
-/// Now, if `force_from_dep_node()` would always fail, it would be pretty useless.
-/// Fortunately, we can use some contextual information that will allow us to
-/// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we
-/// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a
-/// valid `DefPathHash`. Since we also always build a huge table that maps every
-/// `DefPathHash` in the current codebase to the corresponding `DefId`, we have
-/// everything we need to re-run the query.
-///
-/// Take the `mir_validated` query as an example. Like many other queries, it
-/// just has a single parameter: the `DefId` of the item it will compute the
-/// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode`
-/// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode`
-/// is actually a `DefPathHash`, and can therefore just look up the corresponding
-/// `DefId` in `tcx.def_path_hash_to_def_id`.
-///
-/// When you implement a new query, it will likely have a corresponding new
-/// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As
-/// a rule of thumb, if your query takes a `DefId` or `LocalDefId` as sole parameter,
-/// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
-/// add it to the "We don't have enough information to reconstruct..." group in
-/// the match below.
-pub fn force_from_dep_node<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> bool {
- // We must avoid ever having to call `force_from_dep_node()` for a
- // `DepNode::codegen_unit`:
- // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
- // would always end up having to evaluate the first caller of the
- // `codegen_unit` query that *is* reconstructible. This might very well be
- // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
- // to re-trigger calling the `codegen_unit` query with the right key. At
- // that point we would already have re-done all the work we are trying to
- // avoid doing in the first place.
- // The solution is simple: Just explicitly call the `codegen_unit` query for
- // each CGU, right after partitioning. This way `try_mark_green` will always
- // hit the cache instead of having to go through `force_from_dep_node`.
- // This assertion makes sure, we actually keep applying the solution above.
- debug_assert!(
- dep_node.kind != crate::dep_graph::DepKind::codegen_unit,
- "calling force_from_dep_node() on DepKind::codegen_unit"
- );
-
- if !dep_node.kind.can_reconstruct_query_key() {
- return false;
- }
-
- rustc_dep_node_force!([dep_node, tcx]
- // These are inputs that are expected to be pre-allocated and that
- // should therefore always be red or green already.
- crate::dep_graph::DepKind::CrateMetadata |
-
- // These are anonymous nodes.
- crate::dep_graph::DepKind::TraitSelect |
-
- // We don't have enough information to reconstruct the query key of
- // these.
- crate::dep_graph::DepKind::CompileCodegenUnit => {
- bug!("force_from_dep_node: encountered {:?}", dep_node)
- }
- );
-
- false
-}
-
-pub(crate) fn try_load_from_on_disk_cache<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) {
- rustc_dep_node_try_load_from_on_disk_cache!(dep_node, tcx)
-}
+++ /dev/null
-//! The implementation of the query system itself. This defines the macros that
-//! generate the actual methods on tcx which find and execute the provider,
-//! manage the caches, and so forth.
-
-use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
-use crate::ty::query::caches::QueryCache;
-use crate::ty::query::config::{QueryContext, QueryDescription};
-use crate::ty::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId};
-use crate::ty::query::Query;
-use crate::ty::tls::{self, ImplicitCtxt};
-use crate::ty::{self, TyCtxt};
-
-#[cfg(not(parallel_compiler))]
-use rustc_data_structures::cold_path;
-use rustc_data_structures::fx::{FxHashMap, FxHasher};
-use rustc_data_structures::sharded::Sharded;
-use rustc_data_structures::sync::{Lock, LockGuard};
-use rustc_data_structures::thin_vec::ThinVec;
-use rustc_errors::{struct_span_err, Diagnostic, DiagnosticBuilder, FatalError, Handler, Level};
-use rustc_query_system::dep_graph::{DepContext, DepGraph, DepKind, DepNode};
-use rustc_query_system::HashStableContextProvider;
-use rustc_session::Session;
-use rustc_span::def_id::DefId;
-use rustc_span::source_map::DUMMY_SP;
-use rustc_span::Span;
-use std::collections::hash_map::Entry;
-use std::convert::TryFrom;
-use std::fmt::Debug;
-use std::hash::{Hash, Hasher};
-use std::mem;
-use std::num::NonZeroU32;
-use std::ptr;
-#[cfg(debug_assertions)]
-use std::sync::atomic::{AtomicUsize, Ordering};
-
-pub(crate) struct QueryStateShard<CTX: QueryContext, K, C> {
- cache: C,
- active: FxHashMap<K, QueryResult<CTX>>,
-
- /// Used to generate unique ids for active jobs.
- jobs: u32,
-}
-
-impl<CTX: QueryContext, K, C> QueryStateShard<CTX, K, C> {
- fn get_cache(&mut self) -> &mut C {
- &mut self.cache
- }
-}
-
-impl<CTX: QueryContext, K, C: Default> Default for QueryStateShard<CTX, K, C> {
- fn default() -> QueryStateShard<CTX, K, C> {
- QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 }
- }
-}
-
-pub(crate) struct QueryState<CTX: QueryContext, C: QueryCache<CTX>> {
- cache: C,
- shards: Sharded<QueryStateShard<CTX, C::Key, C::Sharded>>,
- #[cfg(debug_assertions)]
- pub(super) cache_hits: AtomicUsize,
-}
-
-impl<CTX: QueryContext, C: QueryCache<CTX>> QueryState<CTX, C> {
- pub(super) fn get_lookup<K2: Hash>(
- &'tcx self,
- key: &K2,
- ) -> QueryLookup<'tcx, CTX, C::Key, C::Sharded> {
- // We compute the key's hash once and then use it for both the
- // shard lookup and the hashmap lookup. This relies on the fact
- // that both of them use `FxHasher`.
- let mut hasher = FxHasher::default();
- key.hash(&mut hasher);
- let key_hash = hasher.finish();
-
- let shard = self.shards.get_shard_index_by_hash(key_hash);
- let lock = self.shards.get_shard_by_index(shard).lock();
- QueryLookup { key_hash, shard, lock }
- }
-}
-
-/// Indicates the state of a query for a given key in a query map.
-enum QueryResult<CTX: QueryContext> {
- /// An already executing query. The query job can be used to await for its completion.
- Started(QueryJob<CTX>),
-
- /// The query panicked. Queries trying to wait on this will raise a fatal error which will
- /// silently panic.
- Poisoned,
-}
-
-impl<CTX: QueryContext, C: QueryCache<CTX>> QueryState<CTX, C> {
- pub(super) fn iter_results<R>(
- &self,
- f: impl for<'a> FnOnce(
- Box<dyn Iterator<Item = (&'a C::Key, &'a C::Value, DepNodeIndex)> + 'a>,
- ) -> R,
- ) -> R {
- self.cache.iter(&self.shards, |shard| &mut shard.cache, f)
- }
- pub(super) fn all_inactive(&self) -> bool {
- let shards = self.shards.lock_shards();
- shards.iter().all(|shard| shard.active.is_empty())
- }
-
- pub(super) fn try_collect_active_jobs(
- &self,
- kind: CTX::DepKind,
- make_query: fn(C::Key) -> CTX::Query,
- jobs: &mut FxHashMap<QueryJobId<CTX::DepKind>, QueryJobInfo<CTX>>,
- ) -> Option<()>
- where
- C::Key: Clone,
- {
- // We use try_lock_shards here since we are called from the
- // deadlock handler, and this shouldn't be locked.
- let shards = self.shards.try_lock_shards()?;
- let shards = shards.iter().enumerate();
- jobs.extend(shards.flat_map(|(shard_id, shard)| {
- shard.active.iter().filter_map(move |(k, v)| {
- if let QueryResult::Started(ref job) = *v {
- let id =
- QueryJobId { job: job.id, shard: u16::try_from(shard_id).unwrap(), kind };
- let info = QueryInfo { span: job.span, query: make_query(k.clone()) };
- Some((id, QueryJobInfo { info, job: job.clone() }))
- } else {
- None
- }
- })
- }));
-
- Some(())
- }
-}
-
-impl<CTX: QueryContext, C: QueryCache<CTX>> Default for QueryState<CTX, C> {
- fn default() -> QueryState<CTX, C> {
- QueryState {
- cache: C::default(),
- shards: Default::default(),
- #[cfg(debug_assertions)]
- cache_hits: AtomicUsize::new(0),
- }
- }
-}
-
-/// Values used when checking a query cache which can be reused on a cache-miss to execute the query.
-pub(crate) struct QueryLookup<'tcx, CTX: QueryContext, K, C> {
- pub(super) key_hash: u64,
- shard: usize,
- pub(super) lock: LockGuard<'tcx, QueryStateShard<CTX, K, C>>,
-}
-
-/// A type representing the responsibility to execute the job in the `job` field.
-/// This will poison the relevant query if dropped.
-struct JobOwner<'tcx, CTX: QueryContext, C>
-where
- C: QueryCache<CTX>,
- C::Key: Eq + Hash + Clone + Debug,
- C::Value: Clone,
-{
- state: &'tcx QueryState<CTX, C>,
- key: C::Key,
- id: QueryJobId<CTX::DepKind>,
-}
-
-impl<'tcx, CTX: QueryContext, C> JobOwner<'tcx, CTX, C>
-where
- C: QueryCache<CTX>,
- C::Key: Eq + Hash + Clone + Debug,
- C::Value: Clone,
-{
- /// Either gets a `JobOwner` corresponding the query, allowing us to
- /// start executing the query, or returns with the result of the query.
- /// This function assumes that `try_get_cached` is already called and returned `lookup`.
- /// If the query is executing elsewhere, this will wait for it and return the result.
- /// If the query panicked, this will silently panic.
- ///
- /// This function is inlined because that results in a noticeable speed-up
- /// for some compile-time benchmarks.
- #[inline(always)]
- fn try_start<'a, 'b, Q, K>(
- tcx: CTX,
- span: Span,
- key: &C::Key,
- mut lookup: QueryLookup<'a, CTX, C::Key, C::Sharded>,
- ) -> TryGetJob<'b, CTX, C>
- where
- K: DepKind,
- Q: QueryDescription<CTX, Key = C::Key, Value = C::Value, Cache = C>,
- CTX: QueryContext<DepKind = K>,
- {
- let lock = &mut *lookup.lock;
-
- let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) {
- Entry::Occupied(mut entry) => {
- match entry.get_mut() {
- QueryResult::Started(job) => {
- // For parallel queries, we'll block and wait until the query running
- // in another thread has completed. Record how long we wait in the
- // self-profiler.
- let _query_blocked_prof_timer = if cfg!(parallel_compiler) {
- Some(tcx.profiler().query_blocked())
- } else {
- None
- };
-
- // Create the id of the job we're waiting for
- let id = QueryJobId::new(job.id, lookup.shard, Q::DEP_KIND);
-
- (job.latch(id), _query_blocked_prof_timer)
- }
- QueryResult::Poisoned => FatalError.raise(),
- }
- }
- Entry::Vacant(entry) => {
- // No job entry for this query. Return a new one to be started later.
-
- // Generate an id unique within this shard.
- let id = lock.jobs.checked_add(1).unwrap();
- lock.jobs = id;
- let id = QueryShardJobId(NonZeroU32::new(id).unwrap());
-
- let global_id = QueryJobId::new(id, lookup.shard, Q::DEP_KIND);
-
- let job = tcx.read_query_job(|query| QueryJob::new(id, span, query));
-
- entry.insert(QueryResult::Started(job));
-
- let owner =
- JobOwner { state: Q::query_state(tcx), id: global_id, key: (*key).clone() };
- return TryGetJob::NotYetStarted(owner);
- }
- };
- mem::drop(lookup.lock);
-
- // If we are single-threaded we know that we have cycle error,
- // so we just return the error.
- #[cfg(not(parallel_compiler))]
- return TryGetJob::Cycle(cold_path(|| {
- Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span))
- }));
-
- // With parallel queries we might just have to wait on some other
- // thread.
- #[cfg(parallel_compiler)]
- {
- let result = latch.wait_on(tcx, span);
-
- if let Err(cycle) = result {
- return TryGetJob::Cycle(Q::handle_cycle_error(tcx, cycle));
- }
-
- let cached = try_get_cached(
- tcx,
- Q::query_state(tcx),
- (*key).clone(),
- |value, index| (value.clone(), index),
- |_, _| panic!("value must be in cache after waiting"),
- );
-
- if let Some(prof_timer) = _query_blocked_prof_timer.take() {
- prof_timer.finish_with_query_invocation_id(cached.1.into());
- }
-
- return TryGetJob::JobCompleted(cached);
- }
- }
-
- /// Completes the query by updating the query cache with the `result`,
- /// signals the waiter and forgets the JobOwner, so it won't poison the query
- #[inline(always)]
- fn complete(self, tcx: CTX, result: &C::Value, dep_node_index: DepNodeIndex) {
- // We can move out of `self` here because we `mem::forget` it below
- let key = unsafe { ptr::read(&self.key) };
- let state = self.state;
-
- // Forget ourself so our destructor won't poison the query
- mem::forget(self);
-
- let job = {
- let result = result.clone();
- let mut lock = state.shards.get_shard_by_value(&key).lock();
- let job = match lock.active.remove(&key).unwrap() {
- QueryResult::Started(job) => job,
- QueryResult::Poisoned => panic!(),
- };
- state.cache.complete(tcx, &mut lock.cache, key, result, dep_node_index);
- job
- };
-
- job.signal_complete();
- }
-}
-
-#[inline(always)]
-fn with_diagnostics<F, R>(f: F) -> (R, ThinVec<Diagnostic>)
-where
- F: FnOnce(Option<&Lock<ThinVec<Diagnostic>>>) -> R,
-{
- let diagnostics = Lock::new(ThinVec::new());
- let result = f(Some(&diagnostics));
- (result, diagnostics.into_inner())
-}
-
-impl<'tcx, CTX: QueryContext, C: QueryCache<CTX>> Drop for JobOwner<'tcx, CTX, C>
-where
- C::Key: Eq + Hash + Clone + Debug,
- C::Value: Clone,
-{
- #[inline(never)]
- #[cold]
- fn drop(&mut self) {
- // Poison the query so jobs waiting on it panic.
- let state = self.state;
- let shard = state.shards.get_shard_by_value(&self.key);
- let job = {
- let mut shard = shard.lock();
- let job = match shard.active.remove(&self.key).unwrap() {
- QueryResult::Started(job) => job,
- QueryResult::Poisoned => panic!(),
- };
- shard.active.insert(self.key.clone(), QueryResult::Poisoned);
- job
- };
- // Also signal the completion of the job, so waiters
- // will continue execution.
- job.signal_complete();
- }
-}
-
-#[derive(Clone)]
-pub(crate) struct CycleError<Q> {
- /// The query and related span that uses the cycle.
- pub(super) usage: Option<(Span, Q)>,
- pub(super) cycle: Vec<QueryInfo<Q>>,
-}
-
-/// The result of `try_start`.
-enum TryGetJob<'tcx, CTX: QueryContext, C: QueryCache<CTX>>
-where
- C::Key: Eq + Hash + Clone + Debug,
- C::Value: Clone,
-{
- /// The query is not yet started. Contains a guard to the cache eventually used to start it.
- NotYetStarted(JobOwner<'tcx, CTX, C>),
-
- /// The query was already completed.
- /// Returns the result of the query and its dep-node index
- /// if it succeeded or a cycle error if it failed.
- #[cfg(parallel_compiler)]
- JobCompleted((C::Value, DepNodeIndex)),
-
- /// Trying to execute the query resulted in a cycle.
- Cycle(C::Value),
-}
-
-impl QueryContext for TyCtxt<'tcx> {
- type Query = Query<'tcx>;
-
- fn session(&self) -> &Session {
- &self.sess
- }
-
- fn def_path_str(&self, def_id: DefId) -> String {
- TyCtxt::def_path_str(*self, def_id)
- }
-
- fn dep_graph(&self) -> &DepGraph<Self::DepKind> {
- &self.dep_graph
- }
-
- fn read_query_job<R>(&self, op: impl FnOnce(Option<QueryJobId<Self::DepKind>>) -> R) -> R {
- tls::with_related_context(*self, move |icx| op(icx.query))
- }
-
- fn try_collect_active_jobs(
- &self,
- ) -> Option<FxHashMap<QueryJobId<Self::DepKind>, QueryJobInfo<Self>>> {
- self.queries.try_collect_active_jobs()
- }
-
- /// Executes a job by changing the `ImplicitCtxt` to point to the
- /// new query job while it executes. It returns the diagnostics
- /// captured during execution and the actual result.
- #[inline(always)]
- fn start_query<R>(
- &self,
- token: QueryJobId<Self::DepKind>,
- diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
- compute: impl FnOnce(Self) -> R,
- ) -> R {
- // The `TyCtxt` stored in TLS has the same global interner lifetime
- // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
- // when accessing the `ImplicitCtxt`.
- tls::with_related_context(*self, move |current_icx| {
- // Update the `ImplicitCtxt` to point to our new query job.
- let new_icx = ImplicitCtxt {
- tcx: *self,
- query: Some(token),
- diagnostics,
- layout_depth: current_icx.layout_depth,
- task_deps: current_icx.task_deps,
- };
-
- // Use the `ImplicitCtxt` while we execute the query.
- tls::enter_context(&new_icx, |_| compute(*self))
- })
- }
-}
-
-impl<'tcx> TyCtxt<'tcx> {
- #[inline(never)]
- #[cold]
- pub(super) fn report_cycle(
- self,
- CycleError { usage, cycle: stack }: CycleError<Query<'tcx>>,
- ) -> DiagnosticBuilder<'tcx> {
- assert!(!stack.is_empty());
-
- let fix_span = |span: Span, query: &Query<'tcx>| {
- self.sess.source_map().guess_head_span(query.default_span(self, span))
- };
-
- // Disable naming impls with types in this path, since that
- // sometimes cycles itself, leading to extra cycle errors.
- // (And cycle errors around impls tend to occur during the
- // collect/coherence phases anyhow.)
- ty::print::with_forced_impl_filename_line(|| {
- let span = fix_span(stack[1 % stack.len()].span, &stack[0].query);
- let mut err = struct_span_err!(
- self.sess,
- span,
- E0391,
- "cycle detected when {}",
- stack[0].query.describe(self)
- );
-
- for i in 1..stack.len() {
- let query = &stack[i].query;
- let span = fix_span(stack[(i + 1) % stack.len()].span, query);
- err.span_note(span, &format!("...which requires {}...", query.describe(self)));
- }
-
- err.note(&format!(
- "...which again requires {}, completing the cycle",
- stack[0].query.describe(self)
- ));
-
- if let Some((span, query)) = usage {
- err.span_note(
- fix_span(span, &query),
- &format!("cycle used when {}", query.describe(self)),
- );
- }
-
- err
- })
- }
-
- pub fn try_print_query_stack(handler: &Handler) {
- eprintln!("query stack during panic:");
-
- // Be careful reyling on global state here: this code is called from
- // a panic hook, which means that the global `Handler` may be in a weird
- // state if it was responsible for triggering the panic.
- tls::with_context_opt(|icx| {
- if let Some(icx) = icx {
- let query_map = icx.tcx.queries.try_collect_active_jobs();
-
- let mut current_query = icx.query;
- let mut i = 0;
-
- while let Some(query) = current_query {
- let query_info =
- if let Some(info) = query_map.as_ref().and_then(|map| map.get(&query)) {
- info
- } else {
- break;
- };
- let mut diag = Diagnostic::new(
- Level::FailureNote,
- &format!(
- "#{} [{}] {}",
- i,
- query_info.info.query.name(),
- query_info.info.query.describe(icx.tcx)
- ),
- );
- diag.span =
- icx.tcx.sess.source_map().guess_head_span(query_info.info.span).into();
- handler.force_print_diagnostic(diag);
-
- current_query = query_info.job.parent;
- i += 1;
- }
- }
- });
-
- eprintln!("end of query stack");
- }
-}
-
- /// Checks if the query is already computed and in the cache.
- /// It returns the shard index and a lock guard to the shard,
- /// which will be used if the query is not in the cache and we need
- /// to compute it.
- #[inline(always)]
- fn try_get_cached<CTX, C, R, OnHit, OnMiss>(
- tcx: CTX,
- state: &QueryState<CTX, C>,
- key: C::Key,
- // `on_hit` can be called while holding a lock to the query cache
- on_hit: OnHit,
- on_miss: OnMiss,
- ) -> R
- where
- C: QueryCache<CTX>,
- CTX: QueryContext,
- OnHit: FnOnce(&C::Value, DepNodeIndex) -> R,
- OnMiss: FnOnce(C::Key, QueryLookup<'_, CTX, C::Key, C::Sharded>) -> R,
- {
- state.cache.lookup(
- state,
- QueryStateShard::<CTX, C::Key, C::Sharded>::get_cache,
- key,
- |value, index| {
- if unlikely!(tcx.profiler().enabled()) {
- tcx.profiler().query_cache_hit(index.into());
- }
- #[cfg(debug_assertions)]
- {
- state.cache_hits.fetch_add(1, Ordering::Relaxed);
- }
- on_hit(value, index)
- },
- on_miss,
- )
- }
-
- #[inline(always)]
- fn try_execute_query<Q, CTX, K>(
- tcx: CTX,
- span: Span,
- key: Q::Key,
- lookup: QueryLookup<
- '_,
- CTX,
- Q::Key,
- <Q::Cache as QueryCache<CTX>>::Sharded,
- >,
- ) -> Q::Value
- where
- Q: QueryDescription<CTX>,
- CTX: QueryContext<DepKind = K>,
- CTX: HashStableContextProvider<<CTX as DepContext>::StableHashingContext>,
- K: DepKind,
- {
- let job = match JobOwner::try_start::<Q, _>(tcx, span, &key, lookup) {
- TryGetJob::NotYetStarted(job) => job,
- TryGetJob::Cycle(result) => return result,
- #[cfg(parallel_compiler)]
- TryGetJob::JobCompleted((v, index)) => {
- tcx.dep_graph().read_index(index);
- return v;
- }
- };
-
- // Fast path for when incr. comp. is off. `to_dep_node` is
- // expensive for some `DepKind`s.
- if !tcx.dep_graph().is_fully_enabled() {
- let null_dep_node = DepNode::new_no_params(DepKind::NULL);
- return force_query_with_job::<Q, _, _>(tcx, key, job, null_dep_node).0;
- }
-
- if Q::ANON {
- let prof_timer = tcx.profiler().query_provider();
-
- let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
- tcx.start_query(job.id, diagnostics, |tcx| {
- tcx.dep_graph().with_anon_task(Q::DEP_KIND, || Q::compute(tcx, key))
- })
- });
-
- prof_timer.finish_with_query_invocation_id(dep_node_index.into());
-
- tcx.dep_graph().read_index(dep_node_index);
-
- if unlikely!(!diagnostics.is_empty()) {
- tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics);
- }
-
- job.complete(tcx, &result, dep_node_index);
-
- return result;
- }
-
- let dep_node = Q::to_dep_node(tcx, &key);
-
- if !Q::EVAL_ALWAYS {
- // The diagnostics for this query will be
- // promoted to the current session during
- // `try_mark_green()`, so we can ignore them here.
- let loaded = tcx.start_query(job.id, None, |tcx| {
- let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node);
- marked.map(|(prev_dep_node_index, dep_node_index)| {
- (
- load_from_disk_and_cache_in_memory::<Q, _>(
- tcx,
- key.clone(),
- prev_dep_node_index,
- dep_node_index,
- &dep_node,
- ),
- dep_node_index,
- )
- })
- });
- if let Some((result, dep_node_index)) = loaded {
- job.complete(tcx, &result, dep_node_index);
- return result;
- }
- }
-
- let (result, dep_node_index) = force_query_with_job::<Q, _, _>(tcx, key, job, dep_node);
- tcx.dep_graph().read_index(dep_node_index);
- result
- }
-
- fn load_from_disk_and_cache_in_memory<Q, CTX>(
- tcx: CTX,
- key: Q::Key,
- prev_dep_node_index: SerializedDepNodeIndex,
- dep_node_index: DepNodeIndex,
- dep_node: &DepNode<CTX::DepKind>,
- ) -> Q::Value
- where
- CTX: QueryContext,
- Q: QueryDescription<CTX>,
- {
- // Note this function can be called concurrently from the same query
- // We must ensure that this is handled correctly.
-
- debug_assert!(tcx.dep_graph().is_green(dep_node));
-
- // First we try to load the result from the on-disk cache.
- let result = if Q::cache_on_disk(tcx, key.clone(), None) {
- let prof_timer = tcx.profiler().incr_cache_loading();
- let result = Q::try_load_from_disk(tcx, prev_dep_node_index);
- prof_timer.finish_with_query_invocation_id(dep_node_index.into());
-
- // We always expect to find a cached result for things that
- // can be forced from `DepNode`.
- debug_assert!(
- !dep_node.kind.can_reconstruct_query_key() || result.is_some(),
- "missing on-disk cache entry for {:?}",
- dep_node
- );
- result
- } else {
- // Some things are never cached on disk.
- None
- };
-
- let result = if let Some(result) = result {
- result
- } else {
- // We could not load a result from the on-disk cache, so
- // recompute.
- let prof_timer = tcx.profiler().query_provider();
-
- // The dep-graph for this computation is already in-place.
- let result = tcx.dep_graph().with_ignore(|| Q::compute(tcx, key));
-
- prof_timer.finish_with_query_invocation_id(dep_node_index.into());
-
- result
- };
-
- // If `-Zincremental-verify-ich` is specified, re-hash results from
- // the cache and make sure that they have the expected fingerprint.
- if unlikely!(tcx.session().opts.debugging_opts.incremental_verify_ich) {
- incremental_verify_ich::<Q, _>(tcx, &result, dep_node, dep_node_index);
- }
-
- result
- }
-
- #[inline(never)]
- #[cold]
- fn incremental_verify_ich<Q, CTX>(
- tcx: CTX,
- result: &Q::Value,
- dep_node: &DepNode<CTX::DepKind>,
- dep_node_index: DepNodeIndex,
- )
- where
- CTX: QueryContext,
- Q: QueryDescription<CTX>,
- {
- use rustc_data_structures::fingerprint::Fingerprint;
-
- assert!(
- Some(tcx.dep_graph().fingerprint_of(dep_node_index))
- == tcx.dep_graph().prev_fingerprint_of(dep_node),
- "fingerprint for green query instance not loaded from cache: {:?}",
- dep_node,
- );
-
- debug!("BEGIN verify_ich({:?})", dep_node);
- let mut hcx = tcx.create_stable_hashing_context();
-
- let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
- debug!("END verify_ich({:?})", dep_node);
-
- let old_hash = tcx.dep_graph().fingerprint_of(dep_node_index);
-
- assert!(new_hash == old_hash, "found unstable fingerprints for {:?}", dep_node,);
- }
-
- #[inline(always)]
- fn force_query_with_job<Q, CTX, K>(
- tcx: CTX,
- key: Q::Key,
- job: JobOwner<'tcx, CTX, Q::Cache>,
- dep_node: DepNode<CTX::DepKind>,
- ) -> (Q::Value, DepNodeIndex)
- where
- Q: QueryDescription<CTX>,
- CTX: QueryContext<DepKind = K>,
- CTX: HashStableContextProvider<<CTX as DepContext>::StableHashingContext>,
- K: DepKind,
- {
- // If the following assertion triggers, it can have two reasons:
- // 1. Something is wrong with DepNode creation, either here or
- // in `DepGraph::try_mark_green()`.
- // 2. Two distinct query keys get mapped to the same `DepNode`
- // (see for example #48923).
- assert!(
- !tcx.dep_graph().dep_node_exists(&dep_node),
- "forcing query with already existing `DepNode`\n\
- - query-key: {:?}\n\
- - dep-node: {:?}",
- key,
- dep_node
- );
-
- let prof_timer = tcx.profiler().query_provider();
-
- let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
- tcx.start_query(job.id, diagnostics, |tcx| {
- if Q::EVAL_ALWAYS {
- tcx.dep_graph().with_eval_always_task(
- dep_node,
- tcx,
- key,
- Q::compute,
- Q::hash_result,
- )
- } else {
- tcx.dep_graph().with_task(dep_node, tcx, key, Q::compute, Q::hash_result)
- }
- })
- });
-
- prof_timer.finish_with_query_invocation_id(dep_node_index.into());
-
- if unlikely!(!diagnostics.is_empty()) {
- if dep_node.kind != DepKind::NULL {
- tcx.store_diagnostics(dep_node_index, diagnostics);
- }
- }
-
- job.complete(tcx, &result, dep_node_index);
-
- (result, dep_node_index)
- }
-
-pub(super) trait QueryGetter: QueryContext {
- fn get_query<Q: QueryDescription<Self>>(
- self,
- span: Span,
- key: Q::Key,
- ) -> Q::Value;
-
- /// Ensure that either this query has all green inputs or been executed.
- /// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
- ///
- /// This function is particularly useful when executing passes for their
- /// side-effects -- e.g., in order to report errors for erroneous programs.
- ///
- /// Note: The optimization is only available during incr. comp.
- fn ensure_query<Q: QueryDescription<Self>>(self, key: Q::Key);
-
- fn force_query<Q: QueryDescription<Self>>(
- self,
- key: Q::Key,
- span: Span,
- dep_node: DepNode<Self::DepKind>,
- );
-}
-
-impl<CTX, K> QueryGetter for CTX
-where
- CTX: QueryContext<DepKind = K>,
- CTX: HashStableContextProvider<<CTX as DepContext>::StableHashingContext>,
- K: DepKind,
-{
- #[inline(never)]
- fn get_query<Q: QueryDescription<Self>>(
- self,
- span: Span,
- key: Q::Key,
- ) -> Q::Value {
- debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
-
- try_get_cached(
- self,
- Q::query_state(self),
- key,
- |value, index| {
- self.dep_graph().read_index(index);
- value.clone()
- },
- |key, lookup| try_execute_query::<Q, _, _>(self, span, key, lookup),
- )
- }
-
- /// Ensure that either this query has all green inputs or been executed.
- /// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
- ///
- /// This function is particularly useful when executing passes for their
- /// side-effects -- e.g., in order to report errors for erroneous programs.
- ///
- /// Note: The optimization is only available during incr. comp.
- fn ensure_query<Q: QueryDescription<Self>>(self, key: Q::Key) {
- if Q::EVAL_ALWAYS {
- let _ = self.get_query::<Q>(DUMMY_SP, key);
- return;
- }
-
- // Ensuring an anonymous query makes no sense
- assert!(!Q::ANON);
-
- let dep_node = Q::to_dep_node(self, &key);
-
- match self.dep_graph().try_mark_green_and_read(self, &dep_node) {
- None => {
- // A None return from `try_mark_green_and_read` means that this is either
- // a new dep node or that the dep node has already been marked red.
- // Either way, we can't call `dep_graph.read()` as we don't have the
- // DepNodeIndex. We must invoke the query itself. The performance cost
- // this introduces should be negligible as we'll immediately hit the
- // in-memory cache, or another query down the line will.
- let _ = self.get_query::<Q>(DUMMY_SP, key);
- }
- Some((_, dep_node_index)) => {
- self.profiler().query_cache_hit(dep_node_index.into());
- }
- }
- }
-
- fn force_query<Q: QueryDescription<Self>>(
- self,
- key: Q::Key,
- span: Span,
- dep_node: DepNode<Self::DepKind>,
- ) {
- // We may be concurrently trying both execute and force a query.
- // Ensure that only one of them runs the query.
-
- try_get_cached(
- self,
- Q::query_state(self),
- key,
- |_, _| {
- // Cache hit, do nothing
- },
- |key, lookup| {
- let job = match JobOwner::try_start::<Q, _>(self, span, &key, lookup) {
- TryGetJob::NotYetStarted(job) => job,
- TryGetJob::Cycle(_) => return,
- #[cfg(parallel_compiler)]
- TryGetJob::JobCompleted(_) => return,
- };
- force_query_with_job::<Q, _, _>(self, key, job, dep_node);
- },
- );
- }
-}
-
-macro_rules! handle_cycle_error {
- ([][$tcx: expr, $error:expr]) => {{
- $tcx.report_cycle($error).emit();
- Value::from_cycle_error($tcx)
- }};
- ([fatal_cycle $($rest:tt)*][$tcx:expr, $error:expr]) => {{
- $tcx.report_cycle($error).emit();
- $tcx.sess.abort_if_errors();
- unreachable!()
- }};
- ([cycle_delay_bug $($rest:tt)*][$tcx:expr, $error:expr]) => {{
- $tcx.report_cycle($error).delay_as_bug();
- Value::from_cycle_error($tcx)
- }};
- ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => {
- handle_cycle_error!([$($($modifiers)*)*][$($args)*])
- };
-}
-
-macro_rules! is_anon {
- ([]) => {{
- false
- }};
- ([anon $($rest:tt)*]) => {{
- true
- }};
- ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*]) => {
- is_anon!([$($($modifiers)*)*])
- };
-}
-
-macro_rules! is_eval_always {
- ([]) => {{
- false
- }};
- ([eval_always $($rest:tt)*]) => {{
- true
- }};
- ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*]) => {
- is_eval_always!([$($($modifiers)*)*])
- };
-}
-
-macro_rules! query_storage {
- (<$tcx:tt>[][$K:ty, $V:ty]) => {
- <<$K as Key>::CacheSelector as CacheSelector<TyCtxt<$tcx>, $K, $V>>::Cache
- };
- (<$tcx:tt>[storage($ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => {
- $ty
- };
- (<$tcx:tt>[$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => {
- query_storage!(<$tcx>[$($($modifiers)*)*][$($args)*])
- };
-}
-
-macro_rules! hash_result {
- ([][$hcx:expr, $result:expr]) => {{
- dep_graph::hash_result($hcx, &$result)
- }};
- ([no_hash $($rest:tt)*][$hcx:expr, $result:expr]) => {{
- None
- }};
- ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => {
- hash_result!([$($($modifiers)*)*][$($args)*])
- };
-}
-
-macro_rules! define_queries {
- (<$tcx:tt> $($category:tt {
- $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*
- },)*) => {
- define_queries_inner! { <$tcx>
- $($( $(#[$attr])* category<$category> [$($modifiers)*] fn $name: $node($K) -> $V,)*)*
- }
- }
-}
-
-macro_rules! define_queries_inner {
- (<$tcx:tt>
- $($(#[$attr:meta])* category<$category:tt>
- [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*) => {
-
- use std::mem;
- use crate::{
- rustc_data_structures::stable_hasher::HashStable,
- rustc_data_structures::stable_hasher::StableHasher,
- ich::StableHashingContext
- };
- use rustc_data_structures::profiling::ProfileCategory;
-
- define_queries_struct! {
- tcx: $tcx,
- input: ($(([$($modifiers)*] [$($attr)*] [$name]))*)
- }
-
- #[allow(nonstandard_style)]
- #[derive(Clone, Debug)]
- pub enum Query<$tcx> {
- $($(#[$attr])* $name($K)),*
- }
-
- impl<$tcx> Query<$tcx> {
- pub fn name(&self) -> &'static str {
- match *self {
- $(Query::$name(_) => stringify!($name),)*
- }
- }
-
- pub fn describe(&self, tcx: TyCtxt<$tcx>) -> Cow<'static, str> {
- let (r, name) = match *self {
- $(Query::$name(key) => {
- (queries::$name::describe(tcx, key), stringify!($name))
- })*
- };
- if tcx.sess.verbose() {
- format!("{} [{}]", r, name).into()
- } else {
- r
- }
- }
-
- // FIXME(eddyb) Get more valid `Span`s on queries.
- pub fn default_span(&self, tcx: TyCtxt<$tcx>, span: Span) -> Span {
- if !span.is_dummy() {
- return span;
- }
- // The `def_span` query is used to calculate `default_span`,
- // so exit to avoid infinite recursion.
- if let Query::def_span(..) = *self {
- return span
- }
- match *self {
- $(Query::$name(key) => key.default_span(tcx),)*
- }
- }
- }
-
- impl<'a, $tcx> HashStable<StableHashingContext<'a>> for Query<$tcx> {
- fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
- mem::discriminant(self).hash_stable(hcx, hasher);
- match *self {
- $(Query::$name(key) => key.hash_stable(hcx, hasher),)*
- }
- }
- }
-
- pub mod queries {
- use std::marker::PhantomData;
-
- $(#[allow(nonstandard_style)]
- pub struct $name<$tcx> {
- data: PhantomData<&$tcx ()>
- })*
- }
-
- $(impl<$tcx> QueryConfig<TyCtxt<$tcx>> for queries::$name<$tcx> {
- type Key = $K;
- type Value = $V;
- const NAME: &'static str = stringify!($name);
- const CATEGORY: ProfileCategory = $category;
- }
-
- impl<$tcx> QueryAccessors<TyCtxt<$tcx>> for queries::$name<$tcx> {
- const ANON: bool = is_anon!([$($modifiers)*]);
- const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]);
- const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::$node;
-
- type Cache = query_storage!(<$tcx>[$($modifiers)*][$K, $V]);
-
- #[inline(always)]
- fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<TyCtxt<$tcx>, Self::Cache> {
- &tcx.queries.$name
- }
-
- #[allow(unused)]
- #[inline(always)]
- fn to_dep_node(tcx: TyCtxt<$tcx>, key: &Self::Key) -> DepNode {
- DepConstructor::$node(tcx, *key)
- }
-
- #[inline]
- fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
- let provider = tcx.queries.providers.get(key.query_crate())
- // HACK(eddyb) it's possible crates may be loaded after
- // the query engine is created, and because crate loading
- // is not yet integrated with the query engine, such crates
- // would be missing appropriate entries in `providers`.
- .unwrap_or(&tcx.queries.fallback_extern_providers)
- .$name;
- provider(tcx, key)
- }
-
- fn hash_result(
- _hcx: &mut StableHashingContext<'_>,
- _result: &Self::Value
- ) -> Option<Fingerprint> {
- hash_result!([$($modifiers)*][_hcx, _result])
- }
-
- fn handle_cycle_error(
- tcx: TyCtxt<'tcx>,
- error: CycleError<Query<'tcx>>
- ) -> Self::Value {
- handle_cycle_error!([$($modifiers)*][tcx, error])
- }
- })*
-
- #[derive(Copy, Clone)]
- pub struct TyCtxtEnsure<'tcx> {
- pub tcx: TyCtxt<'tcx>,
- }
-
- impl TyCtxtEnsure<$tcx> {
- $($(#[$attr])*
- #[inline(always)]
- pub fn $name(self, key: $K) {
- self.tcx.ensure_query::<queries::$name<'_>>(key)
- })*
- }
-
- #[derive(Copy, Clone)]
- pub struct TyCtxtAt<'tcx> {
- pub tcx: TyCtxt<'tcx>,
- pub span: Span,
- }
-
- impl Deref for TyCtxtAt<'tcx> {
- type Target = TyCtxt<'tcx>;
- #[inline(always)]
- fn deref(&self) -> &Self::Target {
- &self.tcx
- }
- }
-
- impl TyCtxt<$tcx> {
- /// Returns a transparent wrapper for `TyCtxt`, which ensures queries
- /// are executed instead of just returning their results.
- #[inline(always)]
- pub fn ensure(self) -> TyCtxtEnsure<$tcx> {
- TyCtxtEnsure {
- tcx: self,
- }
- }
-
- /// Returns a transparent wrapper for `TyCtxt` which uses
- /// `span` as the location of queries performed through it.
- #[inline(always)]
- pub fn at(self, span: Span) -> TyCtxtAt<$tcx> {
- TyCtxtAt {
- tcx: self,
- span
- }
- }
-
- $($(#[$attr])*
- #[inline(always)]
- pub fn $name(self, key: $K) -> $V {
- self.at(DUMMY_SP).$name(key)
- })*
-
- /// All self-profiling events generated by the query engine use
- /// virtual `StringId`s for their `event_id`. This method makes all
- /// those virtual `StringId`s point to actual strings.
- ///
- /// If we are recording only summary data, the ids will point to
- /// just the query names. If we are recording query keys too, we
- /// allocate the corresponding strings here.
- pub fn alloc_self_profile_query_strings(self) {
- use crate::ty::query::profiling_support::{
- alloc_self_profile_query_strings_for_query_cache,
- QueryKeyStringCache,
- };
-
- if !self.prof.enabled() {
- return;
- }
-
- let mut string_cache = QueryKeyStringCache::new();
-
- $({
- alloc_self_profile_query_strings_for_query_cache(
- self,
- stringify!($name),
- &self.queries.$name,
- &mut string_cache,
- );
- })*
- }
- }
-
- impl TyCtxtAt<$tcx> {
- $($(#[$attr])*
- #[inline(always)]
- pub fn $name(self, key: $K) -> $V {
- self.tcx.get_query::<queries::$name<'_>>(self.span, key)
- })*
- }
-
- define_provider_struct! {
- tcx: $tcx,
- input: ($(([$($modifiers)*] [$name] [$K] [$V]))*)
- }
-
- impl<$tcx> Copy for Providers<$tcx> {}
- impl<$tcx> Clone for Providers<$tcx> {
- fn clone(&self) -> Self { *self }
- }
- }
-}
-
-macro_rules! define_queries_struct {
- (tcx: $tcx:tt,
- input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
- pub struct Queries<$tcx> {
- /// This provides access to the incrimental comilation on-disk cache for query results.
- /// Do not access this directly. It is only meant to be used by
- /// `DepGraph::try_mark_green()` and the query infrastructure.
- pub(crate) on_disk_cache: OnDiskCache<'tcx>,
-
- providers: IndexVec<CrateNum, Providers<$tcx>>,
- fallback_extern_providers: Box<Providers<$tcx>>,
-
- $($(#[$attr])* $name: QueryState<
- TyCtxt<$tcx>,
- <queries::$name<$tcx> as QueryAccessors<TyCtxt<'tcx>>>::Cache,
- >,)*
- }
-
- impl<$tcx> Queries<$tcx> {
- pub(crate) fn new(
- providers: IndexVec<CrateNum, Providers<$tcx>>,
- fallback_extern_providers: Providers<$tcx>,
- on_disk_cache: OnDiskCache<'tcx>,
- ) -> Self {
- Queries {
- providers,
- fallback_extern_providers: Box::new(fallback_extern_providers),
- on_disk_cache,
- $($name: Default::default()),*
- }
- }
-
- pub(crate) fn try_collect_active_jobs(
- &self
- ) -> Option<FxHashMap<QueryJobId<crate::dep_graph::DepKind>, QueryJobInfo<TyCtxt<'tcx>>>> {
- let mut jobs = FxHashMap::default();
-
- $(
- self.$name.try_collect_active_jobs(
- <queries::$name<'tcx> as QueryAccessors<TyCtxt<'tcx>>>::DEP_KIND,
- Query::$name,
- &mut jobs,
- )?;
- )*
-
- Some(jobs)
- }
- }
- };
-}
-
-macro_rules! define_provider_struct {
- (tcx: $tcx:tt,
- input: ($(([$($modifiers:tt)*] [$name:ident] [$K:ty] [$R:ty]))*)) => {
- pub struct Providers<$tcx> {
- $(pub $name: fn(TyCtxt<$tcx>, $K) -> $R,)*
- }
-
- impl<$tcx> Default for Providers<$tcx> {
- fn default() -> Self {
- $(fn $name<$tcx>(_: TyCtxt<$tcx>, key: $K) -> $R {
- bug!("`tcx.{}({:?})` unsupported by its crate",
- stringify!($name), key);
- })*
- Providers { $($name),* }
- }
- }
- };
-}
--- /dev/null
+For more information about how the query system works, see the [rustc dev guide].
+
+[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/query.html
--- /dev/null
+use crate::dep_graph::DepNodeIndex;
+use crate::ty::query::config::QueryContext;
+use crate::ty::query::plumbing::{QueryLookup, QueryState, QueryStateShard};
+
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sharded::Sharded;
+use std::default::Default;
+use std::hash::Hash;
+use std::marker::PhantomData;
+
+pub(crate) trait CacheSelector<CTX: QueryContext, K, V> {
+ type Cache: QueryCache<CTX, Key = K, Value = V>;
+}
+
+pub(crate) trait QueryCache<CTX: QueryContext>: Default {
+ type Key;
+ type Value;
+ type Sharded: Default;
+
+ /// Checks if the query is already computed and in the cache.
+ /// It returns the shard index and a lock guard to the shard,
+ /// which will be used if the query is not in the cache and we need
+ /// to compute it.
+ fn lookup<R, GetCache, OnHit, OnMiss>(
+ &self,
+ state: &QueryState<CTX, Self>,
+ get_cache: GetCache,
+ key: Self::Key,
+ // `on_hit` can be called while holding a lock to the query state shard.
+ on_hit: OnHit,
+ on_miss: OnMiss,
+ ) -> R
+ where
+ GetCache: for<'a> Fn(
+ &'a mut QueryStateShard<CTX, Self::Key, Self::Sharded>,
+ ) -> &'a mut Self::Sharded,
+ OnHit: FnOnce(&Self::Value, DepNodeIndex) -> R,
+ OnMiss: FnOnce(Self::Key, QueryLookup<'_, CTX, Self::Key, Self::Sharded>) -> R;
+
+ fn complete(
+ &self,
+ tcx: CTX,
+ lock_sharded_storage: &mut Self::Sharded,
+ key: Self::Key,
+ value: Self::Value,
+ index: DepNodeIndex,
+ );
+
+ fn iter<R, L>(
+ &self,
+ shards: &Sharded<L>,
+ get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
+ f: impl for<'a> FnOnce(
+ Box<dyn Iterator<Item = (&'a Self::Key, &'a Self::Value, DepNodeIndex)> + 'a>,
+ ) -> R,
+ ) -> R;
+}
+
+pub struct DefaultCacheSelector;
+
+impl<CTX: QueryContext, K: Eq + Hash, V: Clone> CacheSelector<CTX, K, V> for DefaultCacheSelector {
+ type Cache = DefaultCache<K, V>;
+}
+
+pub struct DefaultCache<K, V>(PhantomData<(K, V)>);
+
+impl<K, V> Default for DefaultCache<K, V> {
+ fn default() -> Self {
+ DefaultCache(PhantomData)
+ }
+}
+
+impl<CTX: QueryContext, K: Eq + Hash, V: Clone> QueryCache<CTX> for DefaultCache<K, V> {
+ type Key = K;
+ type Value = V;
+ type Sharded = FxHashMap<K, (V, DepNodeIndex)>;
+
+ #[inline(always)]
+ fn lookup<R, GetCache, OnHit, OnMiss>(
+ &self,
+ state: &QueryState<CTX, Self>,
+ get_cache: GetCache,
+ key: K,
+ on_hit: OnHit,
+ on_miss: OnMiss,
+ ) -> R
+ where
+ GetCache:
+ for<'a> Fn(&'a mut QueryStateShard<CTX, K, Self::Sharded>) -> &'a mut Self::Sharded,
+ OnHit: FnOnce(&V, DepNodeIndex) -> R,
+ OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R,
+ {
+ let mut lookup = state.get_lookup(&key);
+ let lock = &mut *lookup.lock;
+
+ let result = get_cache(lock).raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
+
+ if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) }
+ }
+
+ #[inline]
+ fn complete(
+ &self,
+ _: CTX,
+ lock_sharded_storage: &mut Self::Sharded,
+ key: K,
+ value: V,
+ index: DepNodeIndex,
+ ) {
+ lock_sharded_storage.insert(key, (value, index));
+ }
+
+ fn iter<R, L>(
+ &self,
+ shards: &Sharded<L>,
+ get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
+ f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
+ ) -> R {
+ let mut shards = shards.lock_shards();
+ let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
+ let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
+ f(Box::new(results))
+ }
+}
--- /dev/null
+//! Query configuration and description traits.
+
+use crate::dep_graph::SerializedDepNodeIndex;
+use crate::ty::query::caches::QueryCache;
+use crate::ty::query::job::{QueryJobId, QueryJobInfo};
+use crate::ty::query::plumbing::CycleError;
+use crate::ty::query::QueryState;
+use rustc_data_structures::profiling::ProfileCategory;
+use rustc_hir::def_id::DefId;
+
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stable_hasher::HashStable;
+use rustc_data_structures::sync::Lock;
+use rustc_data_structures::thin_vec::ThinVec;
+use rustc_errors::Diagnostic;
+use rustc_query_system::dep_graph::{DepContext, DepGraph, DepNode};
+use rustc_session::Session;
+use std::borrow::Cow;
+use std::fmt::Debug;
+use std::hash::Hash;
+
+pub trait QueryConfig<CTX> {
+ const NAME: &'static str;
+ const CATEGORY: ProfileCategory;
+
+ type Key: Eq + Hash + Clone + Debug;
+ type Value: Clone;
+}
+
+pub trait QueryContext: DepContext {
+ type Query: Clone + HashStable<Self::StableHashingContext>;
+
+ /// Access the session.
+ fn session(&self) -> &Session;
+
+ /// Get string representation from DefPath.
+ fn def_path_str(&self, def_id: DefId) -> String;
+
+ /// Access the DepGraph.
+ fn dep_graph(&self) -> &DepGraph<Self::DepKind>;
+
+ /// Get the query information from the TLS context.
+ fn read_query_job<R>(&self, op: impl FnOnce(Option<QueryJobId<Self::DepKind>>) -> R) -> R;
+
+ fn try_collect_active_jobs(
+ &self,
+ ) -> Option<FxHashMap<QueryJobId<Self::DepKind>, QueryJobInfo<Self>>>;
+
+ /// Executes a job by changing the `ImplicitCtxt` to point to the
+ /// new query job while it executes. It returns the diagnostics
+ /// captured during execution and the actual result.
+ fn start_query<R>(
+ &self,
+ token: QueryJobId<Self::DepKind>,
+ diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
+ compute: impl FnOnce(Self) -> R,
+ ) -> R;
+}
+
+pub(crate) trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> {
+ const ANON: bool;
+ const EVAL_ALWAYS: bool;
+ const DEP_KIND: CTX::DepKind;
+
+ type Cache: QueryCache<CTX, Key = Self::Key, Value = Self::Value>;
+
+ // Don't use this method to access query results, instead use the methods on TyCtxt
+ fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX, Self::Cache>;
+
+ fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind>;
+
+ // Don't use this method to compute query results, instead use the methods on TyCtxt
+ fn compute(tcx: CTX, key: Self::Key) -> Self::Value;
+
+ fn hash_result(
+ hcx: &mut CTX::StableHashingContext,
+ result: &Self::Value,
+ ) -> Option<Fingerprint>;
+
+ fn handle_cycle_error(tcx: CTX, error: CycleError<CTX::Query>) -> Self::Value;
+}
+
+pub(crate) trait QueryDescription<CTX: QueryContext>: QueryAccessors<CTX> {
+ fn describe(tcx: CTX, key: Self::Key) -> Cow<'static, str>;
+
+ #[inline]
+ fn cache_on_disk(_: CTX, _: Self::Key, _: Option<&Self::Value>) -> bool {
+ false
+ }
+
+ fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option<Self::Value> {
+ bug!("QueryDescription::load_from_disk() called for an unsupported query.")
+ }
+}
+
+impl<CTX: QueryContext, M> QueryDescription<CTX> for M
+where
+ M: QueryAccessors<CTX, Key = DefId>,
+{
+ default fn describe(tcx: CTX, def_id: DefId) -> Cow<'static, str> {
+ if !tcx.session().verbose() {
+ format!("processing `{}`", tcx.def_path_str(def_id)).into()
+ } else {
+ let name = ::std::any::type_name::<M>();
+ format!("processing {:?} with query `{}`", def_id, name).into()
+ }
+ }
+
+ default fn cache_on_disk(_: CTX, _: Self::Key, _: Option<&Self::Value>) -> bool {
+ false
+ }
+
+ default fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option<Self::Value> {
+ bug!("QueryDescription::load_from_disk() called for an unsupported query.")
+ }
+}
--- /dev/null
+use crate::ty::query::config::QueryContext;
+use crate::ty::query::plumbing::CycleError;
+#[cfg(parallel_compiler)]
+use crate::ty::tls;
+
+use rustc_data_structures::fx::FxHashMap;
+use rustc_query_system::dep_graph::DepContext;
+use rustc_span::Span;
+
+use std::convert::TryFrom;
+use std::marker::PhantomData;
+use std::num::NonZeroU32;
+
+#[cfg(parallel_compiler)]
+use {
+ parking_lot::{Condvar, Mutex},
+ rustc_data_structures::fx::FxHashSet,
+ rustc_data_structures::stable_hasher::{HashStable, StableHasher},
+ rustc_data_structures::sync::Lock,
+ rustc_data_structures::sync::Lrc,
+ rustc_data_structures::{jobserver, OnDrop},
+ rustc_rayon_core as rayon_core,
+ rustc_span::DUMMY_SP,
+ std::iter::FromIterator,
+ std::{mem, process, thread},
+};
+
+/// Represents a span and a query key.
+#[derive(Clone, Debug)]
+pub struct QueryInfo<Q> {
+ /// The span corresponding to the reason for which this query was required.
+ pub span: Span,
+ pub query: Q,
+}
+
+type QueryMap<CTX> = FxHashMap<QueryJobId<<CTX as DepContext>::DepKind>, QueryJobInfo<CTX>>;
+
+/// A value uniquely identifiying an active query job within a shard in the query cache.
+#[derive(Copy, Clone, Eq, PartialEq, Hash)]
+pub struct QueryShardJobId(pub NonZeroU32);
+
+/// A value uniquely identifiying an active query job.
+#[derive(Copy, Clone, Eq, PartialEq, Hash)]
+pub struct QueryJobId<K> {
+ /// Which job within a shard is this
+ pub job: QueryShardJobId,
+
+ /// In which shard is this job
+ pub shard: u16,
+
+ /// What kind of query this job is
+ pub kind: K,
+}
+
+impl<K: rustc_query_system::dep_graph::DepKind> QueryJobId<K> {
+ pub fn new(job: QueryShardJobId, shard: usize, kind: K) -> Self {
+ QueryJobId { job, shard: u16::try_from(shard).unwrap(), kind }
+ }
+
+ fn query<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> CTX::Query {
+ map.get(&self).unwrap().info.query.clone()
+ }
+
+ #[cfg(parallel_compiler)]
+ fn span<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> Span {
+ map.get(&self).unwrap().job.span
+ }
+
+ #[cfg(parallel_compiler)]
+ fn parent<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> Option<QueryJobId<K>> {
+ map.get(&self).unwrap().job.parent
+ }
+
+ #[cfg(parallel_compiler)]
+ fn latch<'a, CTX: QueryContext<DepKind = K>>(
+ self,
+ map: &'a QueryMap<CTX>,
+ ) -> Option<&'a QueryLatch<CTX>> {
+ map.get(&self).unwrap().job.latch.as_ref()
+ }
+}
+
+pub struct QueryJobInfo<CTX: QueryContext> {
+ pub info: QueryInfo<CTX::Query>,
+ pub job: QueryJob<CTX>,
+}
+
+/// Represents an active query job.
+#[derive(Clone)]
+pub struct QueryJob<CTX: QueryContext> {
+ pub id: QueryShardJobId,
+
+ /// The span corresponding to the reason for which this query was required.
+ pub span: Span,
+
+ /// The parent query job which created this job and is implicitly waiting on it.
+ pub parent: Option<QueryJobId<CTX::DepKind>>,
+
+ /// The latch that is used to wait on this job.
+ #[cfg(parallel_compiler)]
+ latch: Option<QueryLatch<CTX>>,
+
+ dummy: PhantomData<QueryLatch<CTX>>,
+}
+
+impl<CTX: QueryContext> QueryJob<CTX> {
+ /// Creates a new query job.
+ pub fn new(id: QueryShardJobId, span: Span, parent: Option<QueryJobId<CTX::DepKind>>) -> Self {
+ QueryJob {
+ id,
+ span,
+ parent,
+ #[cfg(parallel_compiler)]
+ latch: None,
+ dummy: PhantomData,
+ }
+ }
+
+ #[cfg(parallel_compiler)]
+ pub(super) fn latch(&mut self, _id: QueryJobId<CTX::DepKind>) -> QueryLatch<CTX> {
+ if self.latch.is_none() {
+ self.latch = Some(QueryLatch::new());
+ }
+ self.latch.as_ref().unwrap().clone()
+ }
+
+ #[cfg(not(parallel_compiler))]
+ pub(super) fn latch(&mut self, id: QueryJobId<CTX::DepKind>) -> QueryLatch<CTX> {
+ QueryLatch { id, dummy: PhantomData }
+ }
+
+ /// Signals to waiters that the query is complete.
+ ///
+ /// This does nothing for single threaded rustc,
+ /// as there are no concurrent jobs which could be waiting on us
+ pub fn signal_complete(self) {
+ #[cfg(parallel_compiler)]
+ self.latch.map(|latch| latch.set());
+ }
+}
+
+#[cfg(not(parallel_compiler))]
+#[derive(Clone)]
+pub(super) struct QueryLatch<CTX: QueryContext> {
+ id: QueryJobId<CTX::DepKind>,
+ dummy: PhantomData<CTX>,
+}
+
+#[cfg(not(parallel_compiler))]
+impl<CTX: QueryContext> QueryLatch<CTX> {
+ pub(super) fn find_cycle_in_stack(&self, tcx: CTX, span: Span) -> CycleError<CTX::Query> {
+ let query_map = tcx.try_collect_active_jobs().unwrap();
+
+ // Get the current executing query (waiter) and find the waitee amongst its parents
+ let mut current_job = tcx.read_query_job(|query| query);
+ let mut cycle = Vec::new();
+
+ while let Some(job) = current_job {
+ let info = query_map.get(&job).unwrap();
+ cycle.push(info.info.clone());
+
+ if job == self.id {
+ cycle.reverse();
+
+ // This is the end of the cycle
+ // The span entry we included was for the usage
+ // of the cycle itself, and not part of the cycle
+ // Replace it with the span which caused the cycle to form
+ cycle[0].span = span;
+ // Find out why the cycle itself was used
+ let usage = info
+ .job
+ .parent
+ .as_ref()
+ .map(|parent| (info.info.span, parent.query(&query_map)));
+ return CycleError { usage, cycle };
+ }
+
+ current_job = info.job.parent;
+ }
+
+ panic!("did not find a cycle")
+ }
+}
+
+#[cfg(parallel_compiler)]
+struct QueryWaiter<CTX: QueryContext> {
+ query: Option<QueryJobId<CTX::DepKind>>,
+ condvar: Condvar,
+ span: Span,
+ cycle: Lock<Option<CycleError<CTX::Query>>>,
+}
+
+#[cfg(parallel_compiler)]
+impl<CTX: QueryContext> QueryWaiter<CTX> {
+ fn notify(&self, registry: &rayon_core::Registry) {
+ rayon_core::mark_unblocked(registry);
+ self.condvar.notify_one();
+ }
+}
+
+#[cfg(parallel_compiler)]
+struct QueryLatchInfo<CTX: QueryContext> {
+ complete: bool,
+ waiters: Vec<Lrc<QueryWaiter<CTX>>>,
+}
+
+#[cfg(parallel_compiler)]
+#[derive(Clone)]
+pub(super) struct QueryLatch<CTX: QueryContext> {
+ info: Lrc<Mutex<QueryLatchInfo<CTX>>>,
+}
+
+#[cfg(parallel_compiler)]
+impl<CTX: QueryContext> QueryLatch<CTX> {
+ fn new() -> Self {
+ QueryLatch {
+ info: Lrc::new(Mutex::new(QueryLatchInfo { complete: false, waiters: Vec::new() })),
+ }
+ }
+}
+
+#[cfg(parallel_compiler)]
+impl<CTX: QueryContext> QueryLatch<CTX> {
+ /// Awaits for the query job to complete.
+ pub(super) fn wait_on(&self, tcx: CTX, span: Span) -> Result<(), CycleError<CTX::Query>> {
+ tcx.read_query_job(move |query| {
+ let waiter = Lrc::new(QueryWaiter {
+ query,
+ span,
+ cycle: Lock::new(None),
+ condvar: Condvar::new(),
+ });
+ self.wait_on_inner(&waiter);
+ // FIXME: Get rid of this lock. We have ownership of the QueryWaiter
+ // although another thread may still have a Lrc reference so we cannot
+ // use Lrc::get_mut
+ let mut cycle = waiter.cycle.lock();
+ match cycle.take() {
+ None => Ok(()),
+ Some(cycle) => Err(cycle),
+ }
+ })
+ }
+}
+
+#[cfg(parallel_compiler)]
+impl<CTX: QueryContext> QueryLatch<CTX> {
+ /// Awaits the caller on this latch by blocking the current thread.
+ fn wait_on_inner(&self, waiter: &Lrc<QueryWaiter<CTX>>) {
+ let mut info = self.info.lock();
+ if !info.complete {
+ // We push the waiter on to the `waiters` list. It can be accessed inside
+ // the `wait` call below, by 1) the `set` method or 2) by deadlock detection.
+ // Both of these will remove it from the `waiters` list before resuming
+ // this thread.
+ info.waiters.push(waiter.clone());
+
+ // If this detects a deadlock and the deadlock handler wants to resume this thread
+ // we have to be in the `wait` call. This is ensured by the deadlock handler
+ // getting the self.info lock.
+ rayon_core::mark_blocked();
+ jobserver::release_thread();
+ waiter.condvar.wait(&mut info);
+ // Release the lock before we potentially block in `acquire_thread`
+ mem::drop(info);
+ jobserver::acquire_thread();
+ }
+ }
+
+ /// Sets the latch and resumes all waiters on it
+ fn set(&self) {
+ let mut info = self.info.lock();
+ debug_assert!(!info.complete);
+ info.complete = true;
+ let registry = rayon_core::Registry::current();
+ for waiter in info.waiters.drain(..) {
+ waiter.notify(®istry);
+ }
+ }
+
+ /// Removes a single waiter from the list of waiters.
+ /// This is used to break query cycles.
+ fn extract_waiter(&self, waiter: usize) -> Lrc<QueryWaiter<CTX>> {
+ let mut info = self.info.lock();
+ debug_assert!(!info.complete);
+ // Remove the waiter from the list of waiters
+ info.waiters.remove(waiter)
+ }
+}
+
+/// A resumable waiter of a query. The usize is the index into waiters in the query's latch
+#[cfg(parallel_compiler)]
+type Waiter<K> = (QueryJobId<K>, usize);
+
+/// Visits all the non-resumable and resumable waiters of a query.
+/// Only waiters in a query are visited.
+/// `visit` is called for every waiter and is passed a query waiting on `query_ref`
+/// and a span indicating the reason the query waited on `query_ref`.
+/// If `visit` returns Some, this function returns.
+/// For visits of non-resumable waiters it returns the return value of `visit`.
+/// For visits of resumable waiters it returns Some(Some(Waiter)) which has the
+/// required information to resume the waiter.
+/// If all `visit` calls returns None, this function also returns None.
+#[cfg(parallel_compiler)]
+fn visit_waiters<CTX: QueryContext, F>(
+ query_map: &QueryMap<CTX>,
+ query: QueryJobId<CTX::DepKind>,
+ mut visit: F,
+) -> Option<Option<Waiter<CTX::DepKind>>>
+where
+ F: FnMut(Span, QueryJobId<CTX::DepKind>) -> Option<Option<Waiter<CTX::DepKind>>>,
+{
+ // Visit the parent query which is a non-resumable waiter since it's on the same stack
+ if let Some(parent) = query.parent(query_map) {
+ if let Some(cycle) = visit(query.span(query_map), parent) {
+ return Some(cycle);
+ }
+ }
+
+ // Visit the explicit waiters which use condvars and are resumable
+ if let Some(latch) = query.latch(query_map) {
+ for (i, waiter) in latch.info.lock().waiters.iter().enumerate() {
+ if let Some(waiter_query) = waiter.query {
+ if visit(waiter.span, waiter_query).is_some() {
+ // Return a value which indicates that this waiter can be resumed
+ return Some(Some((query, i)));
+ }
+ }
+ }
+ }
+
+ None
+}
+
+/// Look for query cycles by doing a depth first search starting at `query`.
+/// `span` is the reason for the `query` to execute. This is initially DUMMY_SP.
+/// If a cycle is detected, this initial value is replaced with the span causing
+/// the cycle.
+#[cfg(parallel_compiler)]
+fn cycle_check<CTX: QueryContext>(
+ query_map: &QueryMap<CTX>,
+ query: QueryJobId<CTX::DepKind>,
+ span: Span,
+ stack: &mut Vec<(Span, QueryJobId<CTX::DepKind>)>,
+ visited: &mut FxHashSet<QueryJobId<CTX::DepKind>>,
+) -> Option<Option<Waiter<CTX::DepKind>>> {
+ if !visited.insert(query) {
+ return if let Some(p) = stack.iter().position(|q| q.1 == query) {
+ // We detected a query cycle, fix up the initial span and return Some
+
+ // Remove previous stack entries
+ stack.drain(0..p);
+ // Replace the span for the first query with the cycle cause
+ stack[0].0 = span;
+ Some(None)
+ } else {
+ None
+ };
+ }
+
+ // Query marked as visited is added it to the stack
+ stack.push((span, query));
+
+ // Visit all the waiters
+ let r = visit_waiters(query_map, query, |span, successor| {
+ cycle_check(query_map, successor, span, stack, visited)
+ });
+
+ // Remove the entry in our stack if we didn't find a cycle
+ if r.is_none() {
+ stack.pop();
+ }
+
+ r
+}
+
+/// Finds out if there's a path to the compiler root (aka. code which isn't in a query)
+/// from `query` without going through any of the queries in `visited`.
+/// This is achieved with a depth first search.
+#[cfg(parallel_compiler)]
+fn connected_to_root<CTX: QueryContext>(
+ query_map: &QueryMap<CTX>,
+ query: QueryJobId<CTX::DepKind>,
+ visited: &mut FxHashSet<QueryJobId<CTX::DepKind>>,
+) -> bool {
+ // We already visited this or we're deliberately ignoring it
+ if !visited.insert(query) {
+ return false;
+ }
+
+ // This query is connected to the root (it has no query parent), return true
+ if query.parent(query_map).is_none() {
+ return true;
+ }
+
+ visit_waiters(query_map, query, |_, successor| {
+ connected_to_root(query_map, successor, visited).then_some(None)
+ })
+ .is_some()
+}
+
+// Deterministically pick an query from a list
+#[cfg(parallel_compiler)]
+fn pick_query<'a, CTX, T, F>(query_map: &QueryMap<CTX>, tcx: CTX, queries: &'a [T], f: F) -> &'a T
+where
+ CTX: QueryContext,
+ F: Fn(&T) -> (Span, QueryJobId<CTX::DepKind>),
+{
+ // Deterministically pick an entry point
+ // FIXME: Sort this instead
+ let mut hcx = tcx.create_stable_hashing_context();
+ queries
+ .iter()
+ .min_by_key(|v| {
+ let (span, query) = f(v);
+ let mut stable_hasher = StableHasher::new();
+ query.query(query_map).hash_stable(&mut hcx, &mut stable_hasher);
+ // Prefer entry points which have valid spans for nicer error messages
+ // We add an integer to the tuple ensuring that entry points
+ // with valid spans are picked first
+ let span_cmp = if span == DUMMY_SP { 1 } else { 0 };
+ (span_cmp, stable_hasher.finish::<u64>())
+ })
+ .unwrap()
+}
+
+/// Looks for query cycles starting from the last query in `jobs`.
+/// If a cycle is found, all queries in the cycle is removed from `jobs` and
+/// the function return true.
+/// If a cycle was not found, the starting query is removed from `jobs` and
+/// the function returns false.
+#[cfg(parallel_compiler)]
+fn remove_cycle<CTX: QueryContext>(
+ query_map: &QueryMap<CTX>,
+ jobs: &mut Vec<QueryJobId<CTX::DepKind>>,
+ wakelist: &mut Vec<Lrc<QueryWaiter<CTX>>>,
+ tcx: CTX,
+) -> bool {
+ let mut visited = FxHashSet::default();
+ let mut stack = Vec::new();
+ // Look for a cycle starting with the last query in `jobs`
+ if let Some(waiter) =
+ cycle_check(query_map, jobs.pop().unwrap(), DUMMY_SP, &mut stack, &mut visited)
+ {
+ // The stack is a vector of pairs of spans and queries; reverse it so that
+ // the earlier entries require later entries
+ let (mut spans, queries): (Vec<_>, Vec<_>) = stack.into_iter().rev().unzip();
+
+ // Shift the spans so that queries are matched with the span for their waitee
+ spans.rotate_right(1);
+
+ // Zip them back together
+ let mut stack: Vec<_> = spans.into_iter().zip(queries).collect();
+
+ // Remove the queries in our cycle from the list of jobs to look at
+ for r in &stack {
+ jobs.remove_item(&r.1);
+ }
+
+ // Find the queries in the cycle which are
+ // connected to queries outside the cycle
+ let entry_points = stack
+ .iter()
+ .filter_map(|&(span, query)| {
+ if query.parent(query_map).is_none() {
+ // This query is connected to the root (it has no query parent)
+ Some((span, query, None))
+ } else {
+ let mut waiters = Vec::new();
+ // Find all the direct waiters who lead to the root
+ visit_waiters(query_map, query, |span, waiter| {
+ // Mark all the other queries in the cycle as already visited
+ let mut visited = FxHashSet::from_iter(stack.iter().map(|q| q.1));
+
+ if connected_to_root(query_map, waiter, &mut visited) {
+ waiters.push((span, waiter));
+ }
+
+ None
+ });
+ if waiters.is_empty() {
+ None
+ } else {
+ // Deterministically pick one of the waiters to show to the user
+ let waiter = *pick_query(query_map, tcx, &waiters, |s| *s);
+ Some((span, query, Some(waiter)))
+ }
+ }
+ })
+ .collect::<Vec<(Span, QueryJobId<CTX::DepKind>, Option<(Span, QueryJobId<CTX::DepKind>)>)>>();
+
+ // Deterministically pick an entry point
+ let (_, entry_point, usage) = pick_query(query_map, tcx, &entry_points, |e| (e.0, e.1));
+
+ // Shift the stack so that our entry point is first
+ let entry_point_pos = stack.iter().position(|(_, query)| query == entry_point);
+ if let Some(pos) = entry_point_pos {
+ stack.rotate_left(pos);
+ }
+
+ let usage = usage.as_ref().map(|(span, query)| (*span, query.query(query_map)));
+
+ // Create the cycle error
+ let error = CycleError {
+ usage,
+ cycle: stack
+ .iter()
+ .map(|&(s, ref q)| QueryInfo { span: s, query: q.query(query_map) })
+ .collect(),
+ };
+
+ // We unwrap `waiter` here since there must always be one
+ // edge which is resumeable / waited using a query latch
+ let (waitee_query, waiter_idx) = waiter.unwrap();
+
+ // Extract the waiter we want to resume
+ let waiter = waitee_query.latch(query_map).unwrap().extract_waiter(waiter_idx);
+
+ // Set the cycle error so it will be picked up when resumed
+ *waiter.cycle.lock() = Some(error);
+
+ // Put the waiter on the list of things to resume
+ wakelist.push(waiter);
+
+ true
+ } else {
+ false
+ }
+}
+
+/// Creates a new thread and forwards information in thread locals to it.
+/// The new thread runs the deadlock handler.
+/// Must only be called when a deadlock is about to happen.
+#[cfg(parallel_compiler)]
+pub unsafe fn handle_deadlock() {
+ let registry = rayon_core::Registry::current();
+
+ let gcx_ptr = tls::GCX_PTR.with(|gcx_ptr| gcx_ptr as *const _);
+ let gcx_ptr = &*gcx_ptr;
+
+ let rustc_span_globals =
+ rustc_span::GLOBALS.with(|rustc_span_globals| rustc_span_globals as *const _);
+ let rustc_span_globals = &*rustc_span_globals;
+ let syntax_globals = rustc_ast::attr::GLOBALS.with(|syntax_globals| syntax_globals as *const _);
+ let syntax_globals = &*syntax_globals;
+ thread::spawn(move || {
+ tls::GCX_PTR.set(gcx_ptr, || {
+ rustc_ast::attr::GLOBALS.set(syntax_globals, || {
+ rustc_span::GLOBALS
+ .set(rustc_span_globals, || tls::with_global(|tcx| deadlock(tcx, ®istry)))
+ });
+ })
+ });
+}
+
+/// Detects query cycles by using depth first search over all active query jobs.
+/// If a query cycle is found it will break the cycle by finding an edge which
+/// uses a query latch and then resuming that waiter.
+/// There may be multiple cycles involved in a deadlock, so this searches
+/// all active queries for cycles before finally resuming all the waiters at once.
+#[cfg(parallel_compiler)]
+fn deadlock<CTX: QueryContext>(tcx: CTX, registry: &rayon_core::Registry) {
+ let on_panic = OnDrop(|| {
+ eprintln!("deadlock handler panicked, aborting process");
+ process::abort();
+ });
+
+ let mut wakelist = Vec::new();
+ let query_map = tcx.try_collect_active_jobs().unwrap();
+ let mut jobs: Vec<QueryJobId<CTX::DepKind>> = query_map.keys().cloned().collect();
+
+ let mut found_cycle = false;
+
+ while jobs.len() > 0 {
+ if remove_cycle(&query_map, &mut jobs, &mut wakelist, tcx) {
+ found_cycle = true;
+ }
+ }
+
+ // Check that a cycle was found. It is possible for a deadlock to occur without
+ // a query cycle if a query which can be waited on uses Rayon to do multithreading
+ // internally. Such a query (X) may be executing on 2 threads (A and B) and A may
+ // wait using Rayon on B. Rayon may then switch to executing another query (Y)
+ // which in turn will wait on X causing a deadlock. We have a false dependency from
+ // X to Y due to Rayon waiting and a true dependency from Y to X. The algorithm here
+ // only considers the true dependency and won't detect a cycle.
+ assert!(found_cycle);
+
+ // FIXME: Ensure this won't cause a deadlock before we return
+ for waiter in wakelist.into_iter() {
+ waiter.notify(registry);
+ }
+
+ on_panic.disable();
+}
--- /dev/null
+use crate::dep_graph::{self, DepConstructor, DepNode, DepNodeParams};
+use crate::hir::exports::Export;
+use crate::hir::map;
+use crate::infer::canonical::{self, Canonical};
+use crate::lint::LintLevelMap;
+use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
+use crate::middle::cstore::{CrateSource, DepKind, NativeLibraryKind};
+use crate::middle::cstore::{ExternCrate, ForeignModule, LinkagePreference, NativeLibrary};
+use crate::middle::exported_symbols::{ExportedSymbol, SymbolExportLevel};
+use crate::middle::lang_items::{LangItem, LanguageItems};
+use crate::middle::lib_features::LibFeatures;
+use crate::middle::privacy::AccessLevels;
+use crate::middle::region;
+use crate::middle::resolve_lifetime::{ObjectLifetimeDefault, Region, ResolveLifetimes};
+use crate::middle::stability::{self, DeprecationEntry};
+use crate::mir;
+use crate::mir::interpret::GlobalId;
+use crate::mir::interpret::{ConstEvalRawResult, ConstEvalResult, ConstValue};
+use crate::mir::interpret::{LitToConstError, LitToConstInput};
+use crate::mir::mono::CodegenUnit;
+use crate::traits::query::{
+ CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal,
+ CanonicalTypeOpAscribeUserTypeGoal, CanonicalTypeOpEqGoal, CanonicalTypeOpNormalizeGoal,
+ CanonicalTypeOpProvePredicateGoal, CanonicalTypeOpSubtypeGoal, NoSolution,
+};
+use crate::traits::query::{
+ DropckOutlivesResult, DtorckConstraint, MethodAutoderefStepsResult, NormalizationResult,
+ OutlivesBound,
+};
+use crate::traits::specialization_graph;
+use crate::traits::Clauses;
+use crate::traits::{self, Vtable};
+use crate::ty::steal::Steal;
+use crate::ty::subst::{GenericArg, SubstsRef};
+use crate::ty::util::AlwaysRequiresDrop;
+use crate::ty::{self, AdtSizedConstraint, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt};
+use crate::util::common::ErrorReported;
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
+use rustc_data_structures::profiling::ProfileCategory::*;
+use rustc_data_structures::stable_hasher::StableVec;
+use rustc_data_structures::svh::Svh;
+use rustc_data_structures::sync::Lrc;
+use rustc_hir as hir;
+use rustc_hir::def::DefKind;
+use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId};
+use rustc_hir::{Crate, HirIdSet, ItemLocalId, TraitCandidate};
+use rustc_index::vec::IndexVec;
+use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion};
+use rustc_session::CrateDisambiguator;
+use rustc_target::spec::PanicStrategy;
+
+use rustc_ast::ast;
+use rustc_attr as attr;
+use rustc_span::symbol::Symbol;
+use rustc_span::{Span, DUMMY_SP};
+use std::borrow::Cow;
+use std::collections::BTreeMap;
+use std::ops::Deref;
+use std::sync::Arc;
+
+#[macro_use]
+mod plumbing;
+pub(crate) use self::plumbing::CycleError;
+use self::plumbing::*;
+
+mod stats;
+pub use self::stats::print_stats;
+
+mod job;
+#[cfg(parallel_compiler)]
+pub use self::job::handle_deadlock;
+use self::job::QueryJobInfo;
+pub use self::job::{QueryInfo, QueryJob, QueryJobId};
+
+mod keys;
+use self::keys::Key;
+
+mod values;
+use self::values::Value;
+
+mod caches;
+use self::caches::CacheSelector;
+
+mod config;
+use self::config::QueryAccessors;
+pub use self::config::QueryConfig;
+pub(crate) use self::config::QueryDescription;
+
+mod on_disk_cache;
+pub use self::on_disk_cache::OnDiskCache;
+
+mod profiling_support;
+pub use self::profiling_support::{IntoSelfProfilingString, QueryKeyStringBuilder};
+
+// Each of these queries corresponds to a function pointer field in the
+// `Providers` struct for requesting a value of that type, and a method
+// on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
+// which memoizes and does dep-graph tracking, wrapping around the actual
+// `Providers` that the driver creates (using several `rustc_*` crates).
+//
+// The result type of each query must implement `Clone`, and additionally
+// `ty::query::values::Value`, which produces an appropriate placeholder
+// (error) value if the query resulted in a query cycle.
+// Queries marked with `fatal_cycle` do not need the latter implementation,
+// as they will raise an fatal error on query cycles instead.
+
+rustc_query_append! { [define_queries!][<'tcx>] }
+
+/// The red/green evaluation system will try to mark a specific DepNode in the
+/// dependency graph as green by recursively trying to mark the dependencies of
+/// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode`
+/// where we don't know if it is red or green and we therefore actually have
+/// to recompute its value in order to find out. Since the only piece of
+/// information that we have at that point is the `DepNode` we are trying to
+/// re-evaluate, we need some way to re-run a query from just that. This is what
+/// `force_from_dep_node()` implements.
+///
+/// In the general case, a `DepNode` consists of a `DepKind` and an opaque
+/// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint
+/// is usually constructed by computing a stable hash of the query-key that the
+/// `DepNode` corresponds to. Consequently, it is not in general possible to go
+/// back from hash to query-key (since hash functions are not reversible). For
+/// this reason `force_from_dep_node()` is expected to fail from time to time
+/// because we just cannot find out, from the `DepNode` alone, what the
+/// corresponding query-key is and therefore cannot re-run the query.
+///
+/// The system deals with this case letting `try_mark_green` fail which forces
+/// the root query to be re-evaluated.
+///
+/// Now, if `force_from_dep_node()` would always fail, it would be pretty useless.
+/// Fortunately, we can use some contextual information that will allow us to
+/// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we
+/// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a
+/// valid `DefPathHash`. Since we also always build a huge table that maps every
+/// `DefPathHash` in the current codebase to the corresponding `DefId`, we have
+/// everything we need to re-run the query.
+///
+/// Take the `mir_validated` query as an example. Like many other queries, it
+/// just has a single parameter: the `DefId` of the item it will compute the
+/// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode`
+/// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode`
+/// is actually a `DefPathHash`, and can therefore just look up the corresponding
+/// `DefId` in `tcx.def_path_hash_to_def_id`.
+///
+/// When you implement a new query, it will likely have a corresponding new
+/// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As
+/// a rule of thumb, if your query takes a `DefId` or `LocalDefId` as sole parameter,
+/// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
+/// add it to the "We don't have enough information to reconstruct..." group in
+/// the match below.
+pub fn force_from_dep_node<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> bool {
+ // We must avoid ever having to call `force_from_dep_node()` for a
+ // `DepNode::codegen_unit`:
+ // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
+ // would always end up having to evaluate the first caller of the
+ // `codegen_unit` query that *is* reconstructible. This might very well be
+ // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
+ // to re-trigger calling the `codegen_unit` query with the right key. At
+ // that point we would already have re-done all the work we are trying to
+ // avoid doing in the first place.
+ // The solution is simple: Just explicitly call the `codegen_unit` query for
+ // each CGU, right after partitioning. This way `try_mark_green` will always
+ // hit the cache instead of having to go through `force_from_dep_node`.
+ // This assertion makes sure, we actually keep applying the solution above.
+ debug_assert!(
+ dep_node.kind != crate::dep_graph::DepKind::codegen_unit,
+ "calling force_from_dep_node() on DepKind::codegen_unit"
+ );
+
+ if !dep_node.kind.can_reconstruct_query_key() {
+ return false;
+ }
+
+ rustc_dep_node_force!([dep_node, tcx]
+ // These are inputs that are expected to be pre-allocated and that
+ // should therefore always be red or green already.
+ crate::dep_graph::DepKind::CrateMetadata |
+
+ // These are anonymous nodes.
+ crate::dep_graph::DepKind::TraitSelect |
+
+ // We don't have enough information to reconstruct the query key of
+ // these.
+ crate::dep_graph::DepKind::CompileCodegenUnit => {
+ bug!("force_from_dep_node: encountered {:?}", dep_node)
+ }
+ );
+
+ false
+}
+
+pub(crate) fn try_load_from_on_disk_cache<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) {
+ rustc_dep_node_try_load_from_on_disk_cache!(dep_node, tcx)
+}
--- /dev/null
+//! The implementation of the query system itself. This defines the macros that
+//! generate the actual methods on tcx which find and execute the provider,
+//! manage the caches, and so forth.
+
+use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
+use crate::ty::query::caches::QueryCache;
+use crate::ty::query::config::{QueryContext, QueryDescription};
+use crate::ty::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId};
+use crate::ty::query::Query;
+use crate::ty::tls::{self, ImplicitCtxt};
+use crate::ty::{self, TyCtxt};
+
+#[cfg(not(parallel_compiler))]
+use rustc_data_structures::cold_path;
+use rustc_data_structures::fx::{FxHashMap, FxHasher};
+use rustc_data_structures::sharded::Sharded;
+use rustc_data_structures::sync::{Lock, LockGuard};
+use rustc_data_structures::thin_vec::ThinVec;
+use rustc_errors::{struct_span_err, Diagnostic, DiagnosticBuilder, FatalError, Handler, Level};
+use rustc_query_system::dep_graph::{DepContext, DepGraph, DepKind, DepNode};
+use rustc_query_system::HashStableContextProvider;
+use rustc_session::Session;
+use rustc_span::def_id::DefId;
+use rustc_span::source_map::DUMMY_SP;
+use rustc_span::Span;
+use std::collections::hash_map::Entry;
+use std::convert::TryFrom;
+use std::fmt::Debug;
+use std::hash::{Hash, Hasher};
+use std::mem;
+use std::num::NonZeroU32;
+use std::ptr;
+#[cfg(debug_assertions)]
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+pub(crate) struct QueryStateShard<CTX: QueryContext, K, C> {
+ cache: C,
+ active: FxHashMap<K, QueryResult<CTX>>,
+
+ /// Used to generate unique ids for active jobs.
+ jobs: u32,
+}
+
+impl<CTX: QueryContext, K, C> QueryStateShard<CTX, K, C> {
+ fn get_cache(&mut self) -> &mut C {
+ &mut self.cache
+ }
+}
+
+impl<CTX: QueryContext, K, C: Default> Default for QueryStateShard<CTX, K, C> {
+ fn default() -> QueryStateShard<CTX, K, C> {
+ QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 }
+ }
+}
+
+pub(crate) struct QueryState<CTX: QueryContext, C: QueryCache<CTX>> {
+ cache: C,
+ shards: Sharded<QueryStateShard<CTX, C::Key, C::Sharded>>,
+ #[cfg(debug_assertions)]
+ pub(super) cache_hits: AtomicUsize,
+}
+
+impl<CTX: QueryContext, C: QueryCache<CTX>> QueryState<CTX, C> {
+ pub(super) fn get_lookup<K2: Hash>(
+ &'tcx self,
+ key: &K2,
+ ) -> QueryLookup<'tcx, CTX, C::Key, C::Sharded> {
+ // We compute the key's hash once and then use it for both the
+ // shard lookup and the hashmap lookup. This relies on the fact
+ // that both of them use `FxHasher`.
+ let mut hasher = FxHasher::default();
+ key.hash(&mut hasher);
+ let key_hash = hasher.finish();
+
+ let shard = self.shards.get_shard_index_by_hash(key_hash);
+ let lock = self.shards.get_shard_by_index(shard).lock();
+ QueryLookup { key_hash, shard, lock }
+ }
+}
+
+/// Indicates the state of a query for a given key in a query map.
+enum QueryResult<CTX: QueryContext> {
+ /// An already executing query. The query job can be used to await for its completion.
+ Started(QueryJob<CTX>),
+
+ /// The query panicked. Queries trying to wait on this will raise a fatal error which will
+ /// silently panic.
+ Poisoned,
+}
+
+impl<CTX: QueryContext, C: QueryCache<CTX>> QueryState<CTX, C> {
+ pub(super) fn iter_results<R>(
+ &self,
+ f: impl for<'a> FnOnce(
+ Box<dyn Iterator<Item = (&'a C::Key, &'a C::Value, DepNodeIndex)> + 'a>,
+ ) -> R,
+ ) -> R {
+ self.cache.iter(&self.shards, |shard| &mut shard.cache, f)
+ }
+ pub(super) fn all_inactive(&self) -> bool {
+ let shards = self.shards.lock_shards();
+ shards.iter().all(|shard| shard.active.is_empty())
+ }
+
+ pub(super) fn try_collect_active_jobs(
+ &self,
+ kind: CTX::DepKind,
+ make_query: fn(C::Key) -> CTX::Query,
+ jobs: &mut FxHashMap<QueryJobId<CTX::DepKind>, QueryJobInfo<CTX>>,
+ ) -> Option<()>
+ where
+ C::Key: Clone,
+ {
+ // We use try_lock_shards here since we are called from the
+ // deadlock handler, and this shouldn't be locked.
+ let shards = self.shards.try_lock_shards()?;
+ let shards = shards.iter().enumerate();
+ jobs.extend(shards.flat_map(|(shard_id, shard)| {
+ shard.active.iter().filter_map(move |(k, v)| {
+ if let QueryResult::Started(ref job) = *v {
+ let id =
+ QueryJobId { job: job.id, shard: u16::try_from(shard_id).unwrap(), kind };
+ let info = QueryInfo { span: job.span, query: make_query(k.clone()) };
+ Some((id, QueryJobInfo { info, job: job.clone() }))
+ } else {
+ None
+ }
+ })
+ }));
+
+ Some(())
+ }
+}
+
+impl<CTX: QueryContext, C: QueryCache<CTX>> Default for QueryState<CTX, C> {
+ fn default() -> QueryState<CTX, C> {
+ QueryState {
+ cache: C::default(),
+ shards: Default::default(),
+ #[cfg(debug_assertions)]
+ cache_hits: AtomicUsize::new(0),
+ }
+ }
+}
+
+/// Values used when checking a query cache which can be reused on a cache-miss to execute the query.
+pub(crate) struct QueryLookup<'tcx, CTX: QueryContext, K, C> {
+ pub(super) key_hash: u64,
+ shard: usize,
+ pub(super) lock: LockGuard<'tcx, QueryStateShard<CTX, K, C>>,
+}
+
+/// A type representing the responsibility to execute the job in the `job` field.
+/// This will poison the relevant query if dropped.
+struct JobOwner<'tcx, CTX: QueryContext, C>
+where
+ C: QueryCache<CTX>,
+ C::Key: Eq + Hash + Clone + Debug,
+ C::Value: Clone,
+{
+ state: &'tcx QueryState<CTX, C>,
+ key: C::Key,
+ id: QueryJobId<CTX::DepKind>,
+}
+
+impl<'tcx, CTX: QueryContext, C> JobOwner<'tcx, CTX, C>
+where
+ C: QueryCache<CTX>,
+ C::Key: Eq + Hash + Clone + Debug,
+ C::Value: Clone,
+{
+ /// Either gets a `JobOwner` corresponding the query, allowing us to
+ /// start executing the query, or returns with the result of the query.
+ /// This function assumes that `try_get_cached` is already called and returned `lookup`.
+ /// If the query is executing elsewhere, this will wait for it and return the result.
+ /// If the query panicked, this will silently panic.
+ ///
+ /// This function is inlined because that results in a noticeable speed-up
+ /// for some compile-time benchmarks.
+ #[inline(always)]
+ fn try_start<'a, 'b, Q, K>(
+ tcx: CTX,
+ span: Span,
+ key: &C::Key,
+ mut lookup: QueryLookup<'a, CTX, C::Key, C::Sharded>,
+ ) -> TryGetJob<'b, CTX, C>
+ where
+ K: DepKind,
+ Q: QueryDescription<CTX, Key = C::Key, Value = C::Value, Cache = C>,
+ CTX: QueryContext<DepKind = K>,
+ {
+ let lock = &mut *lookup.lock;
+
+ let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) {
+ Entry::Occupied(mut entry) => {
+ match entry.get_mut() {
+ QueryResult::Started(job) => {
+ // For parallel queries, we'll block and wait until the query running
+ // in another thread has completed. Record how long we wait in the
+ // self-profiler.
+ let _query_blocked_prof_timer = if cfg!(parallel_compiler) {
+ Some(tcx.profiler().query_blocked())
+ } else {
+ None
+ };
+
+ // Create the id of the job we're waiting for
+ let id = QueryJobId::new(job.id, lookup.shard, Q::DEP_KIND);
+
+ (job.latch(id), _query_blocked_prof_timer)
+ }
+ QueryResult::Poisoned => FatalError.raise(),
+ }
+ }
+ Entry::Vacant(entry) => {
+ // No job entry for this query. Return a new one to be started later.
+
+ // Generate an id unique within this shard.
+ let id = lock.jobs.checked_add(1).unwrap();
+ lock.jobs = id;
+ let id = QueryShardJobId(NonZeroU32::new(id).unwrap());
+
+ let global_id = QueryJobId::new(id, lookup.shard, Q::DEP_KIND);
+
+ let job = tcx.read_query_job(|query| QueryJob::new(id, span, query));
+
+ entry.insert(QueryResult::Started(job));
+
+ let owner =
+ JobOwner { state: Q::query_state(tcx), id: global_id, key: (*key).clone() };
+ return TryGetJob::NotYetStarted(owner);
+ }
+ };
+ mem::drop(lookup.lock);
+
+ // If we are single-threaded we know that we have cycle error,
+ // so we just return the error.
+ #[cfg(not(parallel_compiler))]
+ return TryGetJob::Cycle(cold_path(|| {
+ Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span))
+ }));
+
+ // With parallel queries we might just have to wait on some other
+ // thread.
+ #[cfg(parallel_compiler)]
+ {
+ let result = latch.wait_on(tcx, span);
+
+ if let Err(cycle) = result {
+ return TryGetJob::Cycle(Q::handle_cycle_error(tcx, cycle));
+ }
+
+ let cached = try_get_cached(
+ tcx,
+ Q::query_state(tcx),
+ (*key).clone(),
+ |value, index| (value.clone(), index),
+ |_, _| panic!("value must be in cache after waiting"),
+ );
+
+ if let Some(prof_timer) = _query_blocked_prof_timer.take() {
+ prof_timer.finish_with_query_invocation_id(cached.1.into());
+ }
+
+ return TryGetJob::JobCompleted(cached);
+ }
+ }
+
+ /// Completes the query by updating the query cache with the `result`,
+ /// signals the waiter and forgets the JobOwner, so it won't poison the query
+ #[inline(always)]
+ fn complete(self, tcx: CTX, result: &C::Value, dep_node_index: DepNodeIndex) {
+ // We can move out of `self` here because we `mem::forget` it below
+ let key = unsafe { ptr::read(&self.key) };
+ let state = self.state;
+
+ // Forget ourself so our destructor won't poison the query
+ mem::forget(self);
+
+ let job = {
+ let result = result.clone();
+ let mut lock = state.shards.get_shard_by_value(&key).lock();
+ let job = match lock.active.remove(&key).unwrap() {
+ QueryResult::Started(job) => job,
+ QueryResult::Poisoned => panic!(),
+ };
+ state.cache.complete(tcx, &mut lock.cache, key, result, dep_node_index);
+ job
+ };
+
+ job.signal_complete();
+ }
+}
+
+#[inline(always)]
+fn with_diagnostics<F, R>(f: F) -> (R, ThinVec<Diagnostic>)
+where
+ F: FnOnce(Option<&Lock<ThinVec<Diagnostic>>>) -> R,
+{
+ let diagnostics = Lock::new(ThinVec::new());
+ let result = f(Some(&diagnostics));
+ (result, diagnostics.into_inner())
+}
+
+impl<'tcx, CTX: QueryContext, C: QueryCache<CTX>> Drop for JobOwner<'tcx, CTX, C>
+where
+ C::Key: Eq + Hash + Clone + Debug,
+ C::Value: Clone,
+{
+ #[inline(never)]
+ #[cold]
+ fn drop(&mut self) {
+ // Poison the query so jobs waiting on it panic.
+ let state = self.state;
+ let shard = state.shards.get_shard_by_value(&self.key);
+ let job = {
+ let mut shard = shard.lock();
+ let job = match shard.active.remove(&self.key).unwrap() {
+ QueryResult::Started(job) => job,
+ QueryResult::Poisoned => panic!(),
+ };
+ shard.active.insert(self.key.clone(), QueryResult::Poisoned);
+ job
+ };
+ // Also signal the completion of the job, so waiters
+ // will continue execution.
+ job.signal_complete();
+ }
+}
+
+#[derive(Clone)]
+pub(crate) struct CycleError<Q> {
+ /// The query and related span that uses the cycle.
+ pub(super) usage: Option<(Span, Q)>,
+ pub(super) cycle: Vec<QueryInfo<Q>>,
+}
+
+/// The result of `try_start`.
+enum TryGetJob<'tcx, CTX: QueryContext, C: QueryCache<CTX>>
+where
+ C::Key: Eq + Hash + Clone + Debug,
+ C::Value: Clone,
+{
+ /// The query is not yet started. Contains a guard to the cache eventually used to start it.
+ NotYetStarted(JobOwner<'tcx, CTX, C>),
+
+ /// The query was already completed.
+ /// Returns the result of the query and its dep-node index
+ /// if it succeeded or a cycle error if it failed.
+ #[cfg(parallel_compiler)]
+ JobCompleted((C::Value, DepNodeIndex)),
+
+ /// Trying to execute the query resulted in a cycle.
+ Cycle(C::Value),
+}
+
+impl QueryContext for TyCtxt<'tcx> {
+ type Query = Query<'tcx>;
+
+ fn session(&self) -> &Session {
+ &self.sess
+ }
+
+ fn def_path_str(&self, def_id: DefId) -> String {
+ TyCtxt::def_path_str(*self, def_id)
+ }
+
+ fn dep_graph(&self) -> &DepGraph<Self::DepKind> {
+ &self.dep_graph
+ }
+
+ fn read_query_job<R>(&self, op: impl FnOnce(Option<QueryJobId<Self::DepKind>>) -> R) -> R {
+ tls::with_related_context(*self, move |icx| op(icx.query))
+ }
+
+ fn try_collect_active_jobs(
+ &self,
+ ) -> Option<FxHashMap<QueryJobId<Self::DepKind>, QueryJobInfo<Self>>> {
+ self.queries.try_collect_active_jobs()
+ }
+
+ /// Executes a job by changing the `ImplicitCtxt` to point to the
+ /// new query job while it executes. It returns the diagnostics
+ /// captured during execution and the actual result.
+ #[inline(always)]
+ fn start_query<R>(
+ &self,
+ token: QueryJobId<Self::DepKind>,
+ diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
+ compute: impl FnOnce(Self) -> R,
+ ) -> R {
+ // The `TyCtxt` stored in TLS has the same global interner lifetime
+ // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
+ // when accessing the `ImplicitCtxt`.
+ tls::with_related_context(*self, move |current_icx| {
+ // Update the `ImplicitCtxt` to point to our new query job.
+ let new_icx = ImplicitCtxt {
+ tcx: *self,
+ query: Some(token),
+ diagnostics,
+ layout_depth: current_icx.layout_depth,
+ task_deps: current_icx.task_deps,
+ };
+
+ // Use the `ImplicitCtxt` while we execute the query.
+ tls::enter_context(&new_icx, |_| compute(*self))
+ })
+ }
+}
+
+impl<'tcx> TyCtxt<'tcx> {
+ #[inline(never)]
+ #[cold]
+ pub(super) fn report_cycle(
+ self,
+ CycleError { usage, cycle: stack }: CycleError<Query<'tcx>>,
+ ) -> DiagnosticBuilder<'tcx> {
+ assert!(!stack.is_empty());
+
+ let fix_span = |span: Span, query: &Query<'tcx>| {
+ self.sess.source_map().guess_head_span(query.default_span(self, span))
+ };
+
+ // Disable naming impls with types in this path, since that
+ // sometimes cycles itself, leading to extra cycle errors.
+ // (And cycle errors around impls tend to occur during the
+ // collect/coherence phases anyhow.)
+ ty::print::with_forced_impl_filename_line(|| {
+ let span = fix_span(stack[1 % stack.len()].span, &stack[0].query);
+ let mut err = struct_span_err!(
+ self.sess,
+ span,
+ E0391,
+ "cycle detected when {}",
+ stack[0].query.describe(self)
+ );
+
+ for i in 1..stack.len() {
+ let query = &stack[i].query;
+ let span = fix_span(stack[(i + 1) % stack.len()].span, query);
+ err.span_note(span, &format!("...which requires {}...", query.describe(self)));
+ }
+
+ err.note(&format!(
+ "...which again requires {}, completing the cycle",
+ stack[0].query.describe(self)
+ ));
+
+ if let Some((span, query)) = usage {
+ err.span_note(
+ fix_span(span, &query),
+ &format!("cycle used when {}", query.describe(self)),
+ );
+ }
+
+ err
+ })
+ }
+
+ pub fn try_print_query_stack(handler: &Handler) {
+ eprintln!("query stack during panic:");
+
+ // Be careful reyling on global state here: this code is called from
+ // a panic hook, which means that the global `Handler` may be in a weird
+ // state if it was responsible for triggering the panic.
+ tls::with_context_opt(|icx| {
+ if let Some(icx) = icx {
+ let query_map = icx.tcx.queries.try_collect_active_jobs();
+
+ let mut current_query = icx.query;
+ let mut i = 0;
+
+ while let Some(query) = current_query {
+ let query_info =
+ if let Some(info) = query_map.as_ref().and_then(|map| map.get(&query)) {
+ info
+ } else {
+ break;
+ };
+ let mut diag = Diagnostic::new(
+ Level::FailureNote,
+ &format!(
+ "#{} [{}] {}",
+ i,
+ query_info.info.query.name(),
+ query_info.info.query.describe(icx.tcx)
+ ),
+ );
+ diag.span =
+ icx.tcx.sess.source_map().guess_head_span(query_info.info.span).into();
+ handler.force_print_diagnostic(diag);
+
+ current_query = query_info.job.parent;
+ i += 1;
+ }
+ }
+ });
+
+ eprintln!("end of query stack");
+ }
+}
+
+ /// Checks if the query is already computed and in the cache.
+ /// It returns the shard index and a lock guard to the shard,
+ /// which will be used if the query is not in the cache and we need
+ /// to compute it.
+ #[inline(always)]
+ fn try_get_cached<CTX, C, R, OnHit, OnMiss>(
+ tcx: CTX,
+ state: &QueryState<CTX, C>,
+ key: C::Key,
+ // `on_hit` can be called while holding a lock to the query cache
+ on_hit: OnHit,
+ on_miss: OnMiss,
+ ) -> R
+ where
+ C: QueryCache<CTX>,
+ CTX: QueryContext,
+ OnHit: FnOnce(&C::Value, DepNodeIndex) -> R,
+ OnMiss: FnOnce(C::Key, QueryLookup<'_, CTX, C::Key, C::Sharded>) -> R,
+ {
+ state.cache.lookup(
+ state,
+ QueryStateShard::<CTX, C::Key, C::Sharded>::get_cache,
+ key,
+ |value, index| {
+ if unlikely!(tcx.profiler().enabled()) {
+ tcx.profiler().query_cache_hit(index.into());
+ }
+ #[cfg(debug_assertions)]
+ {
+ state.cache_hits.fetch_add(1, Ordering::Relaxed);
+ }
+ on_hit(value, index)
+ },
+ on_miss,
+ )
+ }
+
+ #[inline(always)]
+ fn try_execute_query<Q, CTX, K>(
+ tcx: CTX,
+ span: Span,
+ key: Q::Key,
+ lookup: QueryLookup<
+ '_,
+ CTX,
+ Q::Key,
+ <Q::Cache as QueryCache<CTX>>::Sharded,
+ >,
+ ) -> Q::Value
+ where
+ Q: QueryDescription<CTX>,
+ CTX: QueryContext<DepKind = K>,
+ CTX: HashStableContextProvider<<CTX as DepContext>::StableHashingContext>,
+ K: DepKind,
+ {
+ let job = match JobOwner::try_start::<Q, _>(tcx, span, &key, lookup) {
+ TryGetJob::NotYetStarted(job) => job,
+ TryGetJob::Cycle(result) => return result,
+ #[cfg(parallel_compiler)]
+ TryGetJob::JobCompleted((v, index)) => {
+ tcx.dep_graph().read_index(index);
+ return v;
+ }
+ };
+
+ // Fast path for when incr. comp. is off. `to_dep_node` is
+ // expensive for some `DepKind`s.
+ if !tcx.dep_graph().is_fully_enabled() {
+ let null_dep_node = DepNode::new_no_params(DepKind::NULL);
+ return force_query_with_job::<Q, _, _>(tcx, key, job, null_dep_node).0;
+ }
+
+ if Q::ANON {
+ let prof_timer = tcx.profiler().query_provider();
+
+ let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
+ tcx.start_query(job.id, diagnostics, |tcx| {
+ tcx.dep_graph().with_anon_task(Q::DEP_KIND, || Q::compute(tcx, key))
+ })
+ });
+
+ prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+
+ tcx.dep_graph().read_index(dep_node_index);
+
+ if unlikely!(!diagnostics.is_empty()) {
+ tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics);
+ }
+
+ job.complete(tcx, &result, dep_node_index);
+
+ return result;
+ }
+
+ let dep_node = Q::to_dep_node(tcx, &key);
+
+ if !Q::EVAL_ALWAYS {
+ // The diagnostics for this query will be
+ // promoted to the current session during
+ // `try_mark_green()`, so we can ignore them here.
+ let loaded = tcx.start_query(job.id, None, |tcx| {
+ let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node);
+ marked.map(|(prev_dep_node_index, dep_node_index)| {
+ (
+ load_from_disk_and_cache_in_memory::<Q, _>(
+ tcx,
+ key.clone(),
+ prev_dep_node_index,
+ dep_node_index,
+ &dep_node,
+ ),
+ dep_node_index,
+ )
+ })
+ });
+ if let Some((result, dep_node_index)) = loaded {
+ job.complete(tcx, &result, dep_node_index);
+ return result;
+ }
+ }
+
+ let (result, dep_node_index) = force_query_with_job::<Q, _, _>(tcx, key, job, dep_node);
+ tcx.dep_graph().read_index(dep_node_index);
+ result
+ }
+
+ fn load_from_disk_and_cache_in_memory<Q, CTX>(
+ tcx: CTX,
+ key: Q::Key,
+ prev_dep_node_index: SerializedDepNodeIndex,
+ dep_node_index: DepNodeIndex,
+ dep_node: &DepNode<CTX::DepKind>,
+ ) -> Q::Value
+ where
+ CTX: QueryContext,
+ Q: QueryDescription<CTX>,
+ {
+ // Note this function can be called concurrently from the same query
+ // We must ensure that this is handled correctly.
+
+ debug_assert!(tcx.dep_graph().is_green(dep_node));
+
+ // First we try to load the result from the on-disk cache.
+ let result = if Q::cache_on_disk(tcx, key.clone(), None) {
+ let prof_timer = tcx.profiler().incr_cache_loading();
+ let result = Q::try_load_from_disk(tcx, prev_dep_node_index);
+ prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+
+ // We always expect to find a cached result for things that
+ // can be forced from `DepNode`.
+ debug_assert!(
+ !dep_node.kind.can_reconstruct_query_key() || result.is_some(),
+ "missing on-disk cache entry for {:?}",
+ dep_node
+ );
+ result
+ } else {
+ // Some things are never cached on disk.
+ None
+ };
+
+ let result = if let Some(result) = result {
+ result
+ } else {
+ // We could not load a result from the on-disk cache, so
+ // recompute.
+ let prof_timer = tcx.profiler().query_provider();
+
+ // The dep-graph for this computation is already in-place.
+ let result = tcx.dep_graph().with_ignore(|| Q::compute(tcx, key));
+
+ prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+
+ result
+ };
+
+ // If `-Zincremental-verify-ich` is specified, re-hash results from
+ // the cache and make sure that they have the expected fingerprint.
+ if unlikely!(tcx.session().opts.debugging_opts.incremental_verify_ich) {
+ incremental_verify_ich::<Q, _>(tcx, &result, dep_node, dep_node_index);
+ }
+
+ result
+ }
+
+ #[inline(never)]
+ #[cold]
+ fn incremental_verify_ich<Q, CTX>(
+ tcx: CTX,
+ result: &Q::Value,
+ dep_node: &DepNode<CTX::DepKind>,
+ dep_node_index: DepNodeIndex,
+ )
+ where
+ CTX: QueryContext,
+ Q: QueryDescription<CTX>,
+ {
+ use rustc_data_structures::fingerprint::Fingerprint;
+
+ assert!(
+ Some(tcx.dep_graph().fingerprint_of(dep_node_index))
+ == tcx.dep_graph().prev_fingerprint_of(dep_node),
+ "fingerprint for green query instance not loaded from cache: {:?}",
+ dep_node,
+ );
+
+ debug!("BEGIN verify_ich({:?})", dep_node);
+ let mut hcx = tcx.create_stable_hashing_context();
+
+ let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
+ debug!("END verify_ich({:?})", dep_node);
+
+ let old_hash = tcx.dep_graph().fingerprint_of(dep_node_index);
+
+ assert!(new_hash == old_hash, "found unstable fingerprints for {:?}", dep_node,);
+ }
+
+ #[inline(always)]
+ fn force_query_with_job<Q, CTX, K>(
+ tcx: CTX,
+ key: Q::Key,
+ job: JobOwner<'tcx, CTX, Q::Cache>,
+ dep_node: DepNode<CTX::DepKind>,
+ ) -> (Q::Value, DepNodeIndex)
+ where
+ Q: QueryDescription<CTX>,
+ CTX: QueryContext<DepKind = K>,
+ CTX: HashStableContextProvider<<CTX as DepContext>::StableHashingContext>,
+ K: DepKind,
+ {
+ // If the following assertion triggers, it can have two reasons:
+ // 1. Something is wrong with DepNode creation, either here or
+ // in `DepGraph::try_mark_green()`.
+ // 2. Two distinct query keys get mapped to the same `DepNode`
+ // (see for example #48923).
+ assert!(
+ !tcx.dep_graph().dep_node_exists(&dep_node),
+ "forcing query with already existing `DepNode`\n\
+ - query-key: {:?}\n\
+ - dep-node: {:?}",
+ key,
+ dep_node
+ );
+
+ let prof_timer = tcx.profiler().query_provider();
+
+ let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
+ tcx.start_query(job.id, diagnostics, |tcx| {
+ if Q::EVAL_ALWAYS {
+ tcx.dep_graph().with_eval_always_task(
+ dep_node,
+ tcx,
+ key,
+ Q::compute,
+ Q::hash_result,
+ )
+ } else {
+ tcx.dep_graph().with_task(dep_node, tcx, key, Q::compute, Q::hash_result)
+ }
+ })
+ });
+
+ prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+
+ if unlikely!(!diagnostics.is_empty()) {
+ if dep_node.kind != DepKind::NULL {
+ tcx.store_diagnostics(dep_node_index, diagnostics);
+ }
+ }
+
+ job.complete(tcx, &result, dep_node_index);
+
+ (result, dep_node_index)
+ }
+
+pub(super) trait QueryGetter: QueryContext {
+ fn get_query<Q: QueryDescription<Self>>(
+ self,
+ span: Span,
+ key: Q::Key,
+ ) -> Q::Value;
+
+ /// Ensure that either this query has all green inputs or been executed.
+ /// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
+ ///
+ /// This function is particularly useful when executing passes for their
+ /// side-effects -- e.g., in order to report errors for erroneous programs.
+ ///
+ /// Note: The optimization is only available during incr. comp.
+ fn ensure_query<Q: QueryDescription<Self>>(self, key: Q::Key);
+
+ fn force_query<Q: QueryDescription<Self>>(
+ self,
+ key: Q::Key,
+ span: Span,
+ dep_node: DepNode<Self::DepKind>,
+ );
+}
+
+impl<CTX, K> QueryGetter for CTX
+where
+ CTX: QueryContext<DepKind = K>,
+ CTX: HashStableContextProvider<<CTX as DepContext>::StableHashingContext>,
+ K: DepKind,
+{
+ #[inline(never)]
+ fn get_query<Q: QueryDescription<Self>>(
+ self,
+ span: Span,
+ key: Q::Key,
+ ) -> Q::Value {
+ debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
+
+ try_get_cached(
+ self,
+ Q::query_state(self),
+ key,
+ |value, index| {
+ self.dep_graph().read_index(index);
+ value.clone()
+ },
+ |key, lookup| try_execute_query::<Q, _, _>(self, span, key, lookup),
+ )
+ }
+
+ /// Ensure that either this query has all green inputs or been executed.
+ /// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
+ ///
+ /// This function is particularly useful when executing passes for their
+ /// side-effects -- e.g., in order to report errors for erroneous programs.
+ ///
+ /// Note: The optimization is only available during incr. comp.
+ fn ensure_query<Q: QueryDescription<Self>>(self, key: Q::Key) {
+ if Q::EVAL_ALWAYS {
+ let _ = self.get_query::<Q>(DUMMY_SP, key);
+ return;
+ }
+
+ // Ensuring an anonymous query makes no sense
+ assert!(!Q::ANON);
+
+ let dep_node = Q::to_dep_node(self, &key);
+
+ match self.dep_graph().try_mark_green_and_read(self, &dep_node) {
+ None => {
+ // A None return from `try_mark_green_and_read` means that this is either
+ // a new dep node or that the dep node has already been marked red.
+ // Either way, we can't call `dep_graph.read()` as we don't have the
+ // DepNodeIndex. We must invoke the query itself. The performance cost
+ // this introduces should be negligible as we'll immediately hit the
+ // in-memory cache, or another query down the line will.
+ let _ = self.get_query::<Q>(DUMMY_SP, key);
+ }
+ Some((_, dep_node_index)) => {
+ self.profiler().query_cache_hit(dep_node_index.into());
+ }
+ }
+ }
+
+ fn force_query<Q: QueryDescription<Self>>(
+ self,
+ key: Q::Key,
+ span: Span,
+ dep_node: DepNode<Self::DepKind>,
+ ) {
+ // We may be concurrently trying both execute and force a query.
+ // Ensure that only one of them runs the query.
+
+ try_get_cached(
+ self,
+ Q::query_state(self),
+ key,
+ |_, _| {
+ // Cache hit, do nothing
+ },
+ |key, lookup| {
+ let job = match JobOwner::try_start::<Q, _>(self, span, &key, lookup) {
+ TryGetJob::NotYetStarted(job) => job,
+ TryGetJob::Cycle(_) => return,
+ #[cfg(parallel_compiler)]
+ TryGetJob::JobCompleted(_) => return,
+ };
+ force_query_with_job::<Q, _, _>(self, key, job, dep_node);
+ },
+ );
+ }
+}
+
+macro_rules! handle_cycle_error {
+ ([][$tcx: expr, $error:expr]) => {{
+ $tcx.report_cycle($error).emit();
+ Value::from_cycle_error($tcx)
+ }};
+ ([fatal_cycle $($rest:tt)*][$tcx:expr, $error:expr]) => {{
+ $tcx.report_cycle($error).emit();
+ $tcx.sess.abort_if_errors();
+ unreachable!()
+ }};
+ ([cycle_delay_bug $($rest:tt)*][$tcx:expr, $error:expr]) => {{
+ $tcx.report_cycle($error).delay_as_bug();
+ Value::from_cycle_error($tcx)
+ }};
+ ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => {
+ handle_cycle_error!([$($($modifiers)*)*][$($args)*])
+ };
+}
+
+macro_rules! is_anon {
+ ([]) => {{
+ false
+ }};
+ ([anon $($rest:tt)*]) => {{
+ true
+ }};
+ ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*]) => {
+ is_anon!([$($($modifiers)*)*])
+ };
+}
+
+macro_rules! is_eval_always {
+ ([]) => {{
+ false
+ }};
+ ([eval_always $($rest:tt)*]) => {{
+ true
+ }};
+ ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*]) => {
+ is_eval_always!([$($($modifiers)*)*])
+ };
+}
+
+macro_rules! query_storage {
+ (<$tcx:tt>[][$K:ty, $V:ty]) => {
+ <<$K as Key>::CacheSelector as CacheSelector<TyCtxt<$tcx>, $K, $V>>::Cache
+ };
+ (<$tcx:tt>[storage($ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => {
+ $ty
+ };
+ (<$tcx:tt>[$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => {
+ query_storage!(<$tcx>[$($($modifiers)*)*][$($args)*])
+ };
+}
+
+macro_rules! hash_result {
+ ([][$hcx:expr, $result:expr]) => {{
+ dep_graph::hash_result($hcx, &$result)
+ }};
+ ([no_hash $($rest:tt)*][$hcx:expr, $result:expr]) => {{
+ None
+ }};
+ ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => {
+ hash_result!([$($($modifiers)*)*][$($args)*])
+ };
+}
+
+macro_rules! define_queries {
+ (<$tcx:tt> $($category:tt {
+ $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*
+ },)*) => {
+ define_queries_inner! { <$tcx>
+ $($( $(#[$attr])* category<$category> [$($modifiers)*] fn $name: $node($K) -> $V,)*)*
+ }
+ }
+}
+
+macro_rules! define_queries_inner {
+ (<$tcx:tt>
+ $($(#[$attr:meta])* category<$category:tt>
+ [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*) => {
+
+ use std::mem;
+ use crate::{
+ rustc_data_structures::stable_hasher::HashStable,
+ rustc_data_structures::stable_hasher::StableHasher,
+ ich::StableHashingContext
+ };
+ use rustc_data_structures::profiling::ProfileCategory;
+
+ define_queries_struct! {
+ tcx: $tcx,
+ input: ($(([$($modifiers)*] [$($attr)*] [$name]))*)
+ }
+
+ #[allow(nonstandard_style)]
+ #[derive(Clone, Debug)]
+ pub enum Query<$tcx> {
+ $($(#[$attr])* $name($K)),*
+ }
+
+ impl<$tcx> Query<$tcx> {
+ pub fn name(&self) -> &'static str {
+ match *self {
+ $(Query::$name(_) => stringify!($name),)*
+ }
+ }
+
+ pub fn describe(&self, tcx: TyCtxt<$tcx>) -> Cow<'static, str> {
+ let (r, name) = match *self {
+ $(Query::$name(key) => {
+ (queries::$name::describe(tcx, key), stringify!($name))
+ })*
+ };
+ if tcx.sess.verbose() {
+ format!("{} [{}]", r, name).into()
+ } else {
+ r
+ }
+ }
+
+ // FIXME(eddyb) Get more valid `Span`s on queries.
+ pub fn default_span(&self, tcx: TyCtxt<$tcx>, span: Span) -> Span {
+ if !span.is_dummy() {
+ return span;
+ }
+ // The `def_span` query is used to calculate `default_span`,
+ // so exit to avoid infinite recursion.
+ if let Query::def_span(..) = *self {
+ return span
+ }
+ match *self {
+ $(Query::$name(key) => key.default_span(tcx),)*
+ }
+ }
+ }
+
+ impl<'a, $tcx> HashStable<StableHashingContext<'a>> for Query<$tcx> {
+ fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
+ mem::discriminant(self).hash_stable(hcx, hasher);
+ match *self {
+ $(Query::$name(key) => key.hash_stable(hcx, hasher),)*
+ }
+ }
+ }
+
+ pub mod queries {
+ use std::marker::PhantomData;
+
+ $(#[allow(nonstandard_style)]
+ pub struct $name<$tcx> {
+ data: PhantomData<&$tcx ()>
+ })*
+ }
+
+ $(impl<$tcx> QueryConfig<TyCtxt<$tcx>> for queries::$name<$tcx> {
+ type Key = $K;
+ type Value = $V;
+ const NAME: &'static str = stringify!($name);
+ const CATEGORY: ProfileCategory = $category;
+ }
+
+ impl<$tcx> QueryAccessors<TyCtxt<$tcx>> for queries::$name<$tcx> {
+ const ANON: bool = is_anon!([$($modifiers)*]);
+ const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]);
+ const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::$node;
+
+ type Cache = query_storage!(<$tcx>[$($modifiers)*][$K, $V]);
+
+ #[inline(always)]
+ fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<TyCtxt<$tcx>, Self::Cache> {
+ &tcx.queries.$name
+ }
+
+ #[allow(unused)]
+ #[inline(always)]
+ fn to_dep_node(tcx: TyCtxt<$tcx>, key: &Self::Key) -> DepNode {
+ DepConstructor::$node(tcx, *key)
+ }
+
+ #[inline]
+ fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
+ let provider = tcx.queries.providers.get(key.query_crate())
+ // HACK(eddyb) it's possible crates may be loaded after
+ // the query engine is created, and because crate loading
+ // is not yet integrated with the query engine, such crates
+ // would be missing appropriate entries in `providers`.
+ .unwrap_or(&tcx.queries.fallback_extern_providers)
+ .$name;
+ provider(tcx, key)
+ }
+
+ fn hash_result(
+ _hcx: &mut StableHashingContext<'_>,
+ _result: &Self::Value
+ ) -> Option<Fingerprint> {
+ hash_result!([$($modifiers)*][_hcx, _result])
+ }
+
+ fn handle_cycle_error(
+ tcx: TyCtxt<'tcx>,
+ error: CycleError<Query<'tcx>>
+ ) -> Self::Value {
+ handle_cycle_error!([$($modifiers)*][tcx, error])
+ }
+ })*
+
+ #[derive(Copy, Clone)]
+ pub struct TyCtxtEnsure<'tcx> {
+ pub tcx: TyCtxt<'tcx>,
+ }
+
+ impl TyCtxtEnsure<$tcx> {
+ $($(#[$attr])*
+ #[inline(always)]
+ pub fn $name(self, key: $K) {
+ self.tcx.ensure_query::<queries::$name<'_>>(key)
+ })*
+ }
+
+ #[derive(Copy, Clone)]
+ pub struct TyCtxtAt<'tcx> {
+ pub tcx: TyCtxt<'tcx>,
+ pub span: Span,
+ }
+
+ impl Deref for TyCtxtAt<'tcx> {
+ type Target = TyCtxt<'tcx>;
+ #[inline(always)]
+ fn deref(&self) -> &Self::Target {
+ &self.tcx
+ }
+ }
+
+ impl TyCtxt<$tcx> {
+ /// Returns a transparent wrapper for `TyCtxt`, which ensures queries
+ /// are executed instead of just returning their results.
+ #[inline(always)]
+ pub fn ensure(self) -> TyCtxtEnsure<$tcx> {
+ TyCtxtEnsure {
+ tcx: self,
+ }
+ }
+
+ /// Returns a transparent wrapper for `TyCtxt` which uses
+ /// `span` as the location of queries performed through it.
+ #[inline(always)]
+ pub fn at(self, span: Span) -> TyCtxtAt<$tcx> {
+ TyCtxtAt {
+ tcx: self,
+ span
+ }
+ }
+
+ $($(#[$attr])*
+ #[inline(always)]
+ pub fn $name(self, key: $K) -> $V {
+ self.at(DUMMY_SP).$name(key)
+ })*
+
+ /// All self-profiling events generated by the query engine use
+ /// virtual `StringId`s for their `event_id`. This method makes all
+ /// those virtual `StringId`s point to actual strings.
+ ///
+ /// If we are recording only summary data, the ids will point to
+ /// just the query names. If we are recording query keys too, we
+ /// allocate the corresponding strings here.
+ pub fn alloc_self_profile_query_strings(self) {
+ use crate::ty::query::profiling_support::{
+ alloc_self_profile_query_strings_for_query_cache,
+ QueryKeyStringCache,
+ };
+
+ if !self.prof.enabled() {
+ return;
+ }
+
+ let mut string_cache = QueryKeyStringCache::new();
+
+ $({
+ alloc_self_profile_query_strings_for_query_cache(
+ self,
+ stringify!($name),
+ &self.queries.$name,
+ &mut string_cache,
+ );
+ })*
+ }
+ }
+
+ impl TyCtxtAt<$tcx> {
+ $($(#[$attr])*
+ #[inline(always)]
+ pub fn $name(self, key: $K) -> $V {
+ self.tcx.get_query::<queries::$name<'_>>(self.span, key)
+ })*
+ }
+
+ define_provider_struct! {
+ tcx: $tcx,
+ input: ($(([$($modifiers)*] [$name] [$K] [$V]))*)
+ }
+
+ impl<$tcx> Copy for Providers<$tcx> {}
+ impl<$tcx> Clone for Providers<$tcx> {
+ fn clone(&self) -> Self { *self }
+ }
+ }
+}
+
+macro_rules! define_queries_struct {
+ (tcx: $tcx:tt,
+ input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
+ pub struct Queries<$tcx> {
+ /// This provides access to the incrimental comilation on-disk cache for query results.
+ /// Do not access this directly. It is only meant to be used by
+ /// `DepGraph::try_mark_green()` and the query infrastructure.
+ pub(crate) on_disk_cache: OnDiskCache<'tcx>,
+
+ providers: IndexVec<CrateNum, Providers<$tcx>>,
+ fallback_extern_providers: Box<Providers<$tcx>>,
+
+ $($(#[$attr])* $name: QueryState<
+ TyCtxt<$tcx>,
+ <queries::$name<$tcx> as QueryAccessors<TyCtxt<'tcx>>>::Cache,
+ >,)*
+ }
+
+ impl<$tcx> Queries<$tcx> {
+ pub(crate) fn new(
+ providers: IndexVec<CrateNum, Providers<$tcx>>,
+ fallback_extern_providers: Providers<$tcx>,
+ on_disk_cache: OnDiskCache<'tcx>,
+ ) -> Self {
+ Queries {
+ providers,
+ fallback_extern_providers: Box::new(fallback_extern_providers),
+ on_disk_cache,
+ $($name: Default::default()),*
+ }
+ }
+
+ pub(crate) fn try_collect_active_jobs(
+ &self
+ ) -> Option<FxHashMap<QueryJobId<crate::dep_graph::DepKind>, QueryJobInfo<TyCtxt<'tcx>>>> {
+ let mut jobs = FxHashMap::default();
+
+ $(
+ self.$name.try_collect_active_jobs(
+ <queries::$name<'tcx> as QueryAccessors<TyCtxt<'tcx>>>::DEP_KIND,
+ Query::$name,
+ &mut jobs,
+ )?;
+ )*
+
+ Some(jobs)
+ }
+ }
+ };
+}
+
+macro_rules! define_provider_struct {
+ (tcx: $tcx:tt,
+ input: ($(([$($modifiers:tt)*] [$name:ident] [$K:ty] [$R:ty]))*)) => {
+ pub struct Providers<$tcx> {
+ $(pub $name: fn(TyCtxt<$tcx>, $K) -> $R,)*
+ }
+
+ impl<$tcx> Default for Providers<$tcx> {
+ fn default() -> Self {
+ $(fn $name<$tcx>(_: TyCtxt<$tcx>, key: $K) -> $R {
+ bug!("`tcx.{}({:?})` unsupported by its crate",
+ stringify!($name), key);
+ })*
+ Providers { $($name),* }
+ }
+ }
+ };
+}
projects / rust.git / commitdiff