/// The current query job, if any. This is updated by `JobOwner::start` in
/// `ty::query::plumbing` when executing a query.
- pub query: Option<query::QueryToken>,
+ pub query: Option<query::QueryJobId>,
/// Where to store diagnostics for the current query job, if any.
/// This is updated by `JobOwner::start` in `ty::query::plumbing` when executing a query.
+use crate::dep_graph::DepKind;
use crate::ty::context::TyCtxt;
use crate::ty::query::plumbing::CycleError;
use crate::ty::query::Query;
use rustc_span::Span;
use std::marker::PhantomData;
-use std::num::NonZeroUsize;
+use std::num::NonZeroU32;
#[cfg(parallel_compiler)]
use {
pub query: Query<'tcx>,
}
-type QueryMap<'tcx> = FxHashMap<QueryToken, QueryJobInfo<'tcx>>;
+type QueryMap<'tcx> = FxHashMap<QueryJobId, QueryJobInfo<'tcx>>;
+
+/// A value uniquely identifiying an active query job within a shard in the query cache.
+#[derive(Copy, Clone, Eq, PartialEq, Hash)]
+pub struct QueryShardJobId(pub NonZeroU32);
/// A value uniquely identifiying an active query job.
-/// This value is created from a stack pointer in `get_query` and `force_query`
-/// which is alive while the query executes.
#[derive(Copy, Clone, Eq, PartialEq, Hash)]
-pub struct QueryToken(NonZeroUsize);
+pub struct QueryJobId {
+ /// Which job within a shard is this
+ pub job: QueryShardJobId,
-impl QueryToken {
- pub fn from<T>(v: &T) -> Self {
- QueryToken(NonZeroUsize::new(v as *const T as usize).unwrap())
- }
+ /// In which shard is this job
+ pub shard: u16,
+
+ /// What kind of query this job is
+ pub kind: DepKind,
+}
+impl QueryJobId {
fn query<'tcx>(self, map: &QueryMap<'tcx>) -> Query<'tcx> {
map.get(&self).unwrap().info.query.clone()
}
}
#[cfg(parallel_compiler)]
- fn parent(self, map: &QueryMap<'_>) -> Option<QueryToken> {
+ fn parent(self, map: &QueryMap<'_>) -> Option<QueryJobId> {
map.get(&self).unwrap().job.parent
}
/// Represents an active query job.
#[derive(Clone)]
pub struct QueryJob<'tcx> {
- pub token: QueryToken,
+ pub id: QueryShardJobId,
/// The span corresponding to the reason for which this query was required.
pub span: Span,
/// The parent query job which created this job and is implicitly waiting on it.
- pub parent: Option<QueryToken>,
+ pub parent: Option<QueryJobId>,
/// The latch that is used to wait on this job.
#[cfg(parallel_compiler)]
impl<'tcx> QueryJob<'tcx> {
/// Creates a new query job.
- pub fn new(token: QueryToken, span: Span, parent: Option<QueryToken>) -> Self {
+ pub fn new(id: QueryShardJobId, span: Span, parent: Option<QueryJobId>) -> Self {
QueryJob {
- token,
+ id,
span,
parent,
#[cfg(parallel_compiler)]
}
#[cfg(parallel_compiler)]
- pub(super) fn latch(&mut self) -> QueryLatch<'tcx> {
+ pub(super) fn latch(&mut self, _id: QueryJobId) -> QueryLatch<'tcx> {
if self.latch.is_none() {
self.latch = Some(QueryLatch::new());
}
}
#[cfg(not(parallel_compiler))]
- pub(super) fn latch(&mut self) -> QueryLatch<'tcx> {
- QueryLatch { token: self.token, dummy: PhantomData }
+ pub(super) fn latch(&mut self, id: QueryJobId) -> QueryLatch<'tcx> {
+ QueryLatch { id, dummy: PhantomData }
}
/// Signals to waiters that the query is complete.
#[cfg(not(parallel_compiler))]
#[derive(Clone)]
pub(super) struct QueryLatch<'tcx> {
- token: QueryToken,
+ id: QueryJobId,
dummy: PhantomData<&'tcx ()>,
}
let info = query_map.get(&job).unwrap();
cycle.push(info.info.clone());
- if job == self.token {
+ if job == self.id {
cycle.reverse();
// This is the end of the cycle
#[cfg(parallel_compiler)]
struct QueryWaiter<'tcx> {
- query: Option<QueryToken>,
+ query: Option<QueryJobId>,
condvar: Condvar,
span: Span,
cycle: Lock<Option<CycleError<'tcx>>>,
/// A resumable waiter of a query. The usize is the index into waiters in the query's latch
#[cfg(parallel_compiler)]
-type Waiter = (QueryToken, usize);
+type Waiter = (QueryJobId, usize);
/// Visits all the non-resumable and resumable waiters of a query.
/// Only waiters in a query are visited.
#[cfg(parallel_compiler)]
fn visit_waiters<'tcx, F>(
query_map: &QueryMap<'tcx>,
- query: QueryToken,
+ query: QueryJobId,
mut visit: F,
) -> Option<Option<Waiter>>
where
- F: FnMut(Span, QueryToken) -> Option<Option<Waiter>>,
+ F: FnMut(Span, QueryJobId) -> Option<Option<Waiter>>,
{
// Visit the parent query which is a non-resumable waiter since it's on the same stack
if let Some(parent) = query.parent(query_map) {
#[cfg(parallel_compiler)]
fn cycle_check<'tcx>(
query_map: &QueryMap<'tcx>,
- query: QueryToken,
+ query: QueryJobId,
span: Span,
- stack: &mut Vec<(Span, QueryToken)>,
- visited: &mut FxHashSet<QueryToken>,
+ stack: &mut Vec<(Span, QueryJobId)>,
+ visited: &mut FxHashSet<QueryJobId>,
) -> Option<Option<Waiter>> {
if !visited.insert(query) {
return if let Some(p) = stack.iter().position(|q| q.1 == query) {
#[cfg(parallel_compiler)]
fn connected_to_root<'tcx>(
query_map: &QueryMap<'tcx>,
- query: QueryToken,
- visited: &mut FxHashSet<QueryToken>,
+ query: QueryJobId,
+ visited: &mut FxHashSet<QueryJobId>,
) -> bool {
// We already visited this or we're deliberately ignoring it
if !visited.insert(query) {
// Deterministically pick an query from a list
#[cfg(parallel_compiler)]
-fn pick_query<'a, 'tcx, T, F: Fn(&T) -> (Span, QueryToken)>(
+fn pick_query<'a, 'tcx, T, F: Fn(&T) -> (Span, QueryJobId)>(
query_map: &QueryMap<'tcx>,
tcx: TyCtxt<'tcx>,
queries: &'a [T],
#[cfg(parallel_compiler)]
fn remove_cycle<'tcx>(
query_map: &QueryMap<'tcx>,
- jobs: &mut Vec<QueryToken>,
+ jobs: &mut Vec<QueryJobId>,
wakelist: &mut Vec<Lrc<QueryWaiter<'tcx>>>,
tcx: TyCtxt<'tcx>,
) -> bool {
}
}
})
- .collect::<Vec<(Span, QueryToken, Option<(Span, QueryToken)>)>>();
+ .collect::<Vec<(Span, QueryJobId, Option<(Span, QueryJobId)>)>>();
// Deterministically pick an entry point
let (_, entry_point, usage) = pick_query(query_map, tcx, &entry_points, |e| (e.0, e.1));
let mut wakelist = Vec::new();
let query_map = tcx.queries.try_collect_active_jobs().unwrap();
- let mut jobs: Vec<QueryToken> = query_map.keys().cloned().collect();
+ let mut jobs: Vec<QueryJobId> = query_map.keys().cloned().collect();
let mut found_cycle = false;
use rustc_span::{Span, DUMMY_SP};
use std::any::type_name;
use std::borrow::Cow;
+use std::convert::TryFrom;
use std::ops::Deref;
use std::sync::Arc;
use syntax::ast;
#[cfg(parallel_compiler)]
pub use self::job::handle_deadlock;
use self::job::QueryJobInfo;
-pub use self::job::{QueryInfo, QueryJob, QueryToken};
+pub use self::job::{QueryInfo, QueryJob, QueryJobId};
mod keys;
use self::keys::Key;
use crate::dep_graph::{DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex};
use crate::ty::query::config::{QueryConfig, QueryDescription};
-use crate::ty::query::job::{QueryInfo, QueryJob, QueryToken};
+use crate::ty::query::job::{QueryInfo, QueryJob, QueryJobId, QueryShardJobId};
use crate::ty::query::Query;
use crate::ty::tls;
use crate::ty::{self, TyCtxt};
use rustc_span::source_map::DUMMY_SP;
use rustc_span::Span;
use std::collections::hash_map::Entry;
+use std::convert::TryFrom;
use std::hash::{Hash, Hasher};
use std::mem;
+use std::num::NonZeroU32;
use std::ptr;
pub struct QueryCache<'tcx, D: QueryConfig<'tcx> + ?Sized> {
pub(super) results: FxHashMap<D::Key, QueryValue<D::Value>>,
pub(super) active: FxHashMap<D::Key, QueryResult<'tcx>>,
+
+ /// Used to generate unique ids for active jobs.
+ pub(super) jobs: u32,
+
#[cfg(debug_assertions)]
pub(super) cache_hits: usize,
}
QueryCache {
results: FxHashMap::default(),
active: FxHashMap::default(),
+ jobs: 0,
#[cfg(debug_assertions)]
cache_hits: 0,
}
pub(super) struct JobOwner<'a, 'tcx, Q: QueryDescription<'tcx>> {
cache: &'a Sharded<QueryCache<'tcx, Q>>,
key: Q::Key,
- token: QueryToken,
+ id: QueryJobId,
}
impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> {
/// This function is inlined because that results in a noticeable speed-up
/// for some compile-time benchmarks.
#[inline(always)]
- pub(super) fn try_get(
- tcx: TyCtxt<'tcx>,
- span: Span,
- key: &Q::Key,
- token: QueryToken,
- ) -> TryGetJob<'a, 'tcx, Q> {
+ pub(super) fn try_get(tcx: TyCtxt<'tcx>, span: Span, key: &Q::Key) -> TryGetJob<'a, 'tcx, Q> {
// Handling the `query_blocked_prof_timer` is a bit weird because of the
// control flow in this function: Blocking is implemented by
// awaiting a running job and, once that is done, entering the loop below
key.hash(&mut state);
let key_hash = state.finish();
- let mut lock = cache.get_shard_by_hash(key_hash).lock();
+ let shard = cache.get_shard_index_by_hash(key_hash);
+ let mut lock = cache.get_shard_by_index(shard).lock();
+ let lock = &mut *lock;
+
if let Some((_, value)) =
lock.results.raw_entry().from_key_hashed_nocheck(key_hash, key)
{
query_blocked_prof_timer = Some(tcx.prof.query_blocked());
}
- job.latch()
+ // Create the id of the job we're waiting for
+ let id = QueryJobId {
+ job: job.id,
+ shard: u16::try_from(shard).unwrap(),
+ kind: Q::dep_kind(),
+ };
+
+ job.latch(id)
}
QueryResult::Poisoned => FatalError.raise(),
}
}
Entry::Vacant(entry) => {
+ let jobs = &mut lock.jobs;
+
// No job entry for this query. Return a new one to be started later.
return tls::with_related_context(tcx, |icx| {
- let job = QueryJob::new(token, span, icx.query);
- let owner = JobOwner { cache, token, key: (*key).clone() };
+ // Generate an id unique within this shard.
+ let id = jobs.checked_add(1).unwrap();
+ *jobs = id;
+ let id = QueryShardJobId(NonZeroU32::new(id).unwrap());
+
+ let global_id = QueryJobId {
+ job: id,
+ shard: u16::try_from(shard).unwrap(),
+ kind: Q::dep_kind(),
+ };
+ let job = QueryJob::new(id, span, icx.query);
+ let owner = JobOwner { cache, id: global_id, key: (*key).clone() };
entry.insert(QueryResult::Started(job));
TryGetJob::NotYetStarted(owner)
});
#[inline(always)]
pub(super) fn start_query<F, R>(
self,
- token: QueryToken,
+ token: QueryJobId,
diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
compute: F,
) -> R
pub(super) fn get_query<Q: QueryDescription<'tcx>>(self, span: Span, key: Q::Key) -> Q::Value {
debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
- // Create a token which uniquely identifies this query amongst the executing queries
- // by using a pointer to `key`. `key` is alive until the query completes execution
- // which will prevent reuse of the token value.
- let token = QueryToken::from(&key);
-
- let job = match JobOwner::try_get(self, span, &key, token) {
+ let job = match JobOwner::try_get(self, span, &key) {
TryGetJob::NotYetStarted(job) => job,
TryGetJob::Cycle(result) => return result,
TryGetJob::JobCompleted((v, index)) => {
let prof_timer = self.prof.query_provider();
let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
- self.start_query(job.token, diagnostics, |tcx| {
+ self.start_query(job.id, diagnostics, |tcx| {
tcx.dep_graph.with_anon_task(Q::dep_kind(), || Q::compute(tcx, key))
})
});
// The diagnostics for this query will be
// promoted to the current session during
// `try_mark_green()`, so we can ignore them here.
- let loaded = self.start_query(job.token, None, |tcx| {
+ let loaded = self.start_query(job.id, None, |tcx| {
let marked = tcx.dep_graph.try_mark_green_and_read(tcx, &dep_node);
marked.map(|(prev_dep_node_index, dep_node_index)| {
(
let prof_timer = self.prof.query_provider();
let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
- self.start_query(job.token, diagnostics, |tcx| {
+ self.start_query(job.id, diagnostics, |tcx| {
if Q::EVAL_ALWAYS {
tcx.dep_graph.with_eval_always_task(
dep_node,
#[allow(dead_code)]
fn force_query<Q: QueryDescription<'tcx>>(self, key: Q::Key, span: Span, dep_node: DepNode) {
- // Create a token which uniquely identifies this query amongst the executing queries
- // by using a pointer to `key`. `key` is alive until the query completes execution
- // which will prevent reuse of the token value.
- let token = QueryToken::from(&key);
-
// We may be concurrently trying both execute and force a query.
// Ensure that only one of them runs the query.
- let job = match JobOwner::try_get(self, span, &key, token) {
+ let job = match JobOwner::try_get(self, span, &key) {
TryGetJob::NotYetStarted(job) => job,
TryGetJob::Cycle(_) | TryGetJob::JobCompleted(_) => return,
};
pub fn try_collect_active_jobs(
&self
- ) -> Option<FxHashMap<QueryToken, QueryJobInfo<'tcx>>> {
+ ) -> Option<FxHashMap<QueryJobId, QueryJobInfo<'tcx>>> {
let mut jobs = FxHashMap::default();
$(
// We use try_lock_shards here since we are called from the
// deadlock handler, and this shouldn't be locked.
let shards = self.$name.try_lock_shards()?;
- jobs.extend(shards.iter().flat_map(|shard| shard.active.iter().filter_map(|(k, v)|
- if let QueryResult::Started(ref job) = *v {
- let info = QueryInfo {
- span: job.span,
- query: queries::$name::query(k.clone())
- };
- Some((job.token, QueryJobInfo { info, job: job.clone() }))
- } else {
- None
- }
- )));
+ let shards = shards.iter().enumerate();
+ jobs.extend(shards.flat_map(|(shard_id, shard)| {
+ shard.active.iter().filter_map(move |(k, v)| {
+ if let QueryResult::Started(ref job) = *v {
+ let id = QueryJobId {
+ job: job.id,
+ shard: u16::try_from(shard_id).unwrap(),
+ kind:
+ <queries::$name<'tcx> as QueryAccessors<'tcx>>::dep_kind(),
+ };
+ let info = QueryInfo {
+ span: job.span,
+ query: queries::$name::query(k.clone())
+ };
+ Some((id, QueryJobInfo { info, job: job.clone() }))
+ } else {
+ None
+ }
+ })
+ }));
)*
Some(jobs)
/// `hash` can be computed with any hasher, so long as that hasher is used
/// consistently for each `Sharded` instance.
#[inline]
- pub fn get_shard_by_hash(&self, hash: u64) -> &Lock<T> {
+ pub fn get_shard_index_by_hash(&self, hash: u64) -> usize {
let hash_len = mem::size_of::<usize>();
// Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits.
// hashbrown also uses the lowest bits, so we can't use those
let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize;
- let i = bits % SHARDS;
+ bits % SHARDS
+ }
+
+ #[inline]
+ pub fn get_shard_by_hash(&self, hash: u64) -> &Lock<T> {
+ &self.shards[self.get_shard_index_by_hash(hash)].0
+ }
+
+ #[inline]
+ pub fn get_shard_by_index(&self, i: usize) -> &Lock<T> {
&self.shards[i].0
}
projects / rust.git / commitdiff