compiler/rustc_query_impl/src/plumbing.rs

   1 //! The implementation of the query system itself. This defines the macros that
   2 //! generate the actual methods on tcx which find and execute the provider,
   3 //! manage the caches, and so forth.
   4
   5 use crate::keys::Key;
   6 use crate::{on_disk_cache, Queries};
   7 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
   8 use rustc_data_structures::sync::Lock;
   9 use rustc_errors::{Diagnostic, Handler};
  10 use rustc_middle::dep_graph::{
  11     self, DepKind, DepKindStruct, DepNode, DepNodeIndex, SerializedDepNodeIndex,
  12 };
  13 use rustc_middle::ty::tls::{self, ImplicitCtxt};
  14 use rustc_middle::ty::{self, TyCtxt};
  15 use rustc_query_system::dep_graph::{DepNodeParams, HasDepContext};
  16 use rustc_query_system::ich::StableHashingContext;
  17 use rustc_query_system::query::{
  18     force_query, QueryConfig, QueryContext, QueryDescription, QueryJobId, QueryMap,
  19     QuerySideEffects, QueryStackFrame,
  20 };
  21 use std::any::Any;
  22 use std::num::NonZeroU64;
  23 use thin_vec::ThinVec;
  24
  25 #[derive(Copy, Clone)]
  26 pub struct QueryCtxt<'tcx> {
  27     pub tcx: TyCtxt<'tcx>,
  28     pub queries: &'tcx Queries<'tcx>,
  29 }
  30
  31 impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> {
  32     type Target = TyCtxt<'tcx>;
  33
  34     #[inline]
  35     fn deref(&self) -> &Self::Target {
  36         &self.tcx
  37     }
  38 }
  39
  40 impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
  41     type DepKind = rustc_middle::dep_graph::DepKind;
  42     type DepContext = TyCtxt<'tcx>;
  43
  44     #[inline]
  45     fn dep_context(&self) -> &Self::DepContext {
  46         &self.tcx
  47     }
  48 }
  49
  50 impl QueryContext for QueryCtxt<'_> {
  51     fn next_job_id(&self) -> QueryJobId {
  52         QueryJobId(
  53             NonZeroU64::new(
  54                 self.queries.jobs.fetch_add(1, rustc_data_structures::sync::Ordering::Relaxed),
  55             )
  56             .unwrap(),
  57         )
  58     }
  59
  60     fn current_query_job(&self) -> Option<QueryJobId> {
  61         tls::with_related_context(**self, |icx| icx.query)
  62     }
  63
  64     fn try_collect_active_jobs(&self) -> Option<QueryMap> {
  65         self.queries.try_collect_active_jobs(**self)
  66     }
  67
  68     // Interactions with on_disk_cache
  69     fn load_side_effects(&self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects {
  70         self.queries
  71             .on_disk_cache
  72             .as_ref()
  73             .map(|c| c.load_side_effects(**self, prev_dep_node_index))
  74             .unwrap_or_default()
  75     }
  76
  77     fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
  78         if let Some(c) = self.queries.on_disk_cache.as_ref() {
  79             c.store_side_effects(dep_node_index, side_effects)
  80         }
  81     }
  82
  83     fn store_side_effects_for_anon_node(
  84         &self,
  85         dep_node_index: DepNodeIndex,
  86         side_effects: QuerySideEffects,
  87     ) {
  88         if let Some(c) = self.queries.on_disk_cache.as_ref() {
  89             c.store_side_effects_for_anon_node(dep_node_index, side_effects)
  90         }
  91     }
  92
  93     /// Executes a job by changing the `ImplicitCtxt` to point to the
  94     /// new query job while it executes. It returns the diagnostics
  95     /// captured during execution and the actual result.
  96     #[inline(always)]
  97     fn start_query<R>(
  98         &self,
  99         token: QueryJobId,
 100         depth_limit: bool,
 101         diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
 102         compute: impl FnOnce() -> R,
 103     ) -> R {
 104         // The `TyCtxt` stored in TLS has the same global interner lifetime
 105         // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
 106         // when accessing the `ImplicitCtxt`.
 107         tls::with_related_context(**self, move |current_icx| {
 108             if depth_limit && !self.recursion_limit().value_within_limit(current_icx.query_depth) {
 109                 self.depth_limit_error();
 110             }
 111
 112             // Update the `ImplicitCtxt` to point to our new query job.
 113             let new_icx = ImplicitCtxt {
 114                 tcx: **self,
 115                 query: Some(token),
 116                 diagnostics,
 117                 query_depth: current_icx.query_depth + depth_limit as usize,
 118                 task_deps: current_icx.task_deps,
 119             };
 120
 121             // Use the `ImplicitCtxt` while we execute the query.
 122             tls::enter_context(&new_icx, |_| {
 123                 rustc_data_structures::stack::ensure_sufficient_stack(compute)
 124             })
 125         })
 126     }
 127 }
 128
 129 impl<'tcx> QueryCtxt<'tcx> {
 130     #[inline]
 131     pub fn from_tcx(tcx: TyCtxt<'tcx>) -> Self {
 132         let queries = tcx.queries.as_any();
 133         let queries = unsafe {
 134             let queries = std::mem::transmute::<&dyn Any, &dyn Any>(queries);
 135             let queries = queries.downcast_ref().unwrap();
 136             let queries = std::mem::transmute::<&Queries<'_>, &Queries<'_>>(queries);
 137             queries
 138         };
 139         QueryCtxt { tcx, queries }
 140     }
 141
 142     pub(crate) fn on_disk_cache(self) -> Option<&'tcx on_disk_cache::OnDiskCache<'tcx>> {
 143         self.queries.on_disk_cache.as_ref()
 144     }
 145
 146     pub(super) fn encode_query_results(
 147         self,
 148         encoder: &mut on_disk_cache::CacheEncoder<'_, 'tcx>,
 149         query_result_index: &mut on_disk_cache::EncodedDepNodeIndex,
 150     ) {
 151         macro_rules! encode_queries {
 152             ($($query:ident,)*) => {
 153                 $(
 154                     on_disk_cache::encode_query_results::<_, super::queries::$query<'_>>(
 155                         self,
 156                         encoder,
 157                         query_result_index
 158                     );
 159                 )*
 160             }
 161         }
 162
 163         rustc_cached_queries!(encode_queries!);
 164     }
 165
 166     pub fn try_print_query_stack(
 167         self,
 168         query: Option<QueryJobId>,
 169         handler: &Handler,
 170         num_frames: Option<usize>,
 171     ) -> usize {
 172         rustc_query_system::query::print_query_stack(self, query, handler, num_frames)
 173     }
 174 }
 175
 176 macro_rules! handle_cycle_error {
 177     ([][$tcx: expr, $error:expr]) => {{
 178         $error.emit();
 179         Value::from_cycle_error($tcx)
 180     }};
 181     ([(fatal_cycle) $($rest:tt)*][$tcx:expr, $error:expr]) => {{
 182         $error.emit();
 183         $tcx.sess.abort_if_errors();
 184         unreachable!()
 185     }};
 186     ([(cycle_delay_bug) $($rest:tt)*][$tcx:expr, $error:expr]) => {{
 187         $error.delay_as_bug();
 188         Value::from_cycle_error($tcx)
 189     }};
 190     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
 191         handle_cycle_error!([$($modifiers)*][$($args)*])
 192     };
 193 }
 194
 195 macro_rules! is_anon {
 196     ([]) => {{
 197         false
 198     }};
 199     ([(anon) $($rest:tt)*]) => {{
 200         true
 201     }};
 202     ([$other:tt $($modifiers:tt)*]) => {
 203         is_anon!([$($modifiers)*])
 204     };
 205 }
 206
 207 macro_rules! is_eval_always {
 208     ([]) => {{
 209         false
 210     }};
 211     ([(eval_always) $($rest:tt)*]) => {{
 212         true
 213     }};
 214     ([$other:tt $($modifiers:tt)*]) => {
 215         is_eval_always!([$($modifiers)*])
 216     };
 217 }
 218
 219 macro_rules! depth_limit {
 220     ([]) => {{
 221         false
 222     }};
 223     ([(depth_limit) $($rest:tt)*]) => {{
 224         true
 225     }};
 226     ([$other:tt $($modifiers:tt)*]) => {
 227         depth_limit!([$($modifiers)*])
 228     };
 229 }
 230
 231 macro_rules! hash_result {
 232     ([]) => {{
 233         Some(dep_graph::hash_result)
 234     }};
 235     ([(no_hash) $($rest:tt)*]) => {{
 236         None
 237     }};
 238     ([$other:tt $($modifiers:tt)*]) => {
 239         hash_result!([$($modifiers)*])
 240     };
 241 }
 242
 243 macro_rules! get_provider {
 244     ([][$tcx:expr, $name:ident, $key:expr]) => {{
 245         $tcx.queries.local_providers.$name
 246     }};
 247     ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
 248         if $key.query_crate_is_local() {
 249             $tcx.queries.local_providers.$name
 250         } else {
 251             $tcx.queries.extern_providers.$name
 252         }
 253     }};
 254     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
 255         get_provider!([$($modifiers)*][$($args)*])
 256     };
 257 }
 258
 259 pub(crate) fn create_query_frame<
 260     'tcx,
 261     K: Copy + Key + for<'a> HashStable<StableHashingContext<'a>>,
 262 >(
 263     tcx: QueryCtxt<'tcx>,
 264     do_describe: fn(QueryCtxt<'tcx>, K) -> String,
 265     key: K,
 266     kind: DepKind,
 267     name: &'static str,
 268 ) -> QueryStackFrame {
 269     // Disable visible paths printing for performance reasons.
 270     // Showing visible path instead of any path is not that important in production.
 271     let description = ty::print::with_no_visible_paths!(
 272         // Force filename-line mode to avoid invoking `type_of` query.
 273         ty::print::with_forced_impl_filename_line!(do_describe(tcx, key))
 274     );
 275     let description =
 276         if tcx.sess.verbose() { format!("{} [{}]", description, name) } else { description };
 277     let span = if kind == dep_graph::DepKind::def_span {
 278         // The `def_span` query is used to calculate `default_span`,
 279         // so exit to avoid infinite recursion.
 280         None
 281     } else {
 282         Some(key.default_span(*tcx))
 283     };
 284     let def_kind = if kind == dep_graph::DepKind::opt_def_kind {
 285         // Try to avoid infinite recursion.
 286         None
 287     } else {
 288         key.key_as_def_id()
 289             .and_then(|def_id| def_id.as_local())
 290             .and_then(|def_id| tcx.opt_def_kind(def_id))
 291     };
 292     let hash = || {
 293         tcx.with_stable_hashing_context(|mut hcx| {
 294             let mut hasher = StableHasher::new();
 295             std::mem::discriminant(&kind).hash_stable(&mut hcx, &mut hasher);
 296             key.hash_stable(&mut hcx, &mut hasher);
 297             hasher.finish::<u64>()
 298         })
 299     };
 300
 301     QueryStackFrame::new(name, description, span, def_kind, hash)
 302 }
 303
 304 pub(crate) fn try_load_from_on_disk_cache<'tcx, Q, V>(
 305     tcx: TyCtxt<'tcx>,
 306     dep_node: DepNode,
 307     cache_query_deps: fn(TyCtxt<'tcx>, Q::Key) -> V,
 308 ) where
 309     Q: QueryDescription<QueryCtxt<'tcx>>,
 310     Q::Key: DepNodeParams<TyCtxt<'tcx>>,
 311 {
 312     debug_assert!(tcx.dep_graph.is_green(&dep_node));
 313
 314     let key = Q::Key::recover(tcx, &dep_node).unwrap_or_else(|| {
 315         panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash)
 316     });
 317     if Q::cache_on_disk(tcx, &key) {
 318         let _ = cache_query_deps(tcx, key);
 319     }
 320 }
 321
 322 pub(crate) fn force_from_dep_node<'tcx, Q>(tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool
 323 where
 324     Q: QueryDescription<QueryCtxt<'tcx>>,
 325     Q::Key: DepNodeParams<TyCtxt<'tcx>>,
 326 {
 327     if let Some(key) = Q::Key::recover(tcx, &dep_node) {
 328         #[cfg(debug_assertions)]
 329         let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
 330         let tcx = QueryCtxt::from_tcx(tcx);
 331         force_query::<Q, _>(tcx, key, dep_node);
 332         true
 333     } else {
 334         false
 335     }
 336 }
 337
 338 pub(crate) fn query_callback<'tcx, Q: QueryConfig>(
 339     // NOTE: we can't remove these function pointers, because `recover` is invariant -> `try_load_from_on_disk_cache` takes a concrete lifetime, not a universal lifetime.
 340     // Instead, we infer the correct lifetime at the callsite, so we can pass in a HRTB function pointer to the DepKindStruct.
 341     try_load_from_on_disk_cache: fn(TyCtxt<'_>, DepNode),
 342     force_from_dep_node: fn(TyCtxt<'_>, DepNode) -> bool,
 343     is_anon: bool,
 344     is_eval_always: bool,
 345 ) -> DepKindStruct
 346 where
 347     Q: QueryDescription<QueryCtxt<'tcx>>,
 348     Q::Key: DepNodeParams<TyCtxt<'tcx>>,
 349 {
 350     let fingerprint_style = Q::Key::fingerprint_style();
 351
 352     if is_anon || !fingerprint_style.reconstructible() {
 353         return DepKindStruct {
 354             is_anon,
 355             is_eval_always,
 356             fingerprint_style,
 357             force_from_dep_node: None,
 358             try_load_from_on_disk_cache: None,
 359         };
 360     }
 361
 362     DepKindStruct {
 363         is_anon,
 364         is_eval_always,
 365         fingerprint_style,
 366         force_from_dep_node: Some(force_from_dep_node),
 367         try_load_from_on_disk_cache: Some(try_load_from_on_disk_cache),
 368     }
 369 }
 370
 371 // NOTE: `$V` isn't used here, but we still need to match on it so it can be passed to other macros
 372 // invoked by `rustc_query_append`.
 373 macro_rules! define_queries {
 374     (
 375      $($(#[$attr:meta])*
 376         [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
 377         define_queries_struct! {
 378             input: ($(([$($modifiers)*] [$($attr)*] [$name]))*)
 379         }
 380
 381         #[allow(nonstandard_style)]
 382         mod queries {
 383             use std::marker::PhantomData;
 384
 385             $(pub struct $name<'tcx> {
 386                 data: PhantomData<&'tcx ()>
 387             })*
 388         }
 389
 390         $(impl<'tcx> QueryConfig for queries::$name<'tcx> {
 391             type Key = query_keys::$name<'tcx>;
 392             type Value = query_values::$name<'tcx>;
 393             type Stored = query_stored::$name<'tcx>;
 394             const NAME: &'static str = stringify!($name);
 395         }
 396
 397         impl<'tcx> QueryDescription<QueryCtxt<'tcx>> for queries::$name<'tcx> {
 398             rustc_query_description! { $name }
 399
 400             type Cache = query_storage::$name<'tcx>;
 401
 402             #[inline(always)]
 403             fn query_state<'a>(tcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key>
 404                 where QueryCtxt<'tcx>: 'a
 405             {
 406                 &tcx.queries.$name
 407             }
 408
 409             #[inline(always)]
 410             fn query_cache<'a>(tcx: QueryCtxt<'tcx>) -> &'a Self::Cache
 411                 where 'tcx:'a
 412             {
 413                 &tcx.query_caches.$name
 414             }
 415
 416             #[inline]
 417             fn make_vtable(tcx: QueryCtxt<'tcx>, key: &Self::Key) ->
 418                 QueryVTable<QueryCtxt<'tcx>, Self::Key, Self::Value>
 419             {
 420                 let compute = get_provider!([$($modifiers)*][tcx, $name, key]);
 421                 let cache_on_disk = Self::cache_on_disk(tcx.tcx, key);
 422                 QueryVTable {
 423                     anon: is_anon!([$($modifiers)*]),
 424                     eval_always: is_eval_always!([$($modifiers)*]),
 425                     depth_limit: depth_limit!([$($modifiers)*]),
 426                     dep_kind: dep_graph::DepKind::$name,
 427                     hash_result: hash_result!([$($modifiers)*]),
 428                     handle_cycle_error: |tcx, mut error| handle_cycle_error!([$($modifiers)*][tcx, error]),
 429                     compute,
 430                     cache_on_disk,
 431                     try_load_from_disk: Self::TRY_LOAD_FROM_DISK,
 432                 }
 433             }
 434         })*
 435
 436         #[allow(nonstandard_style)]
 437         mod query_callbacks {
 438             use super::*;
 439             use rustc_query_system::dep_graph::FingerprintStyle;
 440
 441             // We use this for most things when incr. comp. is turned off.
 442             pub fn Null() -> DepKindStruct {
 443                 DepKindStruct {
 444                     is_anon: false,
 445                     is_eval_always: false,
 446                     fingerprint_style: FingerprintStyle::Unit,
 447                     force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
 448                     try_load_from_on_disk_cache: None,
 449                 }
 450             }
 451
 452             // We use this for the forever-red node.
 453             pub fn Red() -> DepKindStruct {
 454                 DepKindStruct {
 455                     is_anon: false,
 456                     is_eval_always: false,
 457                     fingerprint_style: FingerprintStyle::Unit,
 458                     force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
 459                     try_load_from_on_disk_cache: None,
 460                 }
 461             }
 462
 463             pub fn TraitSelect() -> DepKindStruct {
 464                 DepKindStruct {
 465                     is_anon: true,
 466                     is_eval_always: false,
 467                     fingerprint_style: FingerprintStyle::Unit,
 468                     force_from_dep_node: None,
 469                     try_load_from_on_disk_cache: None,
 470                 }
 471             }
 472
 473             pub fn CompileCodegenUnit() -> DepKindStruct {
 474                 DepKindStruct {
 475                     is_anon: false,
 476                     is_eval_always: false,
 477                     fingerprint_style: FingerprintStyle::Opaque,
 478                     force_from_dep_node: None,
 479                     try_load_from_on_disk_cache: None,
 480                 }
 481             }
 482
 483             pub fn CompileMonoItem() -> DepKindStruct {
 484                 DepKindStruct {
 485                     is_anon: false,
 486                     is_eval_always: false,
 487                     fingerprint_style: FingerprintStyle::Opaque,
 488                     force_from_dep_node: None,
 489                     try_load_from_on_disk_cache: None,
 490                 }
 491             }
 492
 493             $(pub(crate) fn $name()-> DepKindStruct {
 494                 let is_anon = is_anon!([$($modifiers)*]);
 495                 let is_eval_always = is_eval_always!([$($modifiers)*]);
 496                 type Q<'tcx> = queries::$name<'tcx>;
 497
 498                 $crate::plumbing::query_callback::<Q<'_>>(
 499                     |tcx, key| $crate::plumbing::try_load_from_on_disk_cache::<Q<'_>, _>(tcx, key, TyCtxt::$name),
 500                     |tcx, key| $crate::plumbing::force_from_dep_node::<Q<'_>>(tcx, key),
 501                     is_anon,
 502                     is_eval_always
 503                 )
 504             })*
 505         }
 506
 507         pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct] {
 508             arena.alloc_from_iter(make_dep_kind_array!(query_callbacks))
 509         }
 510     }
 511 }
 512
 513 macro_rules! define_queries_struct {
 514     (
 515      input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
 516         pub struct Queries<'tcx> {
 517             local_providers: Box<Providers>,
 518             extern_providers: Box<ExternProviders>,
 519
 520             pub on_disk_cache: Option<OnDiskCache<'tcx>>,
 521
 522             jobs: AtomicU64,
 523
 524             $($(#[$attr])*  $name: QueryState<<queries::$name<'tcx> as QueryConfig>::Key>,)*
 525         }
 526
 527         impl<'tcx> Queries<'tcx> {
 528             pub fn new(
 529                 local_providers: Providers,
 530                 extern_providers: ExternProviders,
 531                 on_disk_cache: Option<OnDiskCache<'tcx>>,
 532             ) -> Self {
 533                 Queries {
 534                     local_providers: Box::new(local_providers),
 535                     extern_providers: Box::new(extern_providers),
 536                     on_disk_cache,
 537                     jobs: AtomicU64::new(1),
 538                     $($name: Default::default()),*
 539                 }
 540             }
 541
 542             pub(crate) fn try_collect_active_jobs(
 543                 &'tcx self,
 544                 tcx: TyCtxt<'tcx>,
 545             ) -> Option<QueryMap> {
 546                 let tcx = QueryCtxt { tcx, queries: self };
 547                 let mut jobs = QueryMap::default();
 548
 549                 $(
 550                     let make_query = |tcx, key| {
 551                         let kind = dep_graph::DepKind::$name;
 552                         let name = stringify!($name);
 553                         $crate::plumbing::create_query_frame(tcx, queries::$name::describe, key, kind, name)
 554                     };
 555                     self.$name.try_collect_active_jobs(
 556                         tcx,
 557                         make_query,
 558                         &mut jobs,
 559                     )?;
 560                 )*
 561
 562                 Some(jobs)
 563             }
 564         }
 565
 566         impl<'tcx> QueryEngine<'tcx> for Queries<'tcx> {
 567             fn as_any(&'tcx self) -> &'tcx dyn std::any::Any {
 568                 let this = unsafe { std::mem::transmute::<&Queries<'_>, &Queries<'_>>(self) };
 569                 this as _
 570             }
 571
 572             fn try_mark_green(&'tcx self, tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool {
 573                 let qcx = QueryCtxt { tcx, queries: self };
 574                 tcx.dep_graph.try_mark_green(qcx, dep_node).is_some()
 575             }
 576
 577             $($(#[$attr])*
 578             #[inline(always)]
 579             #[tracing::instrument(level = "trace", skip(self, tcx), ret)]
 580             fn $name(
 581                 &'tcx self,
 582                 tcx: TyCtxt<'tcx>,
 583                 span: Span,
 584                 key: <queries::$name<'tcx> as QueryConfig>::Key,
 585                 mode: QueryMode,
 586             ) -> Option<query_stored::$name<'tcx>> {
 587                 let qcx = QueryCtxt { tcx, queries: self };
 588                 get_query::<queries::$name<'tcx>, _>(qcx, span, key, mode)
 589             })*
 590         }
 591     };
 592 }