Auto merge of #107529 - Zoxc:inline-tweak-profile, r=cjgillot

Optimize query_cache_hit to reduce code size of the query hot path.

A small tweak which improves performance on check builds by 0.33% and reduces `rustc_driver` size by 1%.
<table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check</td><td align="right">1.7978s</td><td align="right">1.7980s</td><td align="right"> 0.01%</td><td align="right">1.7930s</td><td align="right"> -0.27%</td></tr><tr><td>🟣 <b>hyper</b>:check</td><td align="right">0.2594s</td><td align="right">0.2591s</td><td align="right"> -0.12%</td><td align="right">0.2592s</td><td align="right"> -0.09%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check</td><td align="right">6.2522s</td><td align="right">6.2540s</td><td align="right"> 0.03%</td><td align="right">6.2358s</td><td align="right"> -0.26%</td></tr><tr><td>🟣 <b>syn</b>:check</td><td align="right">1.5889s</td><td align="right">1.5880s</td><td align="right"> -0.05%</td><td align="right">1.5799s</td><td align="right"> -0.57%</td></tr><tr><td>🟣 <b>regex</b>:check</td><td align="right">0.9941s</td><td align="right">0.9939s</td><td align="right"> -0.02%</td><td align="right">0.9893s</td><td align="right"> -0.49%</td></tr><tr><td>Total</td><td align="right">10.8925s</td><td align="right">10.8930s</td><td align="right"> 0.01%</td><td align="right">10.8572s</td><td align="right"> -0.32%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">0.9997s</td><td align="right"> -0.03%</td><td align="right">0.9967s</td><td align="right"> -0.33%</td></tr></table>

r? `@cjgillot`

diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index 954e84c303b83d031cc4b2be76c56b4e90e1cea1..7fab8954cb19f1911ff8654061770315bbf0e0a7 100644 (file)
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -11,6 +11,7 @@
 #![feature(associated_type_bounds)]
 #![feature(auto_traits)]
 #![feature(cell_leak)]
+#![feature(core_intrinsics)]
 #![feature(extend_one)]
 #![feature(hash_raw_entry)]
 #![feature(hasher_prefixfree_extras)]

diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs
index 393f173908128457c7f70f01c8eddbbc8240c7cf..3aca03f6e5c63cbae99bda4e58c09b05cddcf065 100644 (file)
--- a/compiler/rustc_data_structures/src/profiling.rs
+++ b/compiler/rustc_data_structures/src/profiling.rs
@@ -88,6 +88,7 @@
 use std::collections::hash_map::Entry;
 use std::error::Error;
 use std::fs;
+use std::intrinsics::unlikely;
 use std::path::Path;
 use std::process;
 use std::sync::Arc;
@@ -395,11 +396,18 @@ pub fn query_provider(&self) -> TimingGuard<'_> {
     /// Record a query in-memory cache hit.
     #[inline(always)]
     pub fn query_cache_hit(&self, query_invocation_id: QueryInvocationId) {
-        self.instant_query_event(
-            |profiler| profiler.query_cache_hit_event_kind,
-            query_invocation_id,
-            EventFilter::QUERY_CACHE_HITS,
-        );
+        #[inline(never)]
+        #[cold]
+        fn cold_call(profiler_ref: &SelfProfilerRef, query_invocation_id: QueryInvocationId) {
+            profiler_ref.instant_query_event(
+                |profiler| profiler.query_cache_hit_event_kind,
+                query_invocation_id,
+            );
+        }
+
+        if unlikely(self.event_filter_mask.contains(EventFilter::QUERY_CACHE_HITS)) {
+            cold_call(self, query_invocation_id);
+        }
     }
 
     /// Start profiling a query being blocked on a concurrent execution.
@@ -444,20 +452,15 @@ fn instant_query_event(
         &self,
         event_kind: fn(&SelfProfiler) -> StringId,
         query_invocation_id: QueryInvocationId,
-        event_filter: EventFilter,
     ) {
-        drop(self.exec(event_filter, |profiler| {
-            let event_id = StringId::new_virtual(query_invocation_id.0);
-            let thread_id = get_thread_id();
-
-            profiler.profiler.record_instant_event(
-                event_kind(profiler),
-                EventId::from_virtual(event_id),
-                thread_id,
-            );
-
-            TimingGuard::none()
-        }));
+        let event_id = StringId::new_virtual(query_invocation_id.0);
+        let thread_id = get_thread_id();
+        let profiler = self.profiler.as_ref().unwrap();
+        profiler.profiler.record_instant_event(
+            event_kind(profiler),
+            EventId::from_virtual(event_id),
+            thread_id,
+        );
     }
 
     pub fn with_profiler(&self, f: impl FnOnce(&SelfProfiler)) {

diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 47b2fd8f8f47a4d31da672b7e76c5de3b69d5d2f..9443ded704d63059c8ae48cdd7acc859876ecc74 100644 (file)
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -47,7 +47,7 @@ impl DepNodeIndex {
 }
 
 impl From<DepNodeIndex> for QueryInvocationId {
-    #[inline]
+    #[inline(always)]
     fn from(dep_node_index: DepNodeIndex) -> Self {
         QueryInvocationId(dep_node_index.as_u32())
     }

diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index ffc413d15f52b0528d7c294b554be089940df5fc..f59d71124ecbf4452d6ada8139d557ac3cfd2c10 100644 (file)
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -346,9 +346,7 @@ pub fn try_get_cached<Tcx, C>(tcx: Tcx, cache: &C, key: &C::Key) -> Option<C::St
 {
     match cache.lookup(&key) {
         Some((value, index)) => {
-            if std::intrinsics::unlikely(tcx.profiler().enabled()) {
-                tcx.profiler().query_cache_hit(index.into());
-            }
+            tcx.profiler().query_cache_hit(index.into());
             tcx.dep_graph().read_index(index);
             Some(value)
         }
@@ -408,9 +406,7 @@ fn try_execute_query<Q, Qcx>(
                 panic!("value must be in cache after waiting")
             };
 
-            if std::intrinsics::unlikely(qcx.dep_context().profiler().enabled()) {
-                qcx.dep_context().profiler().query_cache_hit(index.into());
-            }
+            qcx.dep_context().profiler().query_cache_hit(index.into());
             query_blocked_prof_timer.finish_with_query_invocation_id(index.into());
 
             (v, Some(index))
@@ -776,9 +772,7 @@ pub fn force_query<Q, Qcx, D>(qcx: Qcx, key: Q::Key, dep_node: DepNode<Qcx::DepK
     // Ensure that only one of them runs the query.
     let cache = Q::query_cache(qcx);
     if let Some((_, index)) = cache.lookup(&key) {
-        if std::intrinsics::unlikely(qcx.dep_context().profiler().enabled()) {
-            qcx.dep_context().profiler().query_cache_hit(index.into());
-        }
+        qcx.dep_context().profiler().query_cache_hit(index.into());
         return;
     }