1 //! Machinery for hygienic macros.
3 //! Inspired by Matthew Flatt et al., “Macros That Work Together: Compile-Time Bindings, Partial
4 //! Expansion, and Definition Contexts,” *Journal of Functional Programming* 22, no. 2
5 //! (March 1, 2012): 181–216, <https://doi.org/10.1017/S0956796812000093>.
7 // Hygiene data is stored in a global variable and accessed via TLS, which
8 // means that accesses are somewhat expensive. (`HygieneData::with`
9 // encapsulates a single access.) Therefore, on hot code paths it is worth
10 // ensuring that multiple HygieneData accesses are combined into a single
11 // `HygieneData::with`.
13 // This explains why `HygieneData`, `SyntaxContext` and `ExpnId` have interfaces
14 // with a certain amount of redundancy in them. For example,
15 // `SyntaxContext::outer_expn_data` combines `SyntaxContext::outer` and
16 // `ExpnId::expn_data` so that two `HygieneData` accesses can be performed within
17 // a single `HygieneData::with` call.
19 // It also explains why many functions appear in `HygieneData` and again in
20 // `SyntaxContext` or `ExpnId`. For example, `HygieneData::outer` and
21 // `SyntaxContext::outer` do the same thing, but the former is for use within a
22 // `HygieneData::with` call while the latter is for use outside such a call.
23 // When modifying this file it is important to understand this distinction,
24 // because getting it wrong can lead to nested `HygieneData::with` calls that
25 // trigger runtime aborts. (Fortunately these are obvious and easy to fix.)
27 use crate::edition::Edition;
28 use crate::symbol::{kw, sym, Symbol};
29 use crate::with_session_globals;
30 use crate::{HashStableContext, Span, DUMMY_SP};
32 use crate::def_id::{CrateNum, DefId, CRATE_DEF_INDEX, LOCAL_CRATE};
33 use rustc_data_structures::fingerprint::Fingerprint;
34 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
35 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
36 use rustc_data_structures::sync::{Lock, Lrc};
37 use rustc_data_structures::unhash::UnhashMap;
38 use rustc_macros::HashStable_Generic;
39 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
44 /// A `SyntaxContext` represents a chain of pairs `(ExpnId, Transparency)` named "marks".
45 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
46 pub struct SyntaxContext(u32);
48 #[derive(Debug, Encodable, Decodable, Clone)]
49 pub struct SyntaxContextData {
51 outer_transparency: Transparency,
52 parent: SyntaxContext,
53 /// This context, but with all transparent and semi-transparent expansions filtered away.
54 opaque: SyntaxContext,
55 /// This context, but with all transparent expansions filtered away.
56 opaque_and_semitransparent: SyntaxContext,
57 /// Name of the crate to which `$crate` with this context would resolve.
58 dollar_crate_name: Symbol,
61 /// A unique ID associated with a macro invocation and expansion.
62 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
63 pub struct ExpnId(u32);
65 /// A unique hash value associated to an expansion.
66 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Encodable, Decodable, HashStable_Generic)]
67 pub struct ExpnHash(Fingerprint);
69 /// A property of a macro expansion that determines how identifiers
70 /// produced by that expansion are resolved.
71 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug, Encodable, Decodable)]
72 #[derive(HashStable_Generic)]
73 pub enum Transparency {
74 /// Identifier produced by a transparent expansion is always resolved at call-site.
75 /// Call-site spans in procedural macros, hygiene opt-out in `macro` should use this.
77 /// Identifier produced by a semi-transparent expansion may be resolved
78 /// either at call-site or at definition-site.
79 /// If it's a local variable, label or `$crate` then it's resolved at def-site.
80 /// Otherwise it's resolved at call-site.
81 /// `macro_rules` macros behave like this, built-in macros currently behave like this too,
82 /// but that's an implementation detail.
84 /// Identifier produced by an opaque expansion is always resolved at definition-site.
85 /// Def-site spans in procedural macros, identifiers from `macro` by default use this.
90 pub fn fresh_empty() -> Self {
91 HygieneData::with(|data| data.fresh_expn(None))
94 pub fn fresh(expn_data: ExpnData, ctx: impl HashStableContext) -> Self {
95 let expn_id = HygieneData::with(|data| data.fresh_expn(Some(expn_data)));
96 update_disambiguator(expn_id, ctx);
100 /// The ID of the theoretical expansion that generates freshly parsed, unexpanded AST.
102 pub fn root() -> Self {
107 pub fn as_u32(self) -> u32 {
112 pub fn from_u32(raw: u32) -> ExpnId {
117 pub fn expn_hash(self) -> ExpnHash {
118 HygieneData::with(|data| data.expn_hash(self))
122 pub fn from_hash(hash: ExpnHash) -> Option<ExpnId> {
123 HygieneData::with(|data| data.expn_hash_to_expn_id.get(&hash).copied())
127 pub fn expn_data(self) -> ExpnData {
128 HygieneData::with(|data| data.expn_data(self).clone())
132 pub fn set_expn_data(self, mut expn_data: ExpnData, ctx: impl HashStableContext) {
133 HygieneData::with(|data| {
134 let old_expn_data = &mut data.expn_data[self.0 as usize];
135 assert!(old_expn_data.is_none(), "expansion data is reset for an expansion ID");
136 assert_eq!(expn_data.orig_id, None);
137 expn_data.orig_id = Some(self.as_u32());
138 *old_expn_data = Some(expn_data);
140 update_disambiguator(self, ctx)
143 pub fn is_descendant_of(self, ancestor: ExpnId) -> bool {
144 HygieneData::with(|data| data.is_descendant_of(self, ancestor))
147 /// `expn_id.outer_expn_is_descendant_of(ctxt)` is equivalent to but faster than
148 /// `expn_id.is_descendant_of(ctxt.outer_expn())`.
149 pub fn outer_expn_is_descendant_of(self, ctxt: SyntaxContext) -> bool {
150 HygieneData::with(|data| data.is_descendant_of(self, data.outer_expn(ctxt)))
153 /// Returns span for the macro which originally caused this expansion to happen.
155 /// Stops backtracing at include! boundary.
156 pub fn expansion_cause(mut self) -> Option<Span> {
157 let mut last_macro = None;
159 let expn_data = self.expn_data();
160 // Stop going up the backtrace once include! is encountered
161 if expn_data.is_root()
162 || expn_data.kind == ExpnKind::Macro(MacroKind::Bang, sym::include)
166 self = expn_data.call_site.ctxt().outer_expn();
167 last_macro = Some(expn_data.call_site);
174 pub struct HygieneData {
175 /// Each expansion should have an associated expansion data, but sometimes there's a delay
176 /// between creation of an expansion ID and obtaining its data (e.g. macros are collected
177 /// first and then resolved later), so we use an `Option` here.
178 expn_data: Vec<Option<ExpnData>>,
179 expn_hashes: Vec<ExpnHash>,
180 expn_hash_to_expn_id: UnhashMap<ExpnHash, ExpnId>,
181 syntax_context_data: Vec<SyntaxContextData>,
182 syntax_context_map: FxHashMap<(SyntaxContext, ExpnId, Transparency), SyntaxContext>,
183 /// Maps the `Fingerprint` of an `ExpnData` to the next disambiguator value.
184 /// This is used by `update_disambiguator` to keep track of which `ExpnData`s
185 /// would have collisions without a disambiguator.
186 /// The keys of this map are always computed with `ExpnData.disambiguator`
188 expn_data_disambiguators: FxHashMap<Fingerprint, u32>,
192 crate fn new(edition: Edition) -> Self {
193 let mut root_data = ExpnData::default(
197 Some(DefId::local(CRATE_DEF_INDEX)),
200 root_data.orig_id = Some(0);
203 expn_data: vec![Some(root_data)],
204 expn_hashes: vec![ExpnHash(Fingerprint::ZERO)],
205 expn_hash_to_expn_id: std::iter::once((ExpnHash(Fingerprint::ZERO), ExpnId(0)))
207 syntax_context_data: vec![SyntaxContextData {
208 outer_expn: ExpnId::root(),
209 outer_transparency: Transparency::Opaque,
210 parent: SyntaxContext(0),
211 opaque: SyntaxContext(0),
212 opaque_and_semitransparent: SyntaxContext(0),
213 dollar_crate_name: kw::DollarCrate,
215 syntax_context_map: FxHashMap::default(),
216 expn_data_disambiguators: FxHashMap::default(),
220 pub fn with<T, F: FnOnce(&mut HygieneData) -> T>(f: F) -> T {
221 with_session_globals(|session_globals| f(&mut *session_globals.hygiene_data.borrow_mut()))
224 fn fresh_expn(&mut self, mut expn_data: Option<ExpnData>) -> ExpnId {
225 let raw_id = self.expn_data.len() as u32;
226 if let Some(data) = expn_data.as_mut() {
227 assert_eq!(data.orig_id, None);
228 data.orig_id = Some(raw_id);
230 self.expn_data.push(expn_data);
231 self.expn_hashes.push(ExpnHash(Fingerprint::ZERO));
236 fn expn_hash(&self, expn_id: ExpnId) -> ExpnHash {
237 self.expn_hashes[expn_id.0 as usize]
240 fn expn_data(&self, expn_id: ExpnId) -> &ExpnData {
241 self.expn_data[expn_id.0 as usize].as_ref().expect("no expansion data for an expansion ID")
244 fn is_descendant_of(&self, mut expn_id: ExpnId, ancestor: ExpnId) -> bool {
245 while expn_id != ancestor {
246 if expn_id == ExpnId::root() {
249 expn_id = self.expn_data(expn_id).parent;
254 fn normalize_to_macros_2_0(&self, ctxt: SyntaxContext) -> SyntaxContext {
255 self.syntax_context_data[ctxt.0 as usize].opaque
258 fn normalize_to_macro_rules(&self, ctxt: SyntaxContext) -> SyntaxContext {
259 self.syntax_context_data[ctxt.0 as usize].opaque_and_semitransparent
262 fn outer_expn(&self, ctxt: SyntaxContext) -> ExpnId {
263 self.syntax_context_data[ctxt.0 as usize].outer_expn
266 fn outer_mark(&self, ctxt: SyntaxContext) -> (ExpnId, Transparency) {
267 let data = &self.syntax_context_data[ctxt.0 as usize];
268 (data.outer_expn, data.outer_transparency)
271 fn parent_ctxt(&self, ctxt: SyntaxContext) -> SyntaxContext {
272 self.syntax_context_data[ctxt.0 as usize].parent
275 fn remove_mark(&self, ctxt: &mut SyntaxContext) -> (ExpnId, Transparency) {
276 let outer_mark = self.outer_mark(*ctxt);
277 *ctxt = self.parent_ctxt(*ctxt);
281 fn marks(&self, mut ctxt: SyntaxContext) -> Vec<(ExpnId, Transparency)> {
282 let mut marks = Vec::new();
283 while ctxt != SyntaxContext::root() {
284 debug!("marks: getting parent of {:?}", ctxt);
285 marks.push(self.outer_mark(ctxt));
286 ctxt = self.parent_ctxt(ctxt);
292 fn walk_chain(&self, mut span: Span, to: SyntaxContext) -> Span {
293 debug!("walk_chain({:?}, {:?})", span, to);
294 debug!("walk_chain: span ctxt = {:?}", span.ctxt());
295 while span.from_expansion() && span.ctxt() != to {
296 let outer_expn = self.outer_expn(span.ctxt());
297 debug!("walk_chain({:?}): outer_expn={:?}", span, outer_expn);
298 let expn_data = self.expn_data(outer_expn);
299 debug!("walk_chain({:?}): expn_data={:?}", span, expn_data);
300 span = expn_data.call_site;
305 fn adjust(&self, ctxt: &mut SyntaxContext, expn_id: ExpnId) -> Option<ExpnId> {
306 let mut scope = None;
307 while !self.is_descendant_of(expn_id, self.outer_expn(*ctxt)) {
308 scope = Some(self.remove_mark(ctxt).0);
317 transparency: Transparency,
319 assert_ne!(expn_id, ExpnId::root());
320 if transparency == Transparency::Opaque {
321 return self.apply_mark_internal(ctxt, expn_id, transparency);
324 let call_site_ctxt = self.expn_data(expn_id).call_site.ctxt();
325 let mut call_site_ctxt = if transparency == Transparency::SemiTransparent {
326 self.normalize_to_macros_2_0(call_site_ctxt)
328 self.normalize_to_macro_rules(call_site_ctxt)
331 if call_site_ctxt == SyntaxContext::root() {
332 return self.apply_mark_internal(ctxt, expn_id, transparency);
335 // Otherwise, `expn_id` is a macros 1.0 definition and the call site is in a
336 // macros 2.0 expansion, i.e., a macros 1.0 invocation is in a macros 2.0 definition.
338 // In this case, the tokens from the macros 1.0 definition inherit the hygiene
339 // at their invocation. That is, we pretend that the macros 1.0 definition
340 // was defined at its invocation (i.e., inside the macros 2.0 definition)
341 // so that the macros 2.0 definition remains hygienic.
343 // See the example at `test/ui/hygiene/legacy_interaction.rs`.
344 for (expn_id, transparency) in self.marks(ctxt) {
345 call_site_ctxt = self.apply_mark_internal(call_site_ctxt, expn_id, transparency);
347 self.apply_mark_internal(call_site_ctxt, expn_id, transparency)
350 fn apply_mark_internal(
354 transparency: Transparency,
356 let syntax_context_data = &mut self.syntax_context_data;
357 let mut opaque = syntax_context_data[ctxt.0 as usize].opaque;
358 let mut opaque_and_semitransparent =
359 syntax_context_data[ctxt.0 as usize].opaque_and_semitransparent;
361 if transparency >= Transparency::Opaque {
365 .entry((parent, expn_id, transparency))
367 let new_opaque = SyntaxContext(syntax_context_data.len() as u32);
368 syntax_context_data.push(SyntaxContextData {
370 outer_transparency: transparency,
373 opaque_and_semitransparent: new_opaque,
374 dollar_crate_name: kw::DollarCrate,
380 if transparency >= Transparency::SemiTransparent {
381 let parent = opaque_and_semitransparent;
382 opaque_and_semitransparent = *self
384 .entry((parent, expn_id, transparency))
386 let new_opaque_and_semitransparent =
387 SyntaxContext(syntax_context_data.len() as u32);
388 syntax_context_data.push(SyntaxContextData {
390 outer_transparency: transparency,
393 opaque_and_semitransparent: new_opaque_and_semitransparent,
394 dollar_crate_name: kw::DollarCrate,
396 new_opaque_and_semitransparent
401 *self.syntax_context_map.entry((parent, expn_id, transparency)).or_insert_with(|| {
402 let new_opaque_and_semitransparent_and_transparent =
403 SyntaxContext(syntax_context_data.len() as u32);
404 syntax_context_data.push(SyntaxContextData {
406 outer_transparency: transparency,
409 opaque_and_semitransparent,
410 dollar_crate_name: kw::DollarCrate,
412 new_opaque_and_semitransparent_and_transparent
417 pub fn clear_syntax_context_map() {
418 HygieneData::with(|data| data.syntax_context_map = FxHashMap::default());
421 pub fn walk_chain(span: Span, to: SyntaxContext) -> Span {
422 HygieneData::with(|data| data.walk_chain(span, to))
425 pub fn update_dollar_crate_names(mut get_name: impl FnMut(SyntaxContext) -> Symbol) {
426 // The new contexts that need updating are at the end of the list and have `$crate` as a name.
427 let (len, to_update) = HygieneData::with(|data| {
429 data.syntax_context_data.len(),
430 data.syntax_context_data
433 .take_while(|scdata| scdata.dollar_crate_name == kw::DollarCrate)
437 // The callback must be called from outside of the `HygieneData` lock,
438 // since it will try to acquire it too.
439 let range_to_update = len - to_update..len;
441 range_to_update.clone().map(|idx| get_name(SyntaxContext::from_u32(idx as u32))).collect();
442 HygieneData::with(|data| {
443 range_to_update.zip(names).for_each(|(idx, name)| {
444 data.syntax_context_data[idx].dollar_crate_name = name;
449 pub fn debug_hygiene_data(verbose: bool) -> String {
450 HygieneData::with(|data| {
452 format!("{:#?}", data)
454 let mut s = String::from("");
455 s.push_str("Expansions:");
456 data.expn_data.iter().enumerate().for_each(|(id, expn_info)| {
457 let expn_info = expn_info.as_ref().expect("no expansion data for an expansion ID");
459 "\n{}: parent: {:?}, call_site_ctxt: {:?}, def_site_ctxt: {:?}, kind: {:?}",
462 expn_info.call_site.ctxt(),
463 expn_info.def_site.ctxt(),
467 s.push_str("\n\nSyntaxContexts:");
468 data.syntax_context_data.iter().enumerate().for_each(|(id, ctxt)| {
470 "\n#{}: parent: {:?}, outer_mark: ({:?}, {:?})",
471 id, ctxt.parent, ctxt.outer_expn, ctxt.outer_transparency,
481 pub const fn root() -> Self {
486 crate fn as_u32(self) -> u32 {
491 crate fn from_u32(raw: u32) -> SyntaxContext {
495 /// Extend a syntax context with a given expansion and transparency.
496 crate fn apply_mark(self, expn_id: ExpnId, transparency: Transparency) -> SyntaxContext {
497 HygieneData::with(|data| data.apply_mark(self, expn_id, transparency))
500 /// Pulls a single mark off of the syntax context. This effectively moves the
501 /// context up one macro definition level. That is, if we have a nested macro
502 /// definition as follows:
512 /// and we have a SyntaxContext that is referring to something declared by an invocation
513 /// of g (call it g1), calling remove_mark will result in the SyntaxContext for the
514 /// invocation of f that created g1.
515 /// Returns the mark that was removed.
516 pub fn remove_mark(&mut self) -> ExpnId {
517 HygieneData::with(|data| data.remove_mark(self).0)
520 pub fn marks(self) -> Vec<(ExpnId, Transparency)> {
521 HygieneData::with(|data| data.marks(self))
524 /// Adjust this context for resolution in a scope created by the given expansion.
525 /// For example, consider the following three resolutions of `f`:
528 /// mod foo { pub fn f() {} } // `f`'s `SyntaxContext` is empty.
530 /// macro m($f:ident) {
532 /// pub fn f() {} // `f`'s `SyntaxContext` has a single `ExpnId` from `m`.
533 /// pub fn $f() {} // `$f`'s `SyntaxContext` is empty.
535 /// foo::f(); // `f`'s `SyntaxContext` has a single `ExpnId` from `m`
536 /// //^ Since `mod foo` is outside this expansion, `adjust` removes the mark from `f`,
537 /// //| and it resolves to `::foo::f`.
538 /// bar::f(); // `f`'s `SyntaxContext` has a single `ExpnId` from `m`
539 /// //^ Since `mod bar` not outside this expansion, `adjust` does not change `f`,
540 /// //| and it resolves to `::bar::f`.
541 /// bar::$f(); // `f`'s `SyntaxContext` is empty.
542 /// //^ Since `mod bar` is not outside this expansion, `adjust` does not change `$f`,
543 /// //| and it resolves to `::bar::$f`.
546 /// This returns the expansion whose definition scope we use to privacy check the resolution,
547 /// or `None` if we privacy check as usual (i.e., not w.r.t. a macro definition scope).
548 pub fn adjust(&mut self, expn_id: ExpnId) -> Option<ExpnId> {
549 HygieneData::with(|data| data.adjust(self, expn_id))
552 /// Like `SyntaxContext::adjust`, but also normalizes `self` to macros 2.0.
553 pub fn normalize_to_macros_2_0_and_adjust(&mut self, expn_id: ExpnId) -> Option<ExpnId> {
554 HygieneData::with(|data| {
555 *self = data.normalize_to_macros_2_0(*self);
556 data.adjust(self, expn_id)
560 /// Adjust this context for resolution in a scope created by the given expansion
561 /// via a glob import with the given `SyntaxContext`.
566 /// macro m($i:ident) {
568 /// pub fn f() {} // `f`'s `SyntaxContext` has a single `ExpnId` from `m`.
569 /// pub fn $i() {} // `$i`'s `SyntaxContext` is empty.
572 /// macro n($j:ident) {
574 /// f(); // `f`'s `SyntaxContext` has a mark from `m` and a mark from `n`
575 /// //^ `glob_adjust` removes the mark from `n`, so this resolves to `foo::f`.
576 /// $i(); // `$i`'s `SyntaxContext` has a mark from `n`
577 /// //^ `glob_adjust` removes the mark from `n`, so this resolves to `foo::$i`.
578 /// $j(); // `$j`'s `SyntaxContext` has a mark from `m`
579 /// //^ This cannot be glob-adjusted, so this is a resolution error.
583 /// This returns `None` if the context cannot be glob-adjusted.
584 /// Otherwise, it returns the scope to use when privacy checking (see `adjust` for details).
585 pub fn glob_adjust(&mut self, expn_id: ExpnId, glob_span: Span) -> Option<Option<ExpnId>> {
586 HygieneData::with(|data| {
587 let mut scope = None;
588 let mut glob_ctxt = data.normalize_to_macros_2_0(glob_span.ctxt());
589 while !data.is_descendant_of(expn_id, data.outer_expn(glob_ctxt)) {
590 scope = Some(data.remove_mark(&mut glob_ctxt).0);
591 if data.remove_mark(self).0 != scope.unwrap() {
595 if data.adjust(self, expn_id).is_some() {
602 /// Undo `glob_adjust` if possible:
605 /// if let Some(privacy_checking_scope) = self.reverse_glob_adjust(expansion, glob_ctxt) {
606 /// assert!(self.glob_adjust(expansion, glob_ctxt) == Some(privacy_checking_scope));
609 pub fn reverse_glob_adjust(
613 ) -> Option<Option<ExpnId>> {
614 HygieneData::with(|data| {
615 if data.adjust(self, expn_id).is_some() {
619 let mut glob_ctxt = data.normalize_to_macros_2_0(glob_span.ctxt());
620 let mut marks = Vec::new();
621 while !data.is_descendant_of(expn_id, data.outer_expn(glob_ctxt)) {
622 marks.push(data.remove_mark(&mut glob_ctxt));
625 let scope = marks.last().map(|mark| mark.0);
626 while let Some((expn_id, transparency)) = marks.pop() {
627 *self = data.apply_mark(*self, expn_id, transparency);
633 pub fn hygienic_eq(self, other: SyntaxContext, expn_id: ExpnId) -> bool {
634 HygieneData::with(|data| {
635 let mut self_normalized = data.normalize_to_macros_2_0(self);
636 data.adjust(&mut self_normalized, expn_id);
637 self_normalized == data.normalize_to_macros_2_0(other)
642 pub fn normalize_to_macros_2_0(self) -> SyntaxContext {
643 HygieneData::with(|data| data.normalize_to_macros_2_0(self))
647 pub fn normalize_to_macro_rules(self) -> SyntaxContext {
648 HygieneData::with(|data| data.normalize_to_macro_rules(self))
652 pub fn outer_expn(self) -> ExpnId {
653 HygieneData::with(|data| data.outer_expn(self))
656 /// `ctxt.outer_expn_data()` is equivalent to but faster than
657 /// `ctxt.outer_expn().expn_data()`.
659 pub fn outer_expn_data(self) -> ExpnData {
660 HygieneData::with(|data| data.expn_data(data.outer_expn(self)).clone())
664 pub fn outer_mark(self) -> (ExpnId, Transparency) {
665 HygieneData::with(|data| data.outer_mark(self))
668 pub fn dollar_crate_name(self) -> Symbol {
669 HygieneData::with(|data| data.syntax_context_data[self.0 as usize].dollar_crate_name)
672 pub fn edition(self) -> Edition {
673 HygieneData::with(|data| data.expn_data(data.outer_expn(self)).edition)
677 impl fmt::Debug for SyntaxContext {
678 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
679 write!(f, "#{}", self.0)
684 /// Creates a fresh expansion with given properties.
685 /// Expansions are normally created by macros, but in some cases expansions are created for
686 /// other compiler-generated code to set per-span properties like allowed unstable features.
687 /// The returned span belongs to the created expansion and has the new properties,
688 /// but its location is inherited from the current span.
689 pub fn fresh_expansion(self, expn_data: ExpnData, ctx: impl HashStableContext) -> Span {
690 self.fresh_expansion_with_transparency(expn_data, Transparency::Transparent, ctx)
693 pub fn fresh_expansion_with_transparency(
696 transparency: Transparency,
697 ctx: impl HashStableContext,
699 let expn_id = ExpnId::fresh(expn_data, ctx);
700 HygieneData::with(|data| {
701 self.with_ctxt(data.apply_mark(SyntaxContext::root(), expn_id, transparency))
705 /// Reuses the span but adds information like the kind of the desugaring and features that are
706 /// allowed inside this span.
707 pub fn mark_with_reason(
709 allow_internal_unstable: Option<Lrc<[Symbol]>>,
710 reason: DesugaringKind,
712 ctx: impl HashStableContext,
714 let expn_data = ExpnData {
715 allow_internal_unstable,
716 ..ExpnData::default(ExpnKind::Desugaring(reason), self, edition, None, None)
718 self.fresh_expansion(expn_data, ctx)
722 /// A subset of properties from both macro definition and macro call available through global data.
723 /// Avoid using this if you have access to the original definition or call structures.
724 #[derive(Clone, Debug, Encodable, Decodable, HashStable_Generic)]
725 pub struct ExpnData {
726 // --- The part unique to each expansion.
727 /// The kind of this expansion - macro or compiler desugaring.
729 /// The expansion that produced this expansion.
731 /// The location of the actual macro invocation or syntax sugar , e.g.
732 /// `let x = foo!();` or `if let Some(y) = x {}`
734 /// This may recursively refer to other macro invocations, e.g., if
735 /// `foo!()` invoked `bar!()` internally, and there was an
736 /// expression inside `bar!`; the call_site of the expression in
737 /// the expansion would point to the `bar!` invocation; that
738 /// call_site span would have its own ExpnData, with the call_site
739 /// pointing to the `foo!` invocation.
741 /// The crate that originally created this `ExpnData`. During
742 /// metadata serialization, we only encode `ExpnData`s that were
743 /// created locally - when our serialized metadata is decoded,
744 /// foreign `ExpnId`s will have their `ExpnData` looked up
745 /// from the crate specified by `Crate
747 /// The raw that this `ExpnData` had in its original crate.
748 /// An `ExpnData` can be created before being assigned an `ExpnId`,
749 /// so this might be `None` until `set_expn_data` is called
750 // This is used only for serialization/deserialization purposes:
751 // two `ExpnData`s that differ only in their `orig_id` should
752 // be considered equivalent.
753 #[stable_hasher(ignore)]
754 orig_id: Option<u32>,
755 /// Used to force two `ExpnData`s to have different `Fingerprint`s.
756 /// Due to macro expansion, it's possible to end up with two `ExpnId`s
757 /// that have identical `ExpnData`s. This violates the contract of `HashStable`
758 /// - the two `ExpnId`s are not equal, but their `Fingerprint`s are equal
759 /// (since the numerical `ExpnId` value is not considered by the `HashStable`
762 /// The `disambiguator` field is set by `update_disambiguator` when two distinct
763 /// `ExpnId`s would end up with the same `Fingerprint`. Since `ExpnData` includes
764 /// a `krate` field, this value only needs to be unique within a single crate.
767 // --- The part specific to the macro/desugaring definition.
768 // --- It may be reasonable to share this part between expansions with the same definition,
769 // --- but such sharing is known to bring some minor inconveniences without also bringing
770 // --- noticeable perf improvements (PR #62898).
771 /// The span of the macro definition (possibly dummy).
772 /// This span serves only informational purpose and is not used for resolution.
774 /// List of `#[unstable]`/feature-gated features that the macro is allowed to use
775 /// internally without forcing the whole crate to opt-in
777 pub allow_internal_unstable: Option<Lrc<[Symbol]>>,
778 /// Whether the macro is allowed to use `unsafe` internally
779 /// even if the user crate has `#![forbid(unsafe_code)]`.
780 pub allow_internal_unsafe: bool,
781 /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`)
782 /// for a given macro.
783 pub local_inner_macros: bool,
784 /// Edition of the crate in which the macro is defined.
785 pub edition: Edition,
786 /// The `DefId` of the macro being invoked,
787 /// if this `ExpnData` corresponds to a macro invocation
788 pub macro_def_id: Option<DefId>,
789 /// The normal module (`mod`) in which the expanded macro was defined.
790 pub parent_module: Option<DefId>,
793 // These would require special handling of `orig_id`.
794 impl !PartialEq for ExpnData {}
795 impl !Hash for ExpnData {}
803 allow_internal_unstable: Option<Lrc<[Symbol]>>,
804 allow_internal_unsafe: bool,
805 local_inner_macros: bool,
807 macro_def_id: Option<DefId>,
808 parent_module: Option<DefId>,
815 allow_internal_unstable,
816 allow_internal_unsafe,
827 /// Constructs expansion data with default properties.
832 macro_def_id: Option<DefId>,
833 parent_module: Option<DefId>,
837 parent: ExpnId::root(),
840 allow_internal_unstable: None,
841 allow_internal_unsafe: false,
842 local_inner_macros: false,
852 pub fn allow_unstable(
856 allow_internal_unstable: Lrc<[Symbol]>,
857 macro_def_id: Option<DefId>,
858 parent_module: Option<DefId>,
861 allow_internal_unstable: Some(allow_internal_unstable),
862 ..ExpnData::default(kind, call_site, edition, macro_def_id, parent_module)
867 pub fn is_root(&self) -> bool {
868 matches!(self.kind, ExpnKind::Root)
872 fn hash_expn(&self, ctx: &mut impl HashStableContext) -> Fingerprint {
873 let mut hasher = StableHasher::new();
874 self.hash_stable(ctx, &mut hasher);
880 #[derive(Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
882 /// No expansion, aka root expansion. Only `ExpnId::root()` has this kind.
884 /// Expansion produced by a macro.
885 Macro(MacroKind, Symbol),
886 /// Transform done by the compiler on the AST.
888 /// Desugaring done by the compiler during HIR lowering.
889 Desugaring(DesugaringKind),
895 pub fn descr(&self) -> String {
897 ExpnKind::Root => kw::PathRoot.to_string(),
898 ExpnKind::Macro(macro_kind, name) => match macro_kind {
899 MacroKind::Bang => format!("{}!", name),
900 MacroKind::Attr => format!("#[{}]", name),
901 MacroKind::Derive => format!("#[derive({})]", name),
903 ExpnKind::AstPass(kind) => kind.descr().to_string(),
904 ExpnKind::Desugaring(kind) => format!("desugaring of {}", kind.descr()),
905 ExpnKind::Inlined => "inlined source".to_string(),
910 /// The kind of macro invocation or definition.
911 #[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
912 #[derive(HashStable_Generic)]
914 /// A bang macro `foo!()`.
916 /// An attribute macro `#[foo]`.
918 /// A derive macro `#[derive(Foo)]`
923 pub fn descr(self) -> &'static str {
925 MacroKind::Bang => "macro",
926 MacroKind::Attr => "attribute macro",
927 MacroKind::Derive => "derive macro",
931 pub fn descr_expected(self) -> &'static str {
933 MacroKind::Attr => "attribute",
938 pub fn article(self) -> &'static str {
940 MacroKind::Attr => "an",
946 /// The kind of AST transform.
947 #[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
955 pub fn descr(self) -> &'static str {
957 AstPass::StdImports => "standard library imports",
958 AstPass::TestHarness => "test harness",
959 AstPass::ProcMacroHarness => "proc macro harness",
964 /// The kind of compiler desugaring.
965 #[derive(Clone, Copy, PartialEq, Debug, Encodable, Decodable, HashStable_Generic)]
966 pub enum DesugaringKind {
967 /// We desugar `if c { i } else { e }` to `match $ExprKind::Use(c) { true => i, _ => e }`.
968 /// However, we do not want to blame `c` for unreachability but rather say that `i`
969 /// is unreachable. This desugaring kind allows us to avoid blaming `c`.
970 /// This also applies to `while` loops.
974 /// Desugaring of an `impl Trait` in return type position
975 /// to an `type Foo = impl Trait;` and replacing the
976 /// `impl Trait` with `Foo`.
983 /// A location in the desugaring of a `for` loop
984 #[derive(Clone, Copy, PartialEq, Debug, Encodable, Decodable, HashStable_Generic)]
985 pub enum ForLoopLoc {
990 impl DesugaringKind {
991 /// The description wording should combine well with "desugaring of {}".
992 pub fn descr(self) -> &'static str {
994 DesugaringKind::CondTemporary => "`if` or `while` condition",
995 DesugaringKind::Async => "`async` block or function",
996 DesugaringKind::Await => "`await` expression",
997 DesugaringKind::QuestionMark => "operator `?`",
998 DesugaringKind::TryBlock => "`try` block",
999 DesugaringKind::OpaqueTy => "`impl Trait`",
1000 DesugaringKind::ForLoop(_) => "`for` loop",
1006 pub struct HygieneEncodeContext {
1007 /// All `SyntaxContexts` for which we have written `SyntaxContextData` into crate metadata.
1008 /// This is `None` after we finish encoding `SyntaxContexts`, to ensure
1009 /// that we don't accidentally try to encode any more `SyntaxContexts`
1010 serialized_ctxts: Lock<FxHashSet<SyntaxContext>>,
1011 /// The `SyntaxContexts` that we have serialized (e.g. as a result of encoding `Spans`)
1012 /// in the most recent 'round' of serializnig. Serializing `SyntaxContextData`
1013 /// may cause us to serialize more `SyntaxContext`s, so serialize in a loop
1014 /// until we reach a fixed point.
1015 latest_ctxts: Lock<FxHashSet<SyntaxContext>>,
1017 serialized_expns: Lock<FxHashSet<ExpnId>>,
1019 latest_expns: Lock<FxHashSet<ExpnId>>,
1022 impl HygieneEncodeContext {
1023 pub fn encode<T, R>(
1026 mut encode_ctxt: impl FnMut(&mut T, u32, &SyntaxContextData) -> Result<(), R>,
1027 mut encode_expn: impl FnMut(&mut T, u32, ExpnData, ExpnHash) -> Result<(), R>,
1028 ) -> Result<(), R> {
1029 // When we serialize a `SyntaxContextData`, we may end up serializing
1030 // a `SyntaxContext` that we haven't seen before
1031 while !self.latest_ctxts.lock().is_empty() || !self.latest_expns.lock().is_empty() {
1033 "encode_hygiene: Serializing a round of {:?} SyntaxContextDatas: {:?}",
1034 self.latest_ctxts.lock().len(),
1038 // Consume the current round of SyntaxContexts.
1039 // Drop the lock() temporary early
1040 let latest_ctxts = { std::mem::take(&mut *self.latest_ctxts.lock()) };
1042 // It's fine to iterate over a HashMap, because the serialization
1043 // of the table that we insert data into doesn't depend on insertion
1045 for_all_ctxts_in(latest_ctxts.into_iter(), |index, ctxt, data| {
1046 if self.serialized_ctxts.lock().insert(ctxt) {
1047 encode_ctxt(encoder, index, data)?;
1052 let latest_expns = { std::mem::take(&mut *self.latest_expns.lock()) };
1054 for_all_expns_in(latest_expns.into_iter(), |index, expn, data, hash| {
1055 if self.serialized_expns.lock().insert(expn) {
1056 encode_expn(encoder, index, data, hash)?;
1061 debug!("encode_hygiene: Done serializing SyntaxContextData");
1067 /// Additional information used to assist in decoding hygiene data
1068 pub struct HygieneDecodeContext {
1069 // Maps serialized `SyntaxContext` ids to a `SyntaxContext` in the current
1070 // global `HygieneData`. When we deserialize a `SyntaxContext`, we need to create
1071 // a new id in the global `HygieneData`. This map tracks the ID we end up picking,
1072 // so that multiple occurrences of the same serialized id are decoded to the same
1074 remapped_ctxts: Lock<Vec<Option<SyntaxContext>>>,
1075 // The same as `remapepd_ctxts`, but for `ExpnId`s
1076 remapped_expns: Lock<Vec<Option<ExpnId>>>,
1079 pub fn decode_expn_id<'a, D: Decoder, G>(
1081 mode: ExpnDataDecodeMode<'a, G>,
1082 decode_data: impl FnOnce(&mut D, u32) -> Result<(ExpnData, ExpnHash), D::Error>,
1083 ) -> Result<ExpnId, D::Error>
1085 G: FnOnce(CrateNum) -> &'a HygieneDecodeContext,
1087 let index = u32::decode(d)?;
1088 let context = match mode {
1089 ExpnDataDecodeMode::IncrComp(context) => context,
1090 ExpnDataDecodeMode::Metadata(get_context) => {
1091 let krate = CrateNum::decode(d)?;
1096 // Do this after decoding, so that we decode a `CrateNum`
1098 if index == ExpnId::root().as_u32() {
1099 debug!("decode_expn_id: deserialized root");
1100 return Ok(ExpnId::root());
1103 let outer_expns = &context.remapped_expns;
1105 // Ensure that the lock() temporary is dropped early
1107 if let Some(expn_id) = outer_expns.lock().get(index as usize).copied().flatten() {
1112 // Don't decode the data inside `HygieneData::with`, since we need to recursively decode
1114 let (mut expn_data, hash) = decode_data(d, index)?;
1116 let expn_id = HygieneData::with(|hygiene_data| {
1117 if let Some(&expn_id) = hygiene_data.expn_hash_to_expn_id.get(&hash) {
1121 let expn_id = ExpnId(hygiene_data.expn_data.len() as u32);
1123 // If we just deserialized an `ExpnData` owned by
1124 // the local crate, its `orig_id` will be stale,
1125 // so we need to update it to its own value.
1126 // This only happens when we deserialize the incremental cache,
1127 // since a crate will never decode its own metadata.
1128 if expn_data.krate == LOCAL_CRATE {
1129 expn_data.orig_id = Some(expn_id.0);
1132 hygiene_data.expn_data.push(Some(expn_data));
1133 hygiene_data.expn_hashes.push(hash);
1134 let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id);
1135 debug_assert!(_old_id.is_none());
1137 let mut expns = outer_expns.lock();
1138 let new_len = index as usize + 1;
1139 if expns.len() < new_len {
1140 expns.resize(new_len, None);
1142 expns[index as usize] = Some(expn_id);
1149 // Decodes `SyntaxContext`, using the provided `HygieneDecodeContext`
1150 // to track which `SyntaxContext`s we have already decoded.
1151 // The provided closure will be invoked to deserialize a `SyntaxContextData`
1152 // if we haven't already seen the id of the `SyntaxContext` we are deserializing.
1153 pub fn decode_syntax_context<
1155 F: FnOnce(&mut D, u32) -> Result<SyntaxContextData, D::Error>,
1158 context: &HygieneDecodeContext,
1160 ) -> Result<SyntaxContext, D::Error> {
1161 let raw_id: u32 = Decodable::decode(d)?;
1163 debug!("decode_syntax_context: deserialized root");
1164 // The root is special
1165 return Ok(SyntaxContext::root());
1168 let outer_ctxts = &context.remapped_ctxts;
1170 // Ensure that the lock() temporary is dropped early
1172 if let Some(ctxt) = outer_ctxts.lock().get(raw_id as usize).copied().flatten() {
1177 // Allocate and store SyntaxContext id *before* calling the decoder function,
1178 // as the SyntaxContextData may reference itself.
1179 let new_ctxt = HygieneData::with(|hygiene_data| {
1180 let new_ctxt = SyntaxContext(hygiene_data.syntax_context_data.len() as u32);
1181 // Push a dummy SyntaxContextData to ensure that nobody else can get the
1182 // same ID as us. This will be overwritten after call `decode_Data`
1183 hygiene_data.syntax_context_data.push(SyntaxContextData {
1184 outer_expn: ExpnId::root(),
1185 outer_transparency: Transparency::Transparent,
1186 parent: SyntaxContext::root(),
1187 opaque: SyntaxContext::root(),
1188 opaque_and_semitransparent: SyntaxContext::root(),
1189 dollar_crate_name: kw::Empty,
1191 let mut ctxts = outer_ctxts.lock();
1192 let new_len = raw_id as usize + 1;
1193 if ctxts.len() < new_len {
1194 ctxts.resize(new_len, None);
1196 ctxts[raw_id as usize] = Some(new_ctxt);
1201 // Don't try to decode data while holding the lock, since we need to
1202 // be able to recursively decode a SyntaxContext
1203 let mut ctxt_data = decode_data(d, raw_id)?;
1204 // Reset `dollar_crate_name` so that it will be updated by `update_dollar_crate_names`
1205 // We don't care what the encoding crate set this to - we want to resolve it
1206 // from the perspective of the current compilation session
1207 ctxt_data.dollar_crate_name = kw::DollarCrate;
1209 // Overwrite the dummy data with our decoded SyntaxContextData
1210 HygieneData::with(|hygiene_data| {
1211 let dummy = std::mem::replace(
1212 &mut hygiene_data.syntax_context_data[new_ctxt.as_u32() as usize],
1215 // Make sure nothing weird happening while `decode_data` was running
1216 assert_eq!(dummy.dollar_crate_name, kw::Empty);
1222 fn for_all_ctxts_in<E, F: FnMut(u32, SyntaxContext, &SyntaxContextData) -> Result<(), E>>(
1223 ctxts: impl Iterator<Item = SyntaxContext>,
1225 ) -> Result<(), E> {
1226 let all_data: Vec<_> = HygieneData::with(|data| {
1227 ctxts.map(|ctxt| (ctxt, data.syntax_context_data[ctxt.0 as usize].clone())).collect()
1229 for (ctxt, data) in all_data.into_iter() {
1230 f(ctxt.0, ctxt, &data)?;
1235 fn for_all_expns_in<E>(
1236 expns: impl Iterator<Item = ExpnId>,
1237 mut f: impl FnMut(u32, ExpnId, ExpnData, ExpnHash) -> Result<(), E>,
1238 ) -> Result<(), E> {
1239 let all_data: Vec<_> = HygieneData::with(|data| {
1242 let idx = expn.0 as usize;
1243 (expn, data.expn_data[idx].clone(), data.expn_hashes[idx].clone())
1247 for (expn, data, hash) in all_data.into_iter() {
1248 let data = data.unwrap_or_else(|| panic!("Missing data for {:?}", expn));
1249 f(expn.0, expn, data, hash)?;
1254 impl<E: Encoder> Encodable<E> for ExpnId {
1255 default fn encode(&self, _: &mut E) -> Result<(), E::Error> {
1256 panic!("cannot encode `ExpnId` with `{}`", std::any::type_name::<E>());
1260 impl<D: Decoder> Decodable<D> for ExpnId {
1261 default fn decode(_: &mut D) -> Result<Self, D::Error> {
1262 panic!("cannot decode `ExpnId` with `{}`", std::any::type_name::<D>());
1266 pub fn raw_encode_syntax_context<E: Encoder>(
1267 ctxt: SyntaxContext,
1268 context: &HygieneEncodeContext,
1270 ) -> Result<(), E::Error> {
1271 if !context.serialized_ctxts.lock().contains(&ctxt) {
1272 context.latest_ctxts.lock().insert(ctxt);
1277 pub fn raw_encode_expn_id<E: Encoder>(
1279 context: &HygieneEncodeContext,
1280 mode: ExpnDataEncodeMode,
1282 ) -> Result<(), E::Error> {
1283 // Record the fact that we need to serialize the corresponding
1285 let needs_data = || {
1286 if !context.serialized_expns.lock().contains(&expn) {
1287 context.latest_expns.lock().insert(expn);
1292 ExpnDataEncodeMode::IncrComp => {
1293 // Always serialize the `ExpnData` in incr comp mode
1297 ExpnDataEncodeMode::Metadata => {
1298 let data = expn.expn_data();
1299 // We only need to serialize the ExpnData
1300 // if it comes from this crate.
1301 // We currently don't serialize any hygiene information data for
1302 // proc-macro crates: see the `SpecializedEncoder<Span>` impl
1303 // for crate metadata.
1304 if data.krate == LOCAL_CRATE {
1307 data.orig_id.expect("Missing orig_id").encode(e)?;
1308 data.krate.encode(e)
1313 pub enum ExpnDataEncodeMode {
1318 pub enum ExpnDataDecodeMode<'a, F: FnOnce(CrateNum) -> &'a HygieneDecodeContext> {
1319 IncrComp(&'a HygieneDecodeContext),
1323 impl<'a> ExpnDataDecodeMode<'a, Box<dyn FnOnce(CrateNum) -> &'a HygieneDecodeContext>> {
1324 pub fn incr_comp(ctxt: &'a HygieneDecodeContext) -> Self {
1325 ExpnDataDecodeMode::IncrComp(ctxt)
1329 impl<E: Encoder> Encodable<E> for SyntaxContext {
1330 default fn encode(&self, _: &mut E) -> Result<(), E::Error> {
1331 panic!("cannot encode `SyntaxContext` with `{}`", std::any::type_name::<E>());
1335 impl<D: Decoder> Decodable<D> for SyntaxContext {
1336 default fn decode(_: &mut D) -> Result<Self, D::Error> {
1337 panic!("cannot decode `SyntaxContext` with `{}`", std::any::type_name::<D>());
1341 /// Updates the `disambiguator` field of the corresponding `ExpnData`
1342 /// such that the `Fingerprint` of the `ExpnData` does not collide with
1343 /// any other `ExpnIds`.
1345 /// This method is called only when an `ExpnData` is first associated
1346 /// with an `ExpnId` (when the `ExpnId` is initially constructed, or via
1347 /// `set_expn_data`). It is *not* called for foreign `ExpnId`s deserialized
1348 /// from another crate's metadata - since `ExpnData` includes a `krate` field,
1349 /// collisions are only possible between `ExpnId`s within the same crate.
1350 fn update_disambiguator(expn_id: ExpnId, mut ctx: impl HashStableContext) {
1351 let mut expn_data = expn_id.expn_data();
1352 // This disambiguator should not have been set yet.
1354 expn_data.disambiguator, 0,
1355 "Already set disambiguator for ExpnData: {:?}",
1358 let mut expn_hash = expn_data.hash_expn(&mut ctx);
1360 let disambiguator = HygieneData::with(|data| {
1361 // If this is the first ExpnData with a given hash, then keep our
1362 // disambiguator at 0 (the default u32 value)
1363 let disambig = data.expn_data_disambiguators.entry(expn_hash).or_default();
1364 let disambiguator = *disambig;
1369 if disambiguator != 0 {
1370 debug!("Set disambiguator for {:?} (hash {:?})", expn_id, expn_hash);
1371 debug!("expn_data = {:?}", expn_data);
1373 expn_data.disambiguator = disambiguator;
1374 expn_hash = expn_data.hash_expn(&mut ctx);
1376 // Verify that the new disambiguator makes the hash unique
1377 #[cfg(debug_assertions)]
1378 HygieneData::with(|data| {
1380 data.expn_data_disambiguators.get(&expn_hash),
1382 "Hash collision after disambiguator update!",
1387 let expn_hash = ExpnHash(expn_hash);
1388 HygieneData::with(|data| {
1389 data.expn_data[expn_id.0 as usize].as_mut().unwrap().disambiguator = disambiguator;
1390 debug_assert_eq!(data.expn_hashes[expn_id.0 as usize].0, Fingerprint::ZERO);
1391 data.expn_hashes[expn_id.0 as usize] = expn_hash;
1392 let _old_id = data.expn_hash_to_expn_id.insert(expn_hash, expn_id);
1393 debug_assert!(_old_id.is_none());
1397 impl<CTX: HashStableContext> HashStable<CTX> for SyntaxContext {
1398 fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
1399 const TAG_EXPANSION: u8 = 0;
1400 const TAG_NO_EXPANSION: u8 = 1;
1402 if *self == SyntaxContext::root() {
1403 TAG_NO_EXPANSION.hash_stable(ctx, hasher);
1405 TAG_EXPANSION.hash_stable(ctx, hasher);
1406 let (expn_id, transparency) = self.outer_mark();
1407 expn_id.hash_stable(ctx, hasher);
1408 transparency.hash_stable(ctx, hasher);
1413 impl<CTX: HashStableContext> HashStable<CTX> for ExpnId {
1414 fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
1415 let hash = if *self == ExpnId::root() {
1416 // Avoid fetching TLS storage for a trivial often-used value.
1422 hash.hash_stable(ctx, hasher);