1 //! `hir_expand` deals with macro expansion.
3 //! Specifically, it implements a concept of `MacroFile` -- a file whose syntax
4 //! tree originates not from the text of some `FileId`, but from some macro
11 pub mod builtin_attr_macro;
12 pub mod builtin_derive_macro;
13 pub mod builtin_fn_macro;
18 use base_db::ProcMacroKind;
21 pub use mbe::{ExpandError, ExpandResult};
23 use std::{hash::Hash, iter, sync::Arc};
25 use base_db::{impl_intern_key, salsa, CrateId, FileId, FileRange};
27 algo::skip_trivia_token,
28 ast::{self, AstNode, HasAttrs},
29 Direction, SyntaxNode, SyntaxToken, TextRange,
33 ast_id_map::FileAstId,
34 builtin_attr_macro::BuiltinAttrExpander,
35 builtin_derive_macro::BuiltinDeriveExpander,
36 builtin_fn_macro::{BuiltinFnLikeExpander, EagerExpander},
38 proc_macro::ProcMacroExpander,
44 /// Input to the analyzer is a set of files, where each file is identified by
45 /// `FileId` and contains source code. However, another source of source code in
46 /// Rust are macros: each macro can be thought of as producing a "temporary
47 /// file". To assign an id to such a file, we use the id of the macro call that
48 /// produced the file. So, a `HirFileId` is either a `FileId` (source code
49 /// written by user), or a `MacroCallId` (source code produced by macro).
51 /// What is a `MacroCallId`? Simplifying, it's a `HirFileId` of a file
52 /// containing the call plus the offset of the macro call in the file. Note that
53 /// this is a recursive definition! However, the size_of of `HirFileId` is
54 /// finite (because everything bottoms out at the real `FileId`) and small
55 /// (`MacroCallId` uses the location interning. You can check details here:
56 /// <https://en.wikipedia.org/wiki/String_interning>).
57 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58 pub struct HirFileId(HirFileIdRepr);
60 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
66 impl From<FileId> for HirFileId {
67 fn from(id: FileId) -> Self {
68 HirFileId(HirFileIdRepr::FileId(id))
72 impl From<MacroFile> for HirFileId {
73 fn from(id: MacroFile) -> Self {
74 HirFileId(HirFileIdRepr::MacroFile(id))
79 /// For macro-expansion files, returns the file original source file the
80 /// expansion originated from.
81 pub fn original_file(self, db: &dyn db::AstDatabase) -> FileId {
83 HirFileIdRepr::FileId(file_id) => file_id,
84 HirFileIdRepr::MacroFile(macro_file) => {
85 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
86 let file_id = match &loc.eager {
87 Some(EagerCallInfo { included_file: Some(file), .. }) => (*file).into(),
88 _ => loc.kind.file_id(),
90 file_id.original_file(db)
95 pub fn expansion_level(self, db: &dyn db::AstDatabase) -> u32 {
98 while let HirFileIdRepr::MacroFile(macro_file) = curr.0 {
99 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
102 curr = loc.kind.file_id();
107 /// If this is a macro call, returns the syntax node of the call.
108 pub fn call_node(self, db: &dyn db::AstDatabase) -> Option<InFile<SyntaxNode>> {
110 HirFileIdRepr::FileId(_) => None,
111 HirFileIdRepr::MacroFile(macro_file) => {
112 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
113 Some(loc.kind.to_node(db))
118 /// Return expansion information if it is a macro-expansion file
119 pub fn expansion_info(self, db: &dyn db::AstDatabase) -> Option<ExpansionInfo> {
121 HirFileIdRepr::FileId(_) => None,
122 HirFileIdRepr::MacroFile(macro_file) => {
123 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
125 let arg_tt = loc.kind.arg(db)?;
127 let def = loc.def.ast_id().left().and_then(|id| {
128 let def_tt = match id.to_node(db) {
129 ast::Macro::MacroRules(mac) => mac.token_tree()?,
130 ast::Macro::MacroDef(mac) => mac.body()?,
132 Some(InFile::new(id.file_id, def_tt))
134 let attr_input_or_mac_def = def.or_else(|| match loc.kind {
135 MacroCallKind::Attr { ast_id, invoc_attr_index, .. } => {
139 .nth(invoc_attr_index as usize)?
141 Some(InFile::new(ast_id.file_id, tt))
146 let macro_def = db.macro_def(loc.def).ok()?;
147 let (parse, exp_map) = db.parse_macro_expansion(macro_file).value?;
148 let macro_arg = db.macro_arg(macro_file.macro_call_id)?;
151 expanded: InFile::new(self, parse.syntax_node()),
152 arg: InFile::new(loc.kind.file_id(), arg_tt),
153 attr_input_or_mac_def,
154 macro_arg_shift: mbe::Shift::new(¯o_arg.0),
163 /// Indicate it is macro file generated for builtin derive
164 pub fn is_builtin_derive(&self, db: &dyn db::AstDatabase) -> Option<InFile<ast::Item>> {
166 HirFileIdRepr::FileId(_) => None,
167 HirFileIdRepr::MacroFile(macro_file) => {
168 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
169 let item = match loc.def.kind {
170 MacroDefKind::BuiltInDerive(..) => loc.kind.to_node(db),
173 Some(item.with_value(ast::Item::cast(item.value.clone())?))
178 /// Return whether this file is an include macro
179 pub fn is_include_macro(&self, db: &dyn db::AstDatabase) -> bool {
181 HirFileIdRepr::MacroFile(macro_file) => {
182 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
183 matches!(loc.eager, Some(EagerCallInfo { included_file: Some(_), .. }))
189 /// Return whether this file is an include macro
190 pub fn is_attr_macro(&self, db: &dyn db::AstDatabase) -> bool {
192 HirFileIdRepr::MacroFile(macro_file) => {
193 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
194 matches!(loc.kind, MacroCallKind::Attr { .. })
200 pub fn is_macro(self) -> bool {
201 matches!(self.0, HirFileIdRepr::MacroFile(_))
205 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
206 pub struct MacroFile {
207 pub macro_call_id: MacroCallId,
210 /// `MacroCallId` identifies a particular macro invocation, like
211 /// `println!("Hello, {}", world)`.
212 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
213 pub struct MacroCallId(salsa::InternId);
214 impl_intern_key!(MacroCallId);
216 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
217 pub struct MacroDefId {
219 pub kind: MacroDefKind,
221 pub local_inner: bool,
225 pub fn as_lazy_macro(
227 db: &dyn db::AstDatabase,
231 db.intern_macro(MacroCallLoc { def: self, krate, eager: None, kind })
234 pub fn ast_id(&self) -> Either<AstId<ast::Macro>, AstId<ast::Fn>> {
235 let id = match &self.kind {
236 MacroDefKind::ProcMacro(.., id) => return Either::Right(*id),
237 MacroDefKind::Declarative(id)
238 | MacroDefKind::BuiltIn(_, id)
239 | MacroDefKind::BuiltInAttr(_, id)
240 | MacroDefKind::BuiltInDerive(_, id)
241 | MacroDefKind::BuiltInEager(_, id) => id,
246 pub fn is_proc_macro(&self) -> bool {
247 matches!(self.kind, MacroDefKind::ProcMacro(..))
251 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
252 pub enum MacroDefKind {
253 Declarative(AstId<ast::Macro>),
254 BuiltIn(BuiltinFnLikeExpander, AstId<ast::Macro>),
255 // FIXME: maybe just Builtin and rename BuiltinFnLikeExpander to BuiltinExpander
256 BuiltInAttr(BuiltinAttrExpander, AstId<ast::Macro>),
257 BuiltInDerive(BuiltinDeriveExpander, AstId<ast::Macro>),
258 BuiltInEager(EagerExpander, AstId<ast::Macro>),
259 ProcMacro(ProcMacroExpander, ProcMacroKind, AstId<ast::Fn>),
262 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
263 struct EagerCallInfo {
264 /// NOTE: This can be *either* the expansion result, *or* the argument to the eager macro!
265 arg_or_expansion: Arc<tt::Subtree>,
266 included_file: Option<FileId>,
269 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
270 pub struct MacroCallLoc {
272 pub(crate) krate: CrateId,
273 eager: Option<EagerCallInfo>,
274 pub kind: MacroCallKind,
277 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
278 pub enum MacroCallKind {
280 ast_id: AstId<ast::MacroCall>,
284 ast_id: AstId<ast::Item>,
286 /// Syntactical index of the invoking `#[derive]` attribute.
288 /// Outer attributes are counted first, then inner attributes. This does not support
289 /// out-of-line modules, which may have attributes spread across 2 files!
290 derive_attr_index: u32,
293 ast_id: AstId<ast::Item>,
295 attr_args: (tt::Subtree, mbe::TokenMap),
296 /// Syntactical index of the invoking `#[attribute]`.
298 /// Outer attributes are counted first, then inner attributes. This does not support
299 /// out-of-line modules, which may have attributes spread across 2 files!
300 invoc_attr_index: u32,
304 // FIXME: attribute indices do not account for `cfg_attr`, which means that we'll strip the whole
305 // `cfg_attr` instead of just one of the attributes it expands to
308 /// Returns the file containing the macro invocation.
309 fn file_id(&self) -> HirFileId {
311 MacroCallKind::FnLike { ast_id, .. } => ast_id.file_id,
312 MacroCallKind::Derive { ast_id, .. } | MacroCallKind::Attr { ast_id, .. } => {
318 pub fn to_node(&self, db: &dyn db::AstDatabase) -> InFile<SyntaxNode> {
320 MacroCallKind::FnLike { ast_id, .. } => {
321 ast_id.with_value(ast_id.to_node(db).syntax().clone())
323 MacroCallKind::Derive { ast_id, .. } | MacroCallKind::Attr { ast_id, .. } => {
324 ast_id.with_value(ast_id.to_node(db).syntax().clone())
329 fn arg(&self, db: &dyn db::AstDatabase) -> Option<SyntaxNode> {
331 MacroCallKind::FnLike { ast_id, .. } => {
332 Some(ast_id.to_node(db).token_tree()?.syntax().clone())
334 MacroCallKind::Derive { ast_id, .. } | MacroCallKind::Attr { ast_id, .. } => {
335 Some(ast_id.to_node(db).syntax().clone())
340 fn expand_to(&self) -> ExpandTo {
342 MacroCallKind::FnLike { expand_to, .. } => *expand_to,
343 MacroCallKind::Derive { .. } => ExpandTo::Items,
344 MacroCallKind::Attr { .. } => ExpandTo::Items, // is this always correct?
350 pub fn as_file(self) -> HirFileId {
351 MacroFile { macro_call_id: self }.into()
355 /// ExpansionInfo mainly describes how to map text range between src and expanded macro
356 #[derive(Debug, Clone, PartialEq, Eq)]
357 pub struct ExpansionInfo {
358 expanded: InFile<SyntaxNode>,
359 arg: InFile<SyntaxNode>,
360 /// The `macro_rules!` arguments or attribute input.
361 attr_input_or_mac_def: Option<InFile<ast::TokenTree>>,
363 macro_def: Arc<TokenExpander>,
364 macro_arg: Arc<(tt::Subtree, mbe::TokenMap)>,
365 macro_arg_shift: mbe::Shift,
366 exp_map: Arc<mbe::TokenMap>,
372 pub fn call_node(&self) -> Option<InFile<SyntaxNode>> {
373 Some(self.arg.with_value(self.arg.value.parent()?))
376 pub fn map_token_down(
378 db: &dyn db::AstDatabase,
379 item: Option<ast::Item>,
380 token: InFile<&SyntaxToken>,
381 ) -> Option<impl Iterator<Item = InFile<SyntaxToken>> + '_> {
382 assert_eq!(token.file_id, self.arg.file_id);
383 let token_id = if let Some(item) = item {
384 // check if we are mapping down in an attribute input
385 let call_id = match self.expanded.file_id.0 {
386 HirFileIdRepr::FileId(_) => return None,
387 HirFileIdRepr::MacroFile(macro_file) => macro_file.macro_call_id,
389 let loc = db.lookup_intern_macro(call_id);
391 let token_range = token.value.text_range();
393 MacroCallKind::Attr { attr_args, invoc_attr_index, .. } => {
394 let attr = item.attrs().nth(*invoc_attr_index as usize)?;
395 match attr.token_tree() {
397 if token_tree.syntax().text_range().contains_range(token_range) =>
399 let attr_input_start =
400 token_tree.left_delimiter_token()?.text_range().start();
401 let range = token.value.text_range().checked_sub(attr_input_start)?;
403 self.macro_arg_shift.shift(attr_args.1.token_by_range(range)?);
415 let token_id = match token_id {
416 Some(token_id) => token_id,
419 token.value.text_range().checked_sub(self.arg.value.text_range().start())?;
420 let token_id = self.macro_arg.1.token_by_range(range)?;
421 self.macro_def.map_id_down(token_id)
427 .ranges_by_token(token_id, token.value.kind())
428 .flat_map(move |range| self.expanded.value.covering_element(range).into_token());
430 Some(tokens.map(move |token| self.expanded.with_value(token)))
435 db: &dyn db::AstDatabase,
436 token: InFile<&SyntaxToken>,
437 ) -> Option<(InFile<SyntaxToken>, Origin)> {
438 let token_id = self.exp_map.token_by_range(token.value.text_range())?;
439 let (mut token_id, origin) = self.macro_def.map_id_up(token_id);
441 let call_id = match self.expanded.file_id.0 {
442 HirFileIdRepr::FileId(_) => return None,
443 HirFileIdRepr::MacroFile(macro_file) => macro_file.macro_call_id,
445 let loc = db.lookup_intern_macro(call_id);
447 let (token_map, tt) = match &loc.kind {
448 MacroCallKind::Attr { attr_args, .. } => match self.macro_arg_shift.unshift(token_id) {
450 token_id = unshifted;
451 (&attr_args.1, self.attr_input_or_mac_def.clone()?.syntax().cloned())
453 None => (&self.macro_arg.1, self.arg.clone()),
456 mbe::Origin::Call => (&self.macro_arg.1, self.arg.clone()),
457 mbe::Origin::Def => match (&*self.macro_def, &self.attr_input_or_mac_def) {
459 TokenExpander::MacroRules { def_site_token_map, .. }
460 | TokenExpander::MacroDef { def_site_token_map, .. },
462 ) => (def_site_token_map, tt.syntax().cloned()),
463 _ => panic!("`Origin::Def` used with non-`macro_rules!` macro"),
468 let range = token_map.first_range_by_token(token_id, token.value.kind())?;
470 tt.value.covering_element(range + tt.value.text_range().start()).into_token()?;
471 Some((tt.with_value(token), origin))
475 /// `AstId` points to an AST node in any file.
477 /// It is stable across reparses, and can be used as salsa key/value.
478 // FIXME: isn't this just a `Source<FileAstId<N>>` ?
479 pub type AstId<N> = InFile<FileAstId<N>>;
481 impl<N: AstNode> AstId<N> {
482 pub fn to_node(&self, db: &dyn db::AstDatabase) -> N {
483 let root = db.parse_or_expand(self.file_id).unwrap();
484 db.ast_id_map(self.file_id).get(self.value).to_node(&root)
488 /// `InFile<T>` stores a value of `T` inside a particular file/syntax tree.
490 /// Typical usages are:
492 /// * `InFile<SyntaxNode>` -- syntax node in a file
493 /// * `InFile<ast::FnDef>` -- ast node in a file
494 /// * `InFile<TextSize>` -- offset in a file
495 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
496 pub struct InFile<T> {
497 pub file_id: HirFileId,
502 pub fn new(file_id: HirFileId, value: T) -> InFile<T> {
503 InFile { file_id, value }
506 // Similarly, naming here is stupid...
507 pub fn with_value<U>(&self, value: U) -> InFile<U> {
508 InFile::new(self.file_id, value)
511 pub fn map<F: FnOnce(T) -> U, U>(self, f: F) -> InFile<U> {
512 InFile::new(self.file_id, f(self.value))
514 pub fn as_ref(&self) -> InFile<&T> {
515 self.with_value(&self.value)
517 pub fn file_syntax(&self, db: &dyn db::AstDatabase) -> SyntaxNode {
518 db.parse_or_expand(self.file_id).expect("source created from invalid file")
522 impl<T: Clone> InFile<&T> {
523 pub fn cloned(&self) -> InFile<T> {
524 self.with_value(self.value.clone())
528 impl<T> InFile<Option<T>> {
529 pub fn transpose(self) -> Option<InFile<T>> {
530 let value = self.value?;
531 Some(InFile::new(self.file_id, value))
535 impl InFile<SyntaxNode> {
536 pub fn ancestors_with_macros(
538 db: &dyn db::AstDatabase,
539 ) -> impl Iterator<Item = InFile<SyntaxNode>> + Clone + '_ {
540 iter::successors(Some(self), move |node| match node.value.parent() {
541 Some(parent) => Some(node.with_value(parent)),
543 let parent_node = node.file_id.call_node(db)?;
549 /// Skips the attributed item that caused the macro invocation we are climbing up
550 pub fn ancestors_with_macros_skip_attr_item(
552 db: &dyn db::AstDatabase,
553 ) -> impl Iterator<Item = InFile<SyntaxNode>> + '_ {
554 iter::successors(Some(self), move |node| match node.value.parent() {
555 Some(parent) => Some(node.with_value(parent)),
557 let parent_node = node.file_id.call_node(db)?;
558 if node.file_id.is_attr_macro(db) {
559 // macro call was an attributed item, skip it
560 // FIXME: does this fail if this is a direct expansion of another macro?
561 parent_node.map(|node| node.parent()).transpose()
570 impl<'a> InFile<&'a SyntaxNode> {
571 /// Falls back to the macro call range if the node cannot be mapped up fully.
572 pub fn original_file_range(self, db: &dyn db::AstDatabase) -> FileRange {
573 if let Some(res) = self.original_file_range_opt(db) {
577 // Fall back to whole macro call.
578 let mut node = self.cloned();
579 while let Some(call_node) = node.file_id.call_node(db) {
583 let orig_file = node.file_id.original_file(db);
584 assert_eq!(node.file_id, orig_file.into());
586 FileRange { file_id: orig_file, range: node.value.text_range() }
589 /// Attempts to map the syntax node back up its macro calls.
590 pub fn original_file_range_opt(self, db: &dyn db::AstDatabase) -> Option<FileRange> {
591 match original_range_opt(db, self) {
593 let original_file = range.file_id.original_file(db);
594 if range.file_id != original_file.into() {
595 tracing::error!("Failed mapping up more for {:?}", range);
597 Some(FileRange { file_id: original_file, range: range.value })
599 _ if !self.file_id.is_macro() => Some(FileRange {
600 file_id: self.file_id.original_file(db),
601 range: self.value.text_range(),
608 fn original_range_opt(
609 db: &dyn db::AstDatabase,
610 node: InFile<&SyntaxNode>,
611 ) -> Option<InFile<TextRange>> {
612 let expansion = node.file_id.expansion_info(db)?;
614 // the input node has only one token ?
615 let single = skip_trivia_token(node.value.first_token()?, Direction::Next)?
616 == skip_trivia_token(node.value.last_token()?, Direction::Prev)?;
618 node.value.descendants().find_map(|it| {
619 let first = skip_trivia_token(it.first_token()?, Direction::Next)?;
620 let first = ascend_call_token(db, &expansion, node.with_value(first))?;
622 let last = skip_trivia_token(it.last_token()?, Direction::Prev)?;
623 let last = ascend_call_token(db, &expansion, node.with_value(last))?;
625 if (!single && first == last) || (first.file_id != last.file_id) {
629 Some(first.with_value(first.value.text_range().cover(last.value.text_range())))
633 fn ascend_call_token(
634 db: &dyn db::AstDatabase,
635 expansion: &ExpansionInfo,
636 token: InFile<SyntaxToken>,
637 ) -> Option<InFile<SyntaxToken>> {
638 let (mapped, origin) = expansion.map_token_up(db, token.as_ref())?;
639 if origin != Origin::Call {
642 if let Some(info) = mapped.file_id.expansion_info(db) {
643 return ascend_call_token(db, &info, mapped);
648 impl InFile<SyntaxToken> {
649 pub fn ancestors_with_macros(
651 db: &dyn db::AstDatabase,
652 ) -> impl Iterator<Item = InFile<SyntaxNode>> + '_ {
653 self.value.parent().into_iter().flat_map({
654 let file_id = self.file_id;
655 move |parent| InFile::new(file_id, parent).ancestors_with_macros(db)
660 impl<N: AstNode> InFile<N> {
661 pub fn descendants<T: AstNode>(self) -> impl Iterator<Item = InFile<T>> {
662 self.value.syntax().descendants().filter_map(T::cast).map(move |n| self.with_value(n))
665 pub fn syntax(&self) -> InFile<&SyntaxNode> {
666 self.with_value(self.value.syntax())
669 pub fn nodes_with_attributes<'db>(
671 db: &'db dyn db::AstDatabase,
672 ) -> impl Iterator<Item = InFile<N>> + 'db
676 iter::successors(Some(self), move |node| {
677 let InFile { file_id, value } = node.file_id.call_node(db)?;
678 N::cast(value).map(|n| InFile::new(file_id, n))
682 pub fn node_with_attributes(self, db: &dyn db::AstDatabase) -> InFile<N> {
683 self.nodes_with_attributes(db).last().unwrap()
687 /// In Rust, macros expand token trees to token trees. When we want to turn a
688 /// token tree into an AST node, we need to figure out what kind of AST node we
689 /// want: something like `foo` can be a type, an expression, or a pattern.
691 /// Naively, one would think that "what this expands to" is a property of a
692 /// particular macro: macro `m1` returns an item, while macro `m2` returns an
693 /// expression, etc. That's not the case -- macros are polymorphic in the
694 /// result, and can expand to any type of the AST node.
696 /// What defines the actual AST node is the syntactic context of the macro
697 /// invocation. As a contrived example, in `let T![*] = T![*];` the first `T`
698 /// expands to a pattern, while the second one expands to an expression.
700 /// `ExpandTo` captures this bit of information about a particular macro call
702 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
712 pub fn from_call_site(call: &ast::MacroCall) -> ExpandTo {
713 use syntax::SyntaxKind::*;
715 let syn = call.syntax();
717 let parent = match syn.parent() {
719 None => return ExpandTo::Statements,
722 match parent.kind() {
723 MACRO_ITEMS | SOURCE_FILE | ITEM_LIST => ExpandTo::Items,
724 MACRO_STMTS | EXPR_STMT | STMT_LIST => ExpandTo::Statements,
725 MACRO_PAT => ExpandTo::Pattern,
726 MACRO_TYPE => ExpandTo::Type,
728 ARG_LIST | TRY_EXPR | TUPLE_EXPR | PAREN_EXPR | ARRAY_EXPR | FOR_EXPR | PATH_EXPR
729 | CLOSURE_EXPR | CONDITION | BREAK_EXPR | RETURN_EXPR | MATCH_EXPR | MATCH_ARM
730 | MATCH_GUARD | RECORD_EXPR_FIELD | CALL_EXPR | INDEX_EXPR | METHOD_CALL_EXPR
731 | FIELD_EXPR | AWAIT_EXPR | CAST_EXPR | REF_EXPR | PREFIX_EXPR | RANGE_EXPR
732 | BIN_EXPR => ExpandTo::Expr,
734 // FIXME: Handle LHS Pattern
739 // Unknown , Just guess it is `Items`