1 //! `hir_expand` deals with macro expansion.
3 //! Specifically, it implements a concept of `MacroFile` -- a file whose syntax
4 //! tree originates not from the text of some `FileId`, but from some macro
12 pub mod builtin_derive;
13 pub mod builtin_macro;
18 use base_db::ProcMacroKind;
21 pub use mbe::{ExpandError, ExpandResult};
23 use std::{hash::Hash, iter, sync::Arc};
25 use base_db::{impl_intern_key, salsa, CrateId, FileId, FileRange};
27 algo::skip_trivia_token,
28 ast::{self, AstNode, AttrsOwner},
29 Direction, SyntaxNode, SyntaxToken, TextRange, TextSize,
33 ast_id_map::FileAstId,
34 builtin_attr::BuiltinAttrExpander,
35 builtin_derive::BuiltinDeriveExpander,
36 builtin_macro::{BuiltinFnLikeExpander, EagerExpander},
38 proc_macro::ProcMacroExpander,
44 /// Input to the analyzer is a set of files, where each file is identified by
45 /// `FileId` and contains source code. However, another source of source code in
46 /// Rust are macros: each macro can be thought of as producing a "temporary
47 /// file". To assign an id to such a file, we use the id of the macro call that
48 /// produced the file. So, a `HirFileId` is either a `FileId` (source code
49 /// written by user), or a `MacroCallId` (source code produced by macro).
51 /// What is a `MacroCallId`? Simplifying, it's a `HirFileId` of a file
52 /// containing the call plus the offset of the macro call in the file. Note that
53 /// this is a recursive definition! However, the size_of of `HirFileId` is
54 /// finite (because everything bottoms out at the real `FileId`) and small
55 /// (`MacroCallId` uses the location interning. You can check details here:
56 /// <https://en.wikipedia.org/wiki/String_interning>).
57 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58 pub struct HirFileId(HirFileIdRepr);
60 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
66 impl From<FileId> for HirFileId {
67 fn from(id: FileId) -> Self {
68 HirFileId(HirFileIdRepr::FileId(id))
72 impl From<MacroFile> for HirFileId {
73 fn from(id: MacroFile) -> Self {
74 HirFileId(HirFileIdRepr::MacroFile(id))
79 /// For macro-expansion files, returns the file original source file the
80 /// expansion originated from.
81 pub fn original_file(self, db: &dyn db::AstDatabase) -> FileId {
83 HirFileIdRepr::FileId(file_id) => file_id,
84 HirFileIdRepr::MacroFile(macro_file) => {
85 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
86 let file_id = match &loc.eager {
87 Some(EagerCallInfo { included_file: Some(file), .. }) => (*file).into(),
88 _ => loc.kind.file_id(),
90 file_id.original_file(db)
95 pub fn expansion_level(self, db: &dyn db::AstDatabase) -> u32 {
98 while let HirFileIdRepr::MacroFile(macro_file) = curr.0 {
99 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
102 curr = loc.kind.file_id();
107 /// If this is a macro call, returns the syntax node of the call.
108 pub fn call_node(self, db: &dyn db::AstDatabase) -> Option<InFile<SyntaxNode>> {
110 HirFileIdRepr::FileId(_) => None,
111 HirFileIdRepr::MacroFile(macro_file) => {
112 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
113 Some(loc.kind.to_node(db))
118 /// Return expansion information if it is a macro-expansion file
119 pub fn expansion_info(self, db: &dyn db::AstDatabase) -> Option<ExpansionInfo> {
121 HirFileIdRepr::FileId(_) => None,
122 HirFileIdRepr::MacroFile(macro_file) => {
123 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
125 let arg_tt = loc.kind.arg(db)?;
127 let def = loc.def.ast_id().left().and_then(|id| {
128 let def_tt = match id.to_node(db) {
129 ast::Macro::MacroRules(mac) => mac.token_tree()?,
130 ast::Macro::MacroDef(mac) => mac.body()?,
132 Some(InFile::new(id.file_id, def_tt))
134 let attr_input_or_mac_def = def.or_else(|| match loc.kind {
135 MacroCallKind::Attr { ast_id, invoc_attr_index, .. } => {
139 .nth(invoc_attr_index as usize)?
141 Some(InFile::new(ast_id.file_id, tt))
146 let macro_def = db.macro_def(loc.def)?;
147 let (parse, exp_map) = db.parse_macro_expansion(macro_file).value?;
148 let macro_arg = db.macro_arg(macro_file.macro_call_id)?;
151 expanded: InFile::new(self, parse.syntax_node()),
152 arg: InFile::new(loc.kind.file_id(), arg_tt),
153 attr_input_or_mac_def,
154 macro_arg_shift: mbe::Shift::new(¯o_arg.0),
163 /// Indicate it is macro file generated for builtin derive
164 pub fn is_builtin_derive(&self, db: &dyn db::AstDatabase) -> Option<InFile<ast::Item>> {
166 HirFileIdRepr::FileId(_) => None,
167 HirFileIdRepr::MacroFile(macro_file) => {
168 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
169 let item = match loc.def.kind {
170 MacroDefKind::BuiltInDerive(..) => loc.kind.to_node(db),
173 Some(item.with_value(ast::Item::cast(item.value.clone())?))
178 /// Return whether this file is an include macro
179 pub fn is_include_macro(&self, db: &dyn db::AstDatabase) -> bool {
181 HirFileIdRepr::MacroFile(macro_file) => {
182 let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
183 matches!(loc.eager, Some(EagerCallInfo { included_file: Some(_), .. }))
189 pub fn is_macro(self) -> bool {
190 matches!(self.0, HirFileIdRepr::MacroFile(_))
194 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
195 pub struct MacroFile {
196 macro_call_id: MacroCallId,
199 /// `MacroCallId` identifies a particular macro invocation, like
200 /// `println!("Hello, {}", world)`.
201 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
202 pub struct MacroCallId(salsa::InternId);
203 impl_intern_key!(MacroCallId);
205 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
206 pub struct MacroDefId {
208 pub kind: MacroDefKind,
210 pub local_inner: bool,
214 pub fn as_lazy_macro(
216 db: &dyn db::AstDatabase,
220 db.intern_macro(MacroCallLoc { def: self, krate, eager: None, kind })
223 pub fn ast_id(&self) -> Either<AstId<ast::Macro>, AstId<ast::Fn>> {
224 let id = match &self.kind {
225 MacroDefKind::ProcMacro(.., id) => return Either::Right(*id),
226 MacroDefKind::Declarative(id)
227 | MacroDefKind::BuiltIn(_, id)
228 | MacroDefKind::BuiltInAttr(_, id)
229 | MacroDefKind::BuiltInDerive(_, id)
230 | MacroDefKind::BuiltInEager(_, id) => id,
235 pub fn is_proc_macro(&self) -> bool {
236 matches!(self.kind, MacroDefKind::ProcMacro(..))
240 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
241 pub enum MacroDefKind {
242 Declarative(AstId<ast::Macro>),
243 BuiltIn(BuiltinFnLikeExpander, AstId<ast::Macro>),
244 // FIXME: maybe just Builtin and rename BuiltinFnLikeExpander to BuiltinExpander
245 BuiltInAttr(BuiltinAttrExpander, AstId<ast::Macro>),
246 BuiltInDerive(BuiltinDeriveExpander, AstId<ast::Macro>),
247 BuiltInEager(EagerExpander, AstId<ast::Macro>),
248 ProcMacro(ProcMacroExpander, ProcMacroKind, AstId<ast::Fn>),
251 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
252 struct EagerCallInfo {
253 /// NOTE: This can be *either* the expansion result, *or* the argument to the eager macro!
254 arg_or_expansion: Arc<tt::Subtree>,
255 included_file: Option<FileId>,
258 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
259 pub struct MacroCallLoc {
261 pub(crate) krate: CrateId,
262 eager: Option<EagerCallInfo>,
263 pub kind: MacroCallKind,
266 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
267 pub enum MacroCallKind {
269 ast_id: AstId<ast::MacroCall>,
273 ast_id: AstId<ast::Item>,
275 /// Syntactical index of the invoking `#[derive]` attribute.
277 /// Outer attributes are counted first, then inner attributes. This does not support
278 /// out-of-line modules, which may have attributes spread across 2 files!
279 derive_attr_index: u32,
282 ast_id: AstId<ast::Item>,
284 attr_args: (tt::Subtree, mbe::TokenMap),
285 /// Syntactical index of the invoking `#[attribute]`.
287 /// Outer attributes are counted first, then inner attributes. This does not support
288 /// out-of-line modules, which may have attributes spread across 2 files!
289 invoc_attr_index: u32,
293 // FIXME: attribute indices do not account for `cfg_attr`, which means that we'll strip the whole
294 // `cfg_attr` instead of just one of the attributes it expands to
297 /// Returns the file containing the macro invocation.
298 fn file_id(&self) -> HirFileId {
300 MacroCallKind::FnLike { ast_id, .. } => ast_id.file_id,
301 MacroCallKind::Derive { ast_id, .. } | MacroCallKind::Attr { ast_id, .. } => {
307 pub fn to_node(&self, db: &dyn db::AstDatabase) -> InFile<SyntaxNode> {
309 MacroCallKind::FnLike { ast_id, .. } => {
310 ast_id.with_value(ast_id.to_node(db).syntax().clone())
312 MacroCallKind::Derive { ast_id, .. } | MacroCallKind::Attr { ast_id, .. } => {
313 ast_id.with_value(ast_id.to_node(db).syntax().clone())
318 fn arg(&self, db: &dyn db::AstDatabase) -> Option<SyntaxNode> {
320 MacroCallKind::FnLike { ast_id, .. } => {
321 Some(ast_id.to_node(db).token_tree()?.syntax().clone())
323 MacroCallKind::Derive { ast_id, .. } | MacroCallKind::Attr { ast_id, .. } => {
324 Some(ast_id.to_node(db).syntax().clone())
329 fn expand_to(&self) -> ExpandTo {
331 MacroCallKind::FnLike { expand_to, .. } => *expand_to,
332 MacroCallKind::Derive { .. } => ExpandTo::Items,
333 MacroCallKind::Attr { .. } => ExpandTo::Items, // is this always correct?
339 pub fn as_file(self) -> HirFileId {
340 MacroFile { macro_call_id: self }.into()
344 /// ExpansionInfo mainly describes how to map text range between src and expanded macro
345 #[derive(Debug, Clone, PartialEq, Eq)]
346 pub struct ExpansionInfo {
347 expanded: InFile<SyntaxNode>,
348 arg: InFile<SyntaxNode>,
349 /// The `macro_rules!` arguments or attribute input.
350 attr_input_or_mac_def: Option<InFile<ast::TokenTree>>,
352 macro_def: Arc<TokenExpander>,
353 macro_arg: Arc<(tt::Subtree, mbe::TokenMap)>,
354 macro_arg_shift: mbe::Shift,
355 exp_map: Arc<mbe::TokenMap>,
361 pub fn call_node(&self) -> Option<InFile<SyntaxNode>> {
362 Some(self.arg.with_value(self.arg.value.parent()?))
365 pub fn map_token_down(
367 db: &dyn db::AstDatabase,
368 item: Option<ast::Item>,
369 token: InFile<&SyntaxToken>,
370 ) -> Option<impl Iterator<Item = InFile<SyntaxToken>> + '_> {
371 assert_eq!(token.file_id, self.arg.file_id);
372 let token_id = if let Some(item) = item {
373 // check if we are mapping down in an attribute input
374 let call_id = match self.expanded.file_id.0 {
375 HirFileIdRepr::FileId(_) => return None,
376 HirFileIdRepr::MacroFile(macro_file) => macro_file.macro_call_id,
378 let loc = db.lookup_intern_macro(call_id);
380 let token_range = token.value.text_range();
382 MacroCallKind::Attr { attr_args, invoc_attr_index, .. } => {
383 let attr = item.attrs().nth(*invoc_attr_index as usize)?;
384 match attr.token_tree() {
386 if token_tree.syntax().text_range().contains_range(token_range) =>
388 let attr_input_start =
389 token_tree.left_delimiter_token()?.text_range().start();
390 let range = token.value.text_range().checked_sub(attr_input_start)?;
392 self.macro_arg_shift.shift(attr_args.1.token_by_range(range)?);
404 let token_id = match token_id {
405 Some(token_id) => token_id,
408 token.value.text_range().checked_sub(self.arg.value.text_range().start())?;
409 let token_id = self.macro_arg.1.token_by_range(range)?;
410 self.macro_def.map_id_down(token_id)
416 .ranges_by_token(token_id, token.value.kind())
417 .flat_map(move |range| self.expanded.value.covering_element(range).into_token());
419 Some(tokens.map(move |token| self.expanded.with_value(token)))
424 db: &dyn db::AstDatabase,
425 token: InFile<&SyntaxToken>,
426 ) -> Option<(InFile<SyntaxToken>, Origin)> {
427 let token_id = self.exp_map.token_by_range(token.value.text_range())?;
428 let (mut token_id, origin) = self.macro_def.map_id_up(token_id);
430 let call_id = match self.expanded.file_id.0 {
431 HirFileIdRepr::FileId(_) => return None,
432 HirFileIdRepr::MacroFile(macro_file) => macro_file.macro_call_id,
434 let loc = db.lookup_intern_macro(call_id);
436 let (token_map, tt) = match &loc.kind {
437 MacroCallKind::Attr { attr_args, .. } => match self.macro_arg_shift.unshift(token_id) {
439 token_id = unshifted;
440 (&attr_args.1, self.attr_input_or_mac_def.clone()?.syntax().cloned())
442 None => (&self.macro_arg.1, self.arg.clone()),
445 mbe::Origin::Call => (&self.macro_arg.1, self.arg.clone()),
446 mbe::Origin::Def => match (&*self.macro_def, &self.attr_input_or_mac_def) {
448 TokenExpander::MacroRules { def_site_token_map, .. }
449 | TokenExpander::MacroDef { def_site_token_map, .. },
451 ) => (def_site_token_map, tt.syntax().cloned()),
452 _ => panic!("`Origin::Def` used with non-`macro_rules!` macro"),
457 let range = token_map.first_range_by_token(token_id, token.value.kind())?;
459 tt.value.covering_element(range + tt.value.text_range().start()).into_token()?;
460 Some((tt.with_value(token), origin))
464 /// `AstId` points to an AST node in any file.
466 /// It is stable across reparses, and can be used as salsa key/value.
467 // FIXME: isn't this just a `Source<FileAstId<N>>` ?
468 pub type AstId<N> = InFile<FileAstId<N>>;
470 impl<N: AstNode> AstId<N> {
471 pub fn to_node(&self, db: &dyn db::AstDatabase) -> N {
472 let root = db.parse_or_expand(self.file_id).unwrap();
473 db.ast_id_map(self.file_id).get(self.value).to_node(&root)
477 /// `InFile<T>` stores a value of `T` inside a particular file/syntax tree.
479 /// Typical usages are:
481 /// * `InFile<SyntaxNode>` -- syntax node in a file
482 /// * `InFile<ast::FnDef>` -- ast node in a file
483 /// * `InFile<TextSize>` -- offset in a file
484 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
485 pub struct InFile<T> {
486 pub file_id: HirFileId,
491 pub fn new(file_id: HirFileId, value: T) -> InFile<T> {
492 InFile { file_id, value }
495 // Similarly, naming here is stupid...
496 pub fn with_value<U>(&self, value: U) -> InFile<U> {
497 InFile::new(self.file_id, value)
500 pub fn map<F: FnOnce(T) -> U, U>(self, f: F) -> InFile<U> {
501 InFile::new(self.file_id, f(self.value))
503 pub fn as_ref(&self) -> InFile<&T> {
504 self.with_value(&self.value)
506 pub fn file_syntax(&self, db: &dyn db::AstDatabase) -> SyntaxNode {
507 db.parse_or_expand(self.file_id).expect("source created from invalid file")
511 impl<T: Clone> InFile<&T> {
512 pub fn cloned(&self) -> InFile<T> {
513 self.with_value(self.value.clone())
517 impl<T> InFile<Option<T>> {
518 pub fn transpose(self) -> Option<InFile<T>> {
519 let value = self.value?;
520 Some(InFile::new(self.file_id, value))
524 impl InFile<SyntaxNode> {
525 pub fn ancestors_with_macros(
527 db: &dyn db::AstDatabase,
528 ) -> impl Iterator<Item = InFile<SyntaxNode>> + '_ {
529 iter::successors(Some(self), move |node| match node.value.parent() {
530 Some(parent) => Some(node.with_value(parent)),
532 let parent_node = node.file_id.call_node(db)?;
539 impl<'a> InFile<&'a SyntaxNode> {
540 /// Falls back to the macro call range if the node cannot be mapped up fully.
541 pub fn original_file_range(self, db: &dyn db::AstDatabase) -> FileRange {
542 if let Some(res) = self.original_file_range_opt(db) {
546 // Fall back to whole macro call.
547 let mut node = self.cloned();
548 while let Some(call_node) = node.file_id.call_node(db) {
552 let orig_file = node.file_id.original_file(db);
553 assert_eq!(node.file_id, orig_file.into());
555 FileRange { file_id: orig_file, range: node.value.text_range() }
558 /// Attempts to map the syntax node back up its macro calls.
559 pub fn original_file_range_opt(self, db: &dyn db::AstDatabase) -> Option<FileRange> {
560 match original_range_opt(db, self) {
562 let original_file = range.file_id.original_file(db);
563 if range.file_id != original_file.into() {
564 tracing::error!("Failed mapping up more for {:?}", range);
566 Some(FileRange { file_id: original_file, range: range.value })
568 _ if !self.file_id.is_macro() => Some(FileRange {
569 file_id: self.file_id.original_file(db),
570 range: self.value.text_range(),
577 fn original_range_opt(
578 db: &dyn db::AstDatabase,
579 node: InFile<&SyntaxNode>,
580 ) -> Option<InFile<TextRange>> {
581 let expansion = node.file_id.expansion_info(db)?;
583 // the input node has only one token ?
584 let single = skip_trivia_token(node.value.first_token()?, Direction::Next)?
585 == skip_trivia_token(node.value.last_token()?, Direction::Prev)?;
587 node.value.descendants().find_map(|it| {
588 let first = skip_trivia_token(it.first_token()?, Direction::Next)?;
589 let first = ascend_call_token(db, &expansion, node.with_value(first))?;
591 let last = skip_trivia_token(it.last_token()?, Direction::Prev)?;
592 let last = ascend_call_token(db, &expansion, node.with_value(last))?;
594 if (!single && first == last) || (first.file_id != last.file_id) {
598 Some(first.with_value(first.value.text_range().cover(last.value.text_range())))
602 fn ascend_call_token(
603 db: &dyn db::AstDatabase,
604 expansion: &ExpansionInfo,
605 token: InFile<SyntaxToken>,
606 ) -> Option<InFile<SyntaxToken>> {
607 let (mapped, origin) = expansion.map_token_up(db, token.as_ref())?;
608 if origin != Origin::Call {
611 if let Some(info) = mapped.file_id.expansion_info(db) {
612 return ascend_call_token(db, &info, mapped);
617 impl InFile<SyntaxToken> {
618 pub fn ancestors_with_macros(
620 db: &dyn db::AstDatabase,
621 ) -> impl Iterator<Item = InFile<SyntaxNode>> + '_ {
622 self.value.parent().into_iter().flat_map({
623 let file_id = self.file_id;
624 move |parent| InFile::new(file_id, parent).ancestors_with_macros(db)
629 impl<N: AstNode> InFile<N> {
630 pub fn descendants<T: AstNode>(self) -> impl Iterator<Item = InFile<T>> {
631 self.value.syntax().descendants().filter_map(T::cast).map(move |n| self.with_value(n))
634 pub fn syntax(&self) -> InFile<&SyntaxNode> {
635 self.with_value(self.value.syntax())
638 pub fn nodes_with_attributes<'db>(
640 db: &'db dyn db::AstDatabase,
641 ) -> impl Iterator<Item = InFile<N>> + 'db
645 iter::successors(Some(self), move |node| {
646 let InFile { file_id, value } = node.file_id.call_node(db)?;
647 N::cast(value).map(|n| InFile::new(file_id, n))
651 pub fn node_with_attributes(self, db: &dyn db::AstDatabase) -> InFile<N> {
652 self.nodes_with_attributes(db).last().unwrap()
656 /// In Rust, macros expand token trees to token trees. When we want to turn a
657 /// token tree into an AST node, we need to figure out what kind of AST node we
658 /// want: something like `foo` can be a type, an expression, or a pattern.
660 /// Naively, one would think that "what this expands to" is a property of a
661 /// particular macro: macro `m1` returns an item, while macro `m2` returns an
662 /// expression, etc. That's not the case -- macros are polymorphic in the
663 /// result, and can expand to any type of the AST node.
665 /// What defines the actual AST node is the syntactic context of the macro
666 /// invocation. As a contrived example, in `let T![*] = T![*];` the first `T`
667 /// expands to a pattern, while the second one expands to an expression.
669 /// `ExpandTo` captures this bit of information about a particular macro call
671 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
681 pub fn from_call_site(call: &ast::MacroCall) -> ExpandTo {
682 use syntax::SyntaxKind::*;
684 let syn = call.syntax();
686 let parent = match syn.parent() {
688 None => return ExpandTo::Statements,
691 match parent.kind() {
692 MACRO_ITEMS | SOURCE_FILE | ITEM_LIST => ExpandTo::Items,
693 MACRO_STMTS | EXPR_STMT | BLOCK_EXPR => ExpandTo::Statements,
694 MACRO_PAT => ExpandTo::Pattern,
695 MACRO_TYPE => ExpandTo::Type,
697 ARG_LIST | TRY_EXPR | TUPLE_EXPR | PAREN_EXPR | ARRAY_EXPR | FOR_EXPR | PATH_EXPR
698 | CLOSURE_EXPR | CONDITION | BREAK_EXPR | RETURN_EXPR | MATCH_EXPR | MATCH_ARM
699 | MATCH_GUARD | RECORD_EXPR_FIELD | CALL_EXPR | INDEX_EXPR | METHOD_CALL_EXPR
700 | FIELD_EXPR | AWAIT_EXPR | CAST_EXPR | REF_EXPR | PREFIX_EXPR | RANGE_EXPR
701 | BIN_EXPR => ExpandTo::Expr,
703 // FIXME: Handle LHS Pattern
708 // Unknown , Just guess it is `Items`