1 //! This module contains tests for macro expansion. Effectively, it covers `tt`,
2 //! `mbe`, `proc_macro_api` and `hir_expand` crates. This might seem like a
3 //! wrong architecture at the first glance, but is intentional.
5 //! Physically, macro expansion process is intertwined with name resolution. You
6 //! can not expand *just* the syntax. So, to be able to write integration tests
7 //! of the "expand this code please" form, we have to do it after name
8 //! resolution. That is, in this crate. We *could* fake some dependencies and
9 //! write unit-tests (in fact, we used to do that), but that makes tests brittle
10 //! and harder to understand.
14 mod builtin_derive_macro;
17 use std::{iter, ops::Range, sync::Arc};
20 use base_db::{fixture::WithFixture, ProcMacro, SourceDatabase};
21 use expect_test::Expect;
23 db::{AstDatabase, TokenExpander},
24 AstId, InFile, MacroDefId, MacroDefKind, MacroFile,
28 ast::{self, edit::IndentLevel},
29 AstNode, SyntaxElement,
30 SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT},
31 SyntaxNode, TextRange, T,
33 use tt::{Subtree, TokenId};
36 db::DefDatabase, nameres::ModuleSource, resolver::HasResolver, src::HasSource, test_db::TestDB,
37 AdtId, AsMacroCall, Lookup, ModuleDefId,
41 fn check(ra_fixture: &str, mut expect: Expect) {
42 let extra_proc_macros = vec![(
44 #[proc_macro_attribute]
45 pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream {
51 name: "identity_when_valid".into(),
52 kind: base_db::ProcMacroKind::Attr,
53 expander: Arc::new(IdentityWhenValidProcMacroExpander),
56 let db = TestDB::with_files_extra_proc_macros(ra_fixture, extra_proc_macros);
57 let krate = db.crate_graph().iter().next().unwrap();
58 let def_map = db.crate_def_map(krate);
59 let local_id = def_map.root();
60 let module = def_map.module_id(local_id);
61 let resolver = module.resolver(&db);
62 let source = def_map[local_id].definition_source(&db);
63 let source_file = match source.value {
64 ModuleSource::SourceFile(it) => it,
65 ModuleSource::Module(_) | ModuleSource::BlockExpr(_) => panic!(),
68 // What we want to do is to replace all macros (fn-like, derive, attr) with
69 // their expansions. Turns out, we don't actually store enough information
70 // to do this precisely though! Specifically, if a macro expands to nothing,
71 // it leaves zero traces in def-map, so we can't get its expansion after the
75 // <https://github.com/rust-analyzer/rust-analyzer/issues/3407>
76 // resolve/record tension!
78 // So here we try to do a resolve, which is necessary a heuristic. For macro
79 // calls, we use `as_call_id_with_errors`. For derives, we look at the impls
80 // in the module and assume that, if impls's source is a different
81 // `HirFileId`, than it came from macro expansion.
83 let mut text_edits = Vec::new();
84 let mut expansions = Vec::new();
86 for macro_ in source_file.syntax().descendants().filter_map(ast::Macro::cast) {
87 let mut show_token_ids = false;
88 for comment in macro_.syntax().children_with_tokens().filter(|it| it.kind() == COMMENT) {
89 show_token_ids |= comment.to_string().contains("+tokenids");
95 let call_offset = macro_.syntax().text_range().start().into();
96 let file_ast_id = db.ast_id_map(source.file_id).ast_id(¯o_);
97 let ast_id = AstId::new(source.file_id, file_ast_id.upcast());
98 let kind = MacroDefKind::Declarative(ast_id);
100 let macro_def = db.macro_def(MacroDefId { krate, kind, local_inner: false }).unwrap();
101 if let TokenExpander::DeclarativeMacro { mac, def_site_token_map } = &*macro_def {
102 let tt = match ¯o_ {
103 ast::Macro::MacroRules(mac) => mac.token_tree().unwrap(),
104 ast::Macro::MacroDef(_) => unimplemented!(""),
107 let tt_start = tt.syntax().text_range().start();
108 tt.syntax().descendants_with_tokens().filter_map(SyntaxElement::into_token).for_each(
110 let range = token.text_range().checked_sub(tt_start).unwrap();
111 if let Some(id) = def_site_token_map.token_by_range(range) {
112 let offset = (range.end() + tt_start).into();
113 text_edits.push((offset..offset, format!("#{}", id.0)));
118 call_offset..call_offset,
119 format!("// call ids will be shifted by {:?}\n", mac.shift()),
124 for macro_call in source_file.syntax().descendants().filter_map(ast::MacroCall::cast) {
125 let macro_call = InFile::new(source.file_id, ¯o_call);
126 let mut error = None;
127 let macro_call_id = macro_call
128 .as_call_id_with_errors(
131 |path| resolver.resolve_path_as_macro(&db, &path),
132 &mut |err| error = Some(err),
136 let macro_file = MacroFile { macro_call_id };
137 let mut expansion_result = db.parse_macro_expansion(macro_file);
138 expansion_result.err = expansion_result.err.or(error);
139 expansions.push((macro_call.value.clone(), expansion_result, db.macro_arg(macro_call_id)));
142 for (call, exp, arg) in expansions.into_iter().rev() {
143 let mut tree = false;
144 let mut expect_errors = false;
145 let mut show_token_ids = false;
146 for comment in call.syntax().children_with_tokens().filter(|it| it.kind() == COMMENT) {
147 tree |= comment.to_string().contains("+tree");
148 expect_errors |= comment.to_string().contains("+errors");
149 show_token_ids |= comment.to_string().contains("+tokenids");
152 let mut expn_text = String::new();
153 if let Some(err) = exp.err {
154 format_to!(expn_text, "/* error: {} */", err);
156 if let Some((parse, token_map)) = exp.value {
158 assert!(!parse.errors().is_empty(), "no parse errors in expansion");
159 for e in parse.errors() {
160 format_to!(expn_text, "/* parse error: {} */\n", e);
164 parse.errors().is_empty(),
165 "parse errors in expansion: \n{:#?}",
169 let pp = pretty_print_macro_expansion(
171 show_token_ids.then(|| &*token_map),
173 let indent = IndentLevel::from_node(call.syntax());
174 let pp = reindent(indent, pp);
175 format_to!(expn_text, "{}", pp);
178 let tree = format!("{:#?}", parse.syntax_node())
179 .split_inclusive("\n")
180 .map(|line| format!("// {}", line))
181 .collect::<String>();
182 format_to!(expn_text, "\n{}", tree)
185 let range = call.syntax().text_range();
186 let range: Range<usize> = range.into();
189 if let Some((tree, map, _)) = arg.as_deref() {
190 let tt_range = call.token_tree().unwrap().syntax().text_range();
191 let mut ranges = Vec::new();
192 extract_id_ranges(&mut ranges, &map, &tree);
193 for (range, id) in ranges {
194 let idx = (tt_range.start() + range.end()).into();
195 text_edits.push((idx..idx, format!("#{}", id.0)));
198 text_edits.push((range.start..range.start, "// ".into()));
199 call.to_string().match_indices('\n').for_each(|(offset, _)| {
200 let offset = offset + 1 + range.start;
201 text_edits.push((offset..offset, "// ".into()));
203 text_edits.push((range.end..range.end, "\n".into()));
204 text_edits.push((range.end..range.end, expn_text));
206 text_edits.push((range, expn_text));
210 text_edits.sort_by_key(|(range, _)| range.start);
211 text_edits.reverse();
212 let mut expanded_text = source_file.to_string();
213 for (range, text) in text_edits {
214 expanded_text.replace_range(range, &text);
217 for decl_id in def_map[local_id].scope.declarations() {
218 // FIXME: I'm sure there's already better way to do this
219 let src = match decl_id {
220 ModuleDefId::AdtId(AdtId::StructId(struct_id)) => {
221 Some(struct_id.lookup(&db).source(&db).syntax().cloned())
223 ModuleDefId::FunctionId(function_id) => {
224 Some(function_id.lookup(&db).source(&db).syntax().cloned())
228 if let Some(src) = src {
229 if src.file_id.is_attr_macro(&db) || src.file_id.is_custom_derive(&db) {
230 let pp = pretty_print_macro_expansion(src.value, None);
231 format_to!(expanded_text, "\n{}", pp)
236 for impl_id in def_map[local_id].scope.impls() {
237 let src = impl_id.lookup(&db).source(&db);
238 if src.file_id.is_builtin_derive(&db).is_some() {
239 let pp = pretty_print_macro_expansion(src.value.syntax().clone(), None);
240 format_to!(expanded_text, "\n{}", pp)
244 expect.indent(false);
245 expect.assert_eq(&expanded_text);
248 fn extract_id_ranges(ranges: &mut Vec<(TextRange, TokenId)>, map: &TokenMap, tree: &Subtree) {
249 tree.token_trees.iter().for_each(|tree| match tree {
250 tt::TokenTree::Leaf(leaf) => {
251 let id = match leaf {
252 tt::Leaf::Literal(it) => it.id,
253 tt::Leaf::Punct(it) => it.id,
254 tt::Leaf::Ident(it) => it.id,
256 ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id)));
258 tt::TokenTree::Subtree(tree) => extract_id_ranges(ranges, map, tree),
262 fn reindent(indent: IndentLevel, pp: String) -> String {
263 if !pp.contains('\n') {
266 let mut lines = pp.split_inclusive('\n');
267 let mut res = lines.next().unwrap().to_string();
269 if line.trim().is_empty() {
272 format_to!(res, "{}{}", indent, line)
278 fn pretty_print_macro_expansion(expn: SyntaxNode, map: Option<&TokenMap>) -> String {
279 let mut res = String::new();
280 let mut prev_kind = EOF;
281 let mut indent_level = 0;
282 for token in iter::successors(expn.first_token(), |t| t.next_token()) {
283 let curr_kind = token.kind();
284 let space = match (prev_kind, curr_kind) {
285 _ if prev_kind.is_trivia() || curr_kind.is_trivia() => "",
286 (T!['{'], T!['}']) => "",
287 (T![=], _) | (_, T![=]) => " ",
289 (T![;] | T!['{'] | T!['}'], _) => "\n",
290 (_, T!['}']) => "\n",
291 (IDENT | LIFETIME_IDENT, IDENT | LIFETIME_IDENT) => " ",
292 _ if prev_kind.is_keyword() && curr_kind.is_keyword() => " ",
293 (IDENT, _) if curr_kind.is_keyword() => " ",
294 (_, IDENT) if prev_kind.is_keyword() => " ",
295 (T![>], IDENT) => " ",
296 (T![>], _) if curr_kind.is_keyword() => " ",
297 (T![->], _) | (_, T![->]) => " ",
298 (T![&&], _) | (_, T![&&]) => " ",
300 (T![:], IDENT | T!['(']) => " ",
301 (T![:], _) if curr_kind.is_keyword() => " ",
302 (T![fn], T!['(']) => "",
303 (T![']'], _) if curr_kind.is_keyword() => " ",
304 (T![']'], T![#]) => "\n",
305 _ if prev_kind.is_keyword() => " ",
310 T!['{'] => indent_level += 1,
311 T!['}'] => indent_level -= 1,
317 let level = if curr_kind == T!['}'] { indent_level - 1 } else { indent_level };
318 res.push_str(&" ".repeat(level));
320 prev_kind = curr_kind;
321 format_to!(res, "{}", token);
322 if let Some(map) = map {
323 if let Some(id) = map.token_by_range(token.text_range()) {
324 format_to!(res, "#{}", id.0);
331 // Identity mapping, but only works when the input is syntactically valid. This
332 // simulates common proc macros that unnecessarily parse their input and return
335 struct IdentityWhenValidProcMacroExpander;
336 impl base_db::ProcMacroExpander for IdentityWhenValidProcMacroExpander {
342 ) -> Result<Subtree, base_db::ProcMacroExpansionError> {
344 ::mbe::token_tree_to_syntax_node(subtree, ::mbe::TopEntryPoint::MacroItems);
345 if parse.errors().is_empty() {
348 panic!("got invalid macro input: {:?}", parse.errors());