X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=crates%2Fhir_expand%2Fsrc%2Fdb.rs;h=d6d33b4cd724f144e336c27e9fb91e67863a4cdc;hb=7b89d5ede23cbbbf4bef37b43e0d2d99752ddb51;hp=80ba7a8d148c75c182b2e80013e3563976f2fade;hpb=2b907652ee76c150afaf8b599bb985d502d2afae;p=rust.git diff --git a/crates/hir_expand/src/db.rs b/crates/hir_expand/src/db.rs index 80ba7a8d148..d6d33b4cd72 100644 --- a/crates/hir_expand/src/db.rs +++ b/crates/hir_expand/src/db.rs @@ -3,19 +3,21 @@ use std::sync::Arc; use base_db::{salsa, SourceDatabase}; -use itertools::Itertools; +use either::Either; use limit::Limit; -use mbe::{ExpandError, ExpandResult}; +use mbe::syntax_node_to_token_tree; +use rustc_hash::FxHashSet; use syntax::{ algo::diff, - ast::{self, AttrsOwner, NameOwner}, - AstNode, GreenNode, Parse, SyntaxNode, SyntaxToken, TextRange, T, + ast::{self, HasAttrs, HasDocComments}, + AstNode, GreenNode, Parse, SyntaxNode, SyntaxToken, T, }; use crate::{ - ast_id_map::AstIdMap, hygiene::HygieneFrame, BuiltinAttrExpander, BuiltinDeriveExpander, - BuiltinFnLikeExpander, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, - MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, + ast_id_map::AstIdMap, fixup, hygiene::HygieneFrame, BuiltinAttrExpander, BuiltinDeriveExpander, + BuiltinFnLikeExpander, ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, + MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, + ProcMacroExpander, }; /// Total limit on the number of tokens produced by any macro invocation. @@ -28,10 +30,8 @@ #[derive(Debug, Clone, Eq, PartialEq)] pub enum TokenExpander { - /// Old-style `macro_rules`. - MacroRules { mac: mbe::MacroRules, def_site_token_map: mbe::TokenMap }, - /// AKA macros 2.0. - MacroDef { mac: mbe::MacroDef, def_site_token_map: mbe::TokenMap }, + /// Old-style `macro_rules` or the new macros 2.0 + DeclarativeMacro { mac: mbe::DeclarativeMacro, def_site_token_map: mbe::TokenMap }, /// Stuff like `line!` and `file!`. Builtin(BuiltinFnLikeExpander), /// `global_allocator` and such. @@ -48,11 +48,10 @@ fn expand( db: &dyn AstDatabase, id: MacroCallId, tt: &tt::Subtree, - ) -> mbe::ExpandResult { + ) -> ExpandResult { match self { - TokenExpander::MacroRules { mac, .. } => mac.expand(tt), - TokenExpander::MacroDef { mac, .. } => mac.expand(tt), - TokenExpander::Builtin(it) => it.expand(db, id, tt), + TokenExpander::DeclarativeMacro { mac, .. } => mac.expand(tt).map_err(Into::into), + TokenExpander::Builtin(it) => it.expand(db, id, tt).map_err(Into::into), TokenExpander::BuiltinAttr(it) => it.expand(db, id, tt), TokenExpander::BuiltinDerive(it) => it.expand(db, id, tt), TokenExpander::ProcMacro(_) => { @@ -66,8 +65,7 @@ fn expand( pub(crate) fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId { match self { - TokenExpander::MacroRules { mac, .. } => mac.map_id_down(id), - TokenExpander::MacroDef { mac, .. } => mac.map_id_down(id), + TokenExpander::DeclarativeMacro { mac, .. } => mac.map_id_down(id), TokenExpander::Builtin(..) | TokenExpander::BuiltinAttr(..) | TokenExpander::BuiltinDerive(..) @@ -77,8 +75,7 @@ pub(crate) fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId { pub(crate) fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, mbe::Origin) { match self { - TokenExpander::MacroRules { mac, .. } => mac.map_id_up(id), - TokenExpander::MacroDef { mac, .. } => mac.map_id_up(id), + TokenExpander::DeclarativeMacro { mac, .. } => mac.map_id_up(id), TokenExpander::Builtin(..) | TokenExpander::BuiltinAttr(..) | TokenExpander::BuiltinDerive(..) @@ -108,18 +105,21 @@ fn parse_macro_expansion( /// We encode macro definitions into ids of macro calls, this what allows us /// to be incremental. #[salsa::interned] - fn intern_macro(&self, macro_call: MacroCallLoc) -> MacroCallId; + fn intern_macro_call(&self, macro_call: MacroCallLoc) -> MacroCallId; /// Lowers syntactic macro call to a token tree representation. #[salsa::transparent] - fn macro_arg(&self, id: MacroCallId) -> Option>; + fn macro_arg( + &self, + id: MacroCallId, + ) -> Option>; /// Extracts syntax node, corresponding to a macro call. That's a firewall /// query, only typing in the macro call itself changes the returned /// subtree. fn macro_arg_text(&self, id: MacroCallId) -> Option; /// Gets the expander for this macro. This compiles declarative macros, and /// just fetches procedural ones. - fn macro_def(&self, id: MacroDefId) -> Option>; + fn macro_def(&self, id: MacroDefId) -> Result, mbe::ParseError>; /// Expand macro call to a token tree. This query is LRUed (we keep 128 or so results in memory) fn macro_expand(&self, macro_call: MacroCallId) -> ExpandResult>>; @@ -144,36 +144,78 @@ pub fn expand_speculative( speculative_args: &SyntaxNode, token_to_map: SyntaxToken, ) -> Option<(SyntaxNode, SyntaxToken)> { - let loc = db.lookup_intern_macro(actual_macro_call); - let macro_def = db.macro_def(loc.def)?; + let loc = db.lookup_intern_macro_call(actual_macro_call); + let macro_def = db.macro_def(loc.def).ok()?; + let token_range = token_to_map.text_range(); - // Fetch token id in the speculative args + // Build the subtree and token mapping for the speculative args let censor = censor_for_macro_input(&loc, &speculative_args); - let (tt, args_tmap) = mbe::syntax_node_to_token_tree_censored(&speculative_args, censor); - let range = token_to_map.text_range().checked_sub(speculative_args.text_range().start())?; - let token_id = args_tmap.token_by_range(range)?; - - let speculative_expansion = if let MacroDefKind::ProcMacro(expander, ..) = loc.def.kind { - let attr_arg = match &loc.kind { - // FIXME make attr arg speculative as well - MacroCallKind::Attr { attr_args, .. } => { - let mut attr_args = attr_args.0.clone(); - mbe::Shift::new(&tt).shift_all(&mut attr_args); - Some(attr_args) + let mut fixups = fixup::fixup_syntax(&speculative_args); + fixups.replace.extend(censor.into_iter().map(|node| (node, Vec::new()))); + let (mut tt, spec_args_tmap, _) = mbe::syntax_node_to_token_tree_with_modifications( + &speculative_args, + fixups.token_map, + fixups.next_id, + fixups.replace, + fixups.append, + ); + + let (attr_arg, token_id) = match loc.kind { + MacroCallKind::Attr { invoc_attr_index, .. } => { + // Attributes may have an input token tree, build the subtree and map for this as well + // then try finding a token id for our token if it is inside this input subtree. + let item = ast::Item::cast(speculative_args.clone())?; + let attr = item + .doc_comments_and_attrs() + .nth(invoc_attr_index as usize) + .and_then(Either::left)?; + match attr.token_tree() { + Some(token_tree) => { + let (mut tree, map) = syntax_node_to_token_tree(attr.token_tree()?.syntax()); + tree.delimiter = None; + + let shift = mbe::Shift::new(&tt); + shift.shift_all(&mut tree); + + let token_id = if token_tree.syntax().text_range().contains_range(token_range) { + let attr_input_start = + token_tree.left_delimiter_token()?.text_range().start(); + let range = token_range.checked_sub(attr_input_start)?; + let token_id = shift.shift(map.token_by_range(range)?); + Some(token_id) + } else { + None + }; + (Some(tree), token_id) + } + _ => (None, None), } - _ => None, - }; + } + _ => (None, None), + }; + let token_id = match token_id { + Some(token_id) => token_id, + // token wasn't inside an attribute input so it has to be in the general macro input + None => { + let range = token_range.checked_sub(speculative_args.text_range().start())?; + let token_id = spec_args_tmap.token_by_range(range)?; + macro_def.map_id_down(token_id) + } + }; + // Do the actual expansion, we need to directly expand the proc macro due to the attribute args + // Otherwise the expand query will fetch the non speculative attribute args and pass those instead. + let mut speculative_expansion = if let MacroDefKind::ProcMacro(expander, ..) = loc.def.kind { + tt.delimiter = None; expander.expand(db, loc.krate, &tt, attr_arg.as_ref()) } else { macro_def.expand(db, actual_macro_call, &tt) }; let expand_to = macro_expand_to(db, actual_macro_call); - let (node, rev_tmap) = - token_tree_to_syntax_node(&speculative_expansion.value, expand_to).ok()?; + fixup::reverse_fixups(&mut speculative_expansion.value, &spec_args_tmap, &fixups.undo_info); + let (node, rev_tmap) = token_tree_to_syntax_node(&speculative_expansion.value, expand_to); - let token_id = macro_def.map_id_down(token_id); let range = rev_tmap.first_range_by_token(token_id, token_to_map.kind())?; let token = node.syntax_node().covering_element(range).into_token()?; Some((node.syntax_node(), token)) @@ -188,6 +230,8 @@ fn parse_or_expand(db: &dyn AstDatabase, file_id: HirFileId) -> Option Some(db.parse(file_id).tree().syntax().clone()), HirFileIdRepr::MacroFile(macro_file) => { + // FIXME: Note how we convert from `Parse` to `SyntaxNode` here, + // forgetting about parse errors. db.parse_macro_expansion(macro_file).value.map(|(it, _)| it.syntax_node()) } } @@ -204,7 +248,7 @@ fn parse_macro_expansion( // Note: // The final goal we would like to make all parse_macro success, // such that the following log will not call anyway. - let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id); + let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id); let node = loc.kind.to_node(db); // collect parent information for warning log @@ -231,17 +275,7 @@ fn parse_macro_expansion( tracing::debug!("expanded = {}", tt.as_debug_string()); tracing::debug!("kind = {:?}", expand_to); - let (parse, rev_token_map) = match token_tree_to_syntax_node(&tt, expand_to) { - Ok(it) => it, - Err(err) => { - tracing::debug!( - "failed to parse expansion to {:?} = {}", - expand_to, - tt.as_debug_string() - ); - return ExpandResult::only_err(err); - } - }; + let (parse, rev_token_map) = token_tree_to_syntax_node(&tt, expand_to); match result.err { Some(err) => { @@ -267,44 +301,66 @@ fn parse_macro_expansion( } } -fn macro_arg(db: &dyn AstDatabase, id: MacroCallId) -> Option> { +fn macro_arg( + db: &dyn AstDatabase, + id: MacroCallId, +) -> Option> { let arg = db.macro_arg_text(id)?; - let loc = db.lookup_intern_macro(id); + let loc = db.lookup_intern_macro_call(id); let node = SyntaxNode::new_root(arg); let censor = censor_for_macro_input(&loc, &node); - let (mut tt, tmap) = mbe::syntax_node_to_token_tree_censored(&node, censor); + let mut fixups = fixup::fixup_syntax(&node); + fixups.replace.extend(censor.into_iter().map(|node| (node, Vec::new()))); + let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications( + &node, + fixups.token_map, + fixups.next_id, + fixups.replace, + fixups.append, + ); if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included tt.delimiter = None; } - Some(Arc::new((tt, tmap))) + Some(Arc::new((tt, tmap, fixups.undo_info))) } -fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> Option { - match loc.kind { - MacroCallKind::FnLike { .. } => None, - MacroCallKind::Derive { derive_attr_index, .. } => match ast::Item::cast(node.clone()) { - Some(item) => item - .attrs() - .map(|attr| attr.syntax().text_range()) - .take(derive_attr_index as usize + 1) - .fold1(TextRange::cover), - None => None, - }, - MacroCallKind::Attr { invoc_attr_index, .. } => match ast::Item::cast(node.clone()) { - Some(item) => { - item.attrs().nth(invoc_attr_index as usize).map(|attr| attr.syntax().text_range()) +fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet { + (|| { + let censor = match loc.kind { + MacroCallKind::FnLike { .. } => return None, + MacroCallKind::Derive { derive_attr_index, .. } => { + cov_mark::hit!(derive_censoring); + ast::Item::cast(node.clone())? + .attrs() + .take(derive_attr_index as usize + 1) + // FIXME + .filter(|attr| attr.simple_name().as_deref() == Some("derive")) + .map(|it| it.syntax().clone()) + .collect() } - None => None, - }, - } + MacroCallKind::Attr { is_derive: true, .. } => return None, + MacroCallKind::Attr { invoc_attr_index, .. } => { + cov_mark::hit!(attribute_macro_attr_censoring); + ast::Item::cast(node.clone())? + .doc_comments_and_attrs() + .nth(invoc_attr_index as usize) + .and_then(Either::left) + .map(|attr| attr.syntax().clone()) + .into_iter() + .collect() + } + }; + Some(censor) + })() + .unwrap_or_default() } fn macro_arg_text(db: &dyn AstDatabase, id: MacroCallId) -> Option { - let loc = db.lookup_intern_macro(id); + let loc = db.lookup_intern_macro_call(id); let arg = loc.kind.arg(db)?; if matches!(loc.kind, MacroCallKind::FnLike { .. }) { let first = arg.first_child_or_token().map_or(T![.], |it| it.kind()); @@ -326,51 +382,48 @@ fn macro_arg_text(db: &dyn AstDatabase, id: MacroCallId) -> Option { Some(arg.green().into()) } -fn macro_def(db: &dyn AstDatabase, id: MacroDefId) -> Option> { +fn macro_def(db: &dyn AstDatabase, id: MacroDefId) -> Result, mbe::ParseError> { match id.kind { - MacroDefKind::Declarative(ast_id) => match ast_id.to_node(db) { - ast::Macro::MacroRules(macro_rules) => { - let arg = macro_rules.token_tree()?; - let (tt, def_site_token_map) = mbe::syntax_node_to_token_tree(arg.syntax()); - let mac = match mbe::MacroRules::parse(&tt) { - Ok(it) => it, - Err(err) => { - let name = macro_rules.name().map(|n| n.to_string()).unwrap_or_default(); - tracing::warn!("fail on macro_def parse ({}): {:?} {:#?}", name, err, tt); - return None; - } - }; - Some(Arc::new(TokenExpander::MacroRules { mac, def_site_token_map })) - } - ast::Macro::MacroDef(macro_def) => { - let arg = macro_def.body()?; - let (tt, def_site_token_map) = mbe::syntax_node_to_token_tree(arg.syntax()); - let mac = match mbe::MacroDef::parse(&tt) { - Ok(it) => it, - Err(err) => { - let name = macro_def.name().map(|n| n.to_string()).unwrap_or_default(); - tracing::warn!("fail on macro_def parse ({}): {:?} {:#?}", name, err, tt); - return None; - } - }; - Some(Arc::new(TokenExpander::MacroDef { mac, def_site_token_map })) - } - }, - MacroDefKind::BuiltIn(expander, _) => Some(Arc::new(TokenExpander::Builtin(expander))), + MacroDefKind::Declarative(ast_id) => { + let (mac, def_site_token_map) = match ast_id.to_node(db) { + ast::Macro::MacroRules(macro_rules) => { + let arg = macro_rules + .token_tree() + .ok_or_else(|| mbe::ParseError::Expected("expected a token tree".into()))?; + let (tt, def_site_token_map) = mbe::syntax_node_to_token_tree(arg.syntax()); + let mac = mbe::DeclarativeMacro::parse_macro_rules(&tt)?; + (mac, def_site_token_map) + } + ast::Macro::MacroDef(macro_def) => { + let arg = macro_def + .body() + .ok_or_else(|| mbe::ParseError::Expected("expected a token tree".into()))?; + let (tt, def_site_token_map) = mbe::syntax_node_to_token_tree(arg.syntax()); + let mac = mbe::DeclarativeMacro::parse_macro2(&tt)?; + (mac, def_site_token_map) + } + }; + Ok(Arc::new(TokenExpander::DeclarativeMacro { mac, def_site_token_map })) + } + MacroDefKind::BuiltIn(expander, _) => Ok(Arc::new(TokenExpander::Builtin(expander))), MacroDefKind::BuiltInAttr(expander, _) => { - Some(Arc::new(TokenExpander::BuiltinAttr(expander))) + Ok(Arc::new(TokenExpander::BuiltinAttr(expander))) } MacroDefKind::BuiltInDerive(expander, _) => { - Some(Arc::new(TokenExpander::BuiltinDerive(expander))) + Ok(Arc::new(TokenExpander::BuiltinDerive(expander))) } - MacroDefKind::BuiltInEager(..) => None, - MacroDefKind::ProcMacro(expander, ..) => Some(Arc::new(TokenExpander::ProcMacro(expander))), + MacroDefKind::BuiltInEager(..) => { + // FIXME: Return a random error here just to make the types align. + // This obviously should do something real instead. + Err(mbe::ParseError::UnexpectedToken("unexpected eager macro".into())) + } + MacroDefKind::ProcMacro(expander, ..) => Ok(Arc::new(TokenExpander::ProcMacro(expander))), } } fn macro_expand(db: &dyn AstDatabase, id: MacroCallId) -> ExpandResult>> { let _p = profile::span("macro_expand"); - let loc: MacroCallLoc = db.lookup_intern_macro(id); + let loc: MacroCallLoc = db.lookup_intern_macro_call(id); if let Some(eager) = &loc.eager { return ExpandResult { value: Some(eager.arg_or_expansion.clone()), @@ -381,25 +434,40 @@ fn macro_expand(db: &dyn AstDatabase, id: MacroCallId) -> ExpandResult it, - None => return ExpandResult::str_err("Failed to lower macro args to token tree".into()), + None => { + return ExpandResult::only_err(ExpandError::Other( + "Failed to lower macro args to token tree".into(), + )) + } }; let expander = match db.macro_def(loc.def) { - Some(it) => it, - None => return ExpandResult::str_err("Failed to find macro definition".into()), + Ok(it) => it, + // FIXME: This is weird -- we effectively report macro *definition* + // errors lazily, when we try to expand the macro. Instead, they should + // be reported at the definition site (when we construct a def map). + Err(err) => { + return ExpandResult::only_err(ExpandError::Other( + format!("invalid macro definition: {}", err).into(), + )) + } }; - let ExpandResult { value: tt, err } = expander.expand(db, id, ¯o_arg.0); + let ExpandResult { value: mut tt, err } = expander.expand(db, id, ¯o_arg.0); // Set a hard limit for the expanded tt let count = tt.count(); - // XXX: Make ExpandResult a real error and use .map_err instead? if TOKEN_LIMIT.check(count).is_err() { - return ExpandResult::str_err(format!( - "macro invocation exceeds token limit: produced {} tokens, limit is {}", - count, - TOKEN_LIMIT.inner(), + return ExpandResult::only_err(ExpandError::Other( + format!( + "macro invocation exceeds token limit: produced {} tokens, limit is {}", + count, + TOKEN_LIMIT.inner(), + ) + .into(), )); } + fixup::reverse_fixups(&mut tt, ¯o_arg.1, ¯o_arg.2); + ExpandResult { value: Some(Arc::new(tt)), err } } @@ -408,10 +476,12 @@ fn macro_expand_error(db: &dyn AstDatabase, macro_call: MacroCallId) -> Option ExpandResult { - let loc: MacroCallLoc = db.lookup_intern_macro(id); + let loc: MacroCallLoc = db.lookup_intern_macro_call(id); let macro_arg = match db.macro_arg(id) { Some(it) => it, - None => return ExpandResult::str_err("No arguments for proc-macro".to_string()), + None => { + return ExpandResult::only_err(ExpandError::Other("No arguments for proc-macro".into())) + } }; let expander = match loc.def.kind { @@ -453,20 +523,20 @@ fn hygiene_frame(db: &dyn AstDatabase, file_id: HirFileId) -> Arc } fn macro_expand_to(db: &dyn AstDatabase, id: MacroCallId) -> ExpandTo { - let loc: MacroCallLoc = db.lookup_intern_macro(id); + let loc: MacroCallLoc = db.lookup_intern_macro_call(id); loc.kind.expand_to() } fn token_tree_to_syntax_node( tt: &tt::Subtree, expand_to: ExpandTo, -) -> Result<(Parse, mbe::TokenMap), ExpandError> { +) -> (Parse, mbe::TokenMap) { let entry_point = match expand_to { - ExpandTo::Statements => mbe::ParserEntryPoint::Statements, - ExpandTo::Items => mbe::ParserEntryPoint::Items, - ExpandTo::Pattern => mbe::ParserEntryPoint::Pattern, - ExpandTo::Type => mbe::ParserEntryPoint::Type, - ExpandTo::Expr => mbe::ParserEntryPoint::Expr, + ExpandTo::Statements => mbe::TopEntryPoint::MacroStmts, + ExpandTo::Items => mbe::TopEntryPoint::MacroItems, + ExpandTo::Pattern => mbe::TopEntryPoint::Pattern, + ExpandTo::Type => mbe::TopEntryPoint::Type, + ExpandTo::Expr => mbe::TopEntryPoint::Expr, }; mbe::token_tree_to_syntax_node(tt, entry_point) }