X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=src%2Fmacros.rs;h=190f4b599b0f7053654b3ac355183907a9bd33a8;hb=612e8d5b9be72713a081370c85cc5bed30b6fae6;hp=9ce5c913ca55f48a82cadc4c00923098d9025281;hpb=3f6ea7788b5c65c00e995d04622533554a13dd38;p=rust.git diff --git a/src/macros.rs b/src/macros.rs index 9ce5c913ca5..190f4b599b0 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -1,13 +1,3 @@ -// Copyright 2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - // Format list-like macro invocations. These are invocations whose token trees // can be interpreted as expressions and separated by commas. // Note that these token trees do not actually have to be interpreted as @@ -20,36 +10,40 @@ // and those with brackets will be formatted as array literals. use std::collections::HashMap; +use std::panic::{catch_unwind, AssertUnwindSafe}; + +use rustc_ast::token::{BinOpToken, DelimToken, Token, TokenKind}; +use rustc_ast::tokenstream::{Cursor, LazyTokenStream, TokenStream, TokenTree}; +use rustc_ast::{ast, ptr}; +use rustc_ast_pretty::pprust; +use rustc_parse::parser::{ForceCollect, Parser}; +use rustc_parse::{stream_to_parser, MACRO_ARGUMENTS}; +use rustc_span::{ + symbol::{self, kw}, + BytePos, Span, Symbol, DUMMY_SP, +}; -use config::lists::*; -use syntax::parse::new_parser_from_tts; -use syntax::parse::parser::Parser; -use syntax::parse::token::{BinOpToken, DelimToken, Token}; -use syntax::print::pprust; -use syntax::source_map::{BytePos, Span}; -use syntax::symbol; -use syntax::tokenstream::{Cursor, ThinTokenStream, TokenStream, TokenTree}; -use syntax::ThinVec; -use syntax::{ast, parse, ptr}; - -use comment::{contains_comment, CharClasses, FindUncommented, FullCodeCharKind, LineClasses}; -use expr::rewrite_array; -use lists::{itemize_list, write_list, ListFormatting}; -use overflow; -use rewrite::{Rewrite, RewriteContext}; -use shape::{Indent, Shape}; -use source_map::SpanUtils; -use spanned::Spanned; -use utils::{ - format_visibility, is_empty_line, mk_sp, remove_trailing_white_spaces, rewrite_ident, - trim_left_preserve_layout, wrap_str, NodeIdExt, +use crate::comment::{ + contains_comment, CharClasses, FindUncommented, FullCodeCharKind, LineClasses, }; -use visitor::FmtVisitor; +use crate::config::lists::*; +use crate::expr::rewrite_array; +use crate::lists::{itemize_list, write_list, ListFormatting}; +use crate::overflow; +use crate::rewrite::{Rewrite, RewriteContext}; +use crate::shape::{Indent, Shape}; +use crate::source_map::SpanUtils; +use crate::spanned::Spanned; +use crate::utils::{ + format_visibility, indent_next_line, is_empty_line, mk_sp, remove_trailing_white_spaces, + rewrite_ident, trim_left_preserve_layout, wrap_str, NodeIdExt, +}; +use crate::visitor::FmtVisitor; const FORCED_BRACKET_MACROS: &[&str] = &["vec!"]; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum MacroPosition { +pub(crate) enum MacroPosition { Item, Statement, Expression, @@ -57,11 +51,12 @@ pub enum MacroPosition { } #[derive(Debug)] -pub enum MacroArg { +pub(crate) enum MacroArg { Expr(ptr::P), Ty(ptr::P), Pat(ptr::P), Item(ptr::P), + Keyword(symbol::Ident, Span), } impl MacroArg { @@ -74,8 +69,8 @@ fn is_item(&self) -> bool { } impl Rewrite for ast::Item { - fn rewrite(&self, context: &RewriteContext, shape: Shape) -> Option { - let mut visitor = ::visitor::FmtVisitor::from_context(context); + fn rewrite(&self, context: &RewriteContext<'_>, shape: Shape) -> Option { + let mut visitor = crate::visitor::FmtVisitor::from_context(context); visitor.block_indent = shape.indent; visitor.last_pos = self.span().lo(); visitor.visit_item(self); @@ -84,16 +79,25 @@ fn rewrite(&self, context: &RewriteContext, shape: Shape) -> Option { } impl Rewrite for MacroArg { - fn rewrite(&self, context: &RewriteContext, shape: Shape) -> Option { + fn rewrite(&self, context: &RewriteContext<'_>, shape: Shape) -> Option { match *self { MacroArg::Expr(ref expr) => expr.rewrite(context, shape), MacroArg::Ty(ref ty) => ty.rewrite(context, shape), MacroArg::Pat(ref pat) => pat.rewrite(context, shape), MacroArg::Item(ref item) => item.rewrite(context, shape), + MacroArg::Keyword(ident, _) => Some(ident.name.to_string()), } } } +fn build_parser<'a>(context: &RewriteContext<'a>, cursor: Cursor) -> Parser<'a> { + stream_to_parser( + context.parse_sess.inner(), + cursor.collect(), + MACRO_ARGUMENTS, + ) +} + fn parse_macro_arg<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option { macro_rules! parse_macro_arg { ($macro_arg:ident, $parser:expr, $f:expr) => { @@ -118,23 +122,23 @@ macro_rules! parse_macro_arg { parse_macro_arg!( Expr, - |parser: &mut parse::parser::Parser<'b>| parser.parse_expr(), + |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_expr(), |x: ptr::P| Some(x) ); parse_macro_arg!( Ty, - |parser: &mut parse::parser::Parser<'b>| parser.parse_ty(), + |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_ty(), |x: ptr::P| Some(x) ); parse_macro_arg!( Pat, - |parser: &mut parse::parser::Parser<'b>| parser.parse_pat(None), + |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_pat_no_top_alt(None), |x: ptr::P| Some(x) ); // `parse_item` returns `Option>`. parse_macro_arg!( Item, - |parser: &mut parse::parser::Parser<'b>| parser.parse_item(), + |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_item(ForceCollect::No), |x: Option>| x ); @@ -143,25 +147,25 @@ macro_rules! parse_macro_arg { /// Rewrite macro name without using pretty-printer if possible. fn rewrite_macro_name( - context: &RewriteContext, + context: &RewriteContext<'_>, path: &ast::Path, - extra_ident: Option, + extra_ident: Option, ) -> String { let name = if path.segments.len() == 1 { // Avoid using pretty-printer in the common case. format!("{}!", rewrite_ident(context, path.segments[0].ident)) } else { - format!("{}!", path) + format!("{}!", pprust::path_to_string(path)) }; match extra_ident { - Some(ident) if ident != symbol::keywords::Invalid.ident() => format!("{} {}", name, ident), + Some(ident) if ident.name != kw::Empty => format!("{} {}", name, ident), _ => name, } } // Use this on failing to format the macro call. fn return_macro_parse_failure_fallback( - context: &RewriteContext, + context: &RewriteContext<'_>, indent: Indent, span: Span, ) -> Option { @@ -185,61 +189,86 @@ fn return_macro_parse_failure_fallback( return trim_left_preserve_layout(context.snippet(span), indent, &context.config); } + context.skipped_range.borrow_mut().push(( + context.parse_sess.line_of_byte_pos(span.lo()), + context.parse_sess.line_of_byte_pos(span.hi()), + )); + // Return the snippet unmodified if the macro is not block-like Some(context.snippet(span).to_owned()) } -struct InsideMacroGuard<'a> { - context: &'a RewriteContext<'a>, - is_nested: bool, -} - -impl<'a> InsideMacroGuard<'a> { - fn inside_macro_context(context: &'a RewriteContext) -> InsideMacroGuard<'a> { - let is_nested = context.inside_macro.replace(true); - InsideMacroGuard { context, is_nested } - } -} - -impl<'a> Drop for InsideMacroGuard<'a> { - fn drop(&mut self) { - self.context.inside_macro.replace(self.is_nested); - } -} - -pub fn rewrite_macro( - mac: &ast::Mac, - extra_ident: Option, - context: &RewriteContext, +pub(crate) fn rewrite_macro( + mac: &ast::MacCall, + extra_ident: Option, + context: &RewriteContext<'_>, shape: Shape, position: MacroPosition, ) -> Option { - let guard = InsideMacroGuard::inside_macro_context(context); - let result = rewrite_macro_inner(mac, extra_ident, context, shape, position, guard.is_nested); - if result.is_none() { - context.macro_rewrite_failure.replace(true); + let should_skip = context + .skip_context + .skip_macro(&context.snippet(mac.path.span).to_owned()); + if should_skip { + None + } else { + let guard = context.enter_macro(); + let result = catch_unwind(AssertUnwindSafe(|| { + rewrite_macro_inner( + mac, + extra_ident, + context, + shape, + position, + guard.is_nested(), + ) + })); + match result { + Err(..) | Ok(None) => { + context.macro_rewrite_failure.replace(true); + None + } + Ok(rw) => rw, + } } - result } -pub fn rewrite_macro_inner( - mac: &ast::Mac, - extra_ident: Option, - context: &RewriteContext, +fn check_keyword<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option { + for &keyword in RUST_KW.iter() { + if parser.token.is_keyword(keyword) + && parser.look_ahead(1, |t| { + t.kind == TokenKind::Eof + || t.kind == TokenKind::Comma + || t.kind == TokenKind::CloseDelim(DelimToken::NoDelim) + }) + { + parser.bump(); + return Some(MacroArg::Keyword( + symbol::Ident::with_dummy_span(keyword), + parser.prev_token.span, + )); + } + } + None +} + +fn rewrite_macro_inner( + mac: &ast::MacCall, + extra_ident: Option, + context: &RewriteContext<'_>, shape: Shape, position: MacroPosition, is_nested_macro: bool, ) -> Option { if context.config.use_try_shorthand() { if let Some(expr) = convert_try_mac(mac, context) { - context.inside_macro.replace(false); + context.leave_macro(); return expr.rewrite(context, shape); } } let original_style = macro_style(mac, context); - let macro_name = rewrite_macro_name(context, &mac.node.path, extra_ident); + let macro_name = rewrite_macro_name(context, &mac.path, extra_ident); let style = if FORCED_BRACKET_MACROS.contains(&¯o_name[..]) && !is_nested_macro { DelimToken::Bracket @@ -247,13 +276,16 @@ pub fn rewrite_macro_inner( original_style }; - let ts: TokenStream = mac.node.stream(); - let has_comment = contains_comment(context.snippet(mac.span)); + let ts = mac.args.inner_tokens(); + let has_comment = contains_comment(context.snippet(mac.span())); if ts.is_empty() && !has_comment { return match style { DelimToken::Paren if position == MacroPosition::Item => { Some(format!("{}();", macro_name)) } + DelimToken::Bracket if position == MacroPosition::Item => { + Some(format!("{}[];", macro_name)) + } DelimToken::Paren => Some(format!("{}()", macro_name)), DelimToken::Bracket => Some(format!("{}[]", macro_name)), DelimToken::Brace => Some(format!("{} {{}}", macro_name)), @@ -267,33 +299,34 @@ pub fn rewrite_macro_inner( } } - let mut parser = new_parser_from_tts(context.parse_session, ts.trees().collect()); + let mut parser = build_parser(context, ts.trees()); let mut arg_vec = Vec::new(); let mut vec_with_semi = false; let mut trailing_comma = false; if DelimToken::Brace != style { loop { - match parse_macro_arg(&mut parser) { - Some(arg) => arg_vec.push(arg), - None => { - return return_macro_parse_failure_fallback(context, shape.indent, mac.span); - } + if let Some(arg) = check_keyword(&mut parser) { + arg_vec.push(arg); + } else if let Some(arg) = parse_macro_arg(&mut parser) { + arg_vec.push(arg); + } else { + return return_macro_parse_failure_fallback(context, shape.indent, mac.span()); } - match parser.token { - Token::Eof => break, - Token::Comma => (), - Token::Semi => { + match parser.token.kind { + TokenKind::Eof => break, + TokenKind::Comma => (), + TokenKind::Semi => { // Try to parse `vec![expr; expr]` if FORCED_BRACKET_MACROS.contains(&¯o_name[..]) { parser.bump(); - if parser.token != Token::Eof { + if parser.token.kind != TokenKind::Eof { match parse_macro_arg(&mut parser) { Some(arg) => { arg_vec.push(arg); parser.bump(); - if parser.token == Token::Eof && arg_vec.len() == 2 { + if parser.token.kind == TokenKind::Eof && arg_vec.len() == 2 { vec_with_semi = true; break; } @@ -302,21 +335,21 @@ pub fn rewrite_macro_inner( return return_macro_parse_failure_fallback( context, shape.indent, - mac.span, + mac.span(), ); } } } } - return return_macro_parse_failure_fallback(context, shape.indent, mac.span); + return return_macro_parse_failure_fallback(context, shape.indent, mac.span()); } _ if arg_vec.last().map_or(false, MacroArg::is_item) => continue, - _ => return return_macro_parse_failure_fallback(context, shape.indent, mac.span), + _ => return return_macro_parse_failure_fallback(context, shape.indent, mac.span()), } parser.bump(); - if parser.token == Token::Eof { + if parser.token.kind == TokenKind::Eof { trailing_comma = true; break; } @@ -331,57 +364,41 @@ pub fn rewrite_macro_inner( shape, style, position, - mac.span, + mac.span(), ); } match style { DelimToken::Paren => { - // Format macro invocation as function call, preserve the trailing - // comma because not all macros support them. - overflow::rewrite_with_parens( - context, - ¯o_name, - arg_vec.iter(), - shape, - mac.span, - context.config.width_heuristics().fn_call_width, - if trailing_comma { - Some(SeparatorTactic::Always) - } else { - Some(SeparatorTactic::Never) - }, - ) - .map(|rw| match position { - MacroPosition::Item => format!("{};", rw), - _ => rw, - }) + // Handle special case: `vec!(expr; expr)` + if vec_with_semi { + handle_vec_semi(context, shape, arg_vec, macro_name, style) + } else { + // Format macro invocation as function call, preserve the trailing + // comma because not all macros support them. + overflow::rewrite_with_parens( + context, + ¯o_name, + arg_vec.iter(), + shape, + mac.span(), + context.config.width_heuristics().fn_call_width, + if trailing_comma { + Some(SeparatorTactic::Always) + } else { + Some(SeparatorTactic::Never) + }, + ) + .map(|rw| match position { + MacroPosition::Item => format!("{};", rw), + _ => rw, + }) + } } DelimToken::Bracket => { // Handle special case: `vec![expr; expr]` if vec_with_semi { - let mac_shape = shape.offset_left(macro_name.len())?; - // 8 = `vec![]` + `; ` - let total_overhead = 8; - let nested_shape = mac_shape.block_indent(context.config.tab_spaces()); - let lhs = arg_vec[0].rewrite(context, nested_shape)?; - let rhs = arg_vec[1].rewrite(context, nested_shape)?; - if !lhs.contains('\n') - && !rhs.contains('\n') - && lhs.len() + rhs.len() + total_overhead <= shape.width - { - Some(format!("{}[{}; {}]", macro_name, lhs, rhs)) - } else { - Some(format!( - "{}[{}{};{}{}{}]", - macro_name, - nested_shape.indent.to_string_with_newline(context.config), - lhs, - nested_shape.indent.to_string_with_newline(context.config), - rhs, - shape.indent.to_string_with_newline(context.config), - )) - } + handle_vec_semi(context, shape, arg_vec, macro_name, style) } else { // If we are rewriting `vec!` macro or other special macros, // then we can rewrite this as an usual array literal. @@ -393,7 +410,7 @@ pub fn rewrite_macro_inner( Some(SeparatorTactic::Never) }; if FORCED_BRACKET_MACROS.contains(macro_name) && !is_nested_macro { - context.inside_macro.replace(false); + context.leave_macro(); if context.use_block_indent() { force_trailing_comma = Some(SeparatorTactic::Vertical); }; @@ -401,7 +418,7 @@ pub fn rewrite_macro_inner( let rewrite = rewrite_array( macro_name, arg_vec.iter(), - mac.span, + mac.span(), context, shape, force_trailing_comma, @@ -419,23 +436,63 @@ pub fn rewrite_macro_inner( // For macro invocations with braces, always put a space between // the `macro_name!` and `{ /* macro_body */ }` but skip modifying // anything in between the braces (for now). - let snippet = context.snippet(mac.span); - let macro_raw = snippet.split_at(snippet.find('!')? + 1).1.trim_start(); - match trim_left_preserve_layout(macro_raw, shape.indent, &context.config) { + let snippet = context.snippet(mac.span()).trim_start_matches(|c| c != '{'); + match trim_left_preserve_layout(snippet, shape.indent, &context.config) { Some(macro_body) => Some(format!("{} {}", macro_name, macro_body)), - None => Some(format!("{} {}", macro_name, macro_raw)), + None => Some(format!("{} {}", macro_name, snippet)), } } _ => unreachable!(), } } -pub fn rewrite_macro_def( - context: &RewriteContext, +fn handle_vec_semi( + context: &RewriteContext<'_>, + shape: Shape, + arg_vec: Vec, + macro_name: String, + delim_token: DelimToken, +) -> Option { + let (left, right) = match delim_token { + DelimToken::Paren => ("(", ")"), + DelimToken::Bracket => ("[", "]"), + _ => unreachable!(), + }; + + let mac_shape = shape.offset_left(macro_name.len())?; + // 8 = `vec![]` + `; ` or `vec!()` + `; ` + let total_overhead = 8; + let nested_shape = mac_shape.block_indent(context.config.tab_spaces()); + let lhs = arg_vec[0].rewrite(context, nested_shape)?; + let rhs = arg_vec[1].rewrite(context, nested_shape)?; + if !lhs.contains('\n') + && !rhs.contains('\n') + && lhs.len() + rhs.len() + total_overhead <= shape.width + { + // macro_name(lhs; rhs) or macro_name[lhs; rhs] + Some(format!("{}{}{}; {}{}", macro_name, left, lhs, rhs, right)) + } else { + // macro_name(\nlhs;\nrhs\n) or macro_name[\nlhs;\nrhs\n] + Some(format!( + "{}{}{}{};{}{}{}{}", + macro_name, + left, + nested_shape.indent.to_string_with_newline(context.config), + lhs, + nested_shape.indent.to_string_with_newline(context.config), + rhs, + shape.indent.to_string_with_newline(context.config), + right + )) + } +} + +pub(crate) fn rewrite_macro_def( + context: &RewriteContext<'_>, shape: Shape, indent: Indent, def: &ast::MacroDef, - ident: ast::Ident, + ident: symbol::Ident, vis: &ast::Visibility, span: Span, ) -> Option { @@ -444,13 +501,14 @@ pub fn rewrite_macro_def( return snippet; } - let mut parser = MacroParser::new(def.stream().into_trees()); + let ts = def.body.inner_tokens(); + let mut parser = MacroParser::new(ts.into_trees()); let parsed_def = match parser.parse() { Some(def) => def, None => return snippet, }; - let mut result = if def.legacy { + let mut result = if def.macro_rules { String::from("macro_rules!") } else { format!("{}macro", format_visibility(context, vis)) @@ -459,7 +517,7 @@ pub fn rewrite_macro_def( result += " "; result += rewrite_ident(context, ident); - let multi_branch_style = def.legacy || parsed_def.branches.len() != 1; + let multi_branch_style = def.macro_rules || parsed_def.branches.len() != 1; let arm_shape = if multi_branch_style { shape @@ -480,7 +538,7 @@ pub fn rewrite_macro_def( Some(v) => Some(v), // if the rewrite returned None because a macro could not be rewritten, then return the // original body - None if *context.macro_rewrite_failure.borrow() => { + None if context.macro_rewrite_failure.get() => { Some(context.snippet(branch.body).trim().to_string()) } None => None, @@ -492,7 +550,7 @@ pub fn rewrite_macro_def( .collect::>(); let fmt = ListFormatting::new(arm_shape, context.config) - .separator(if def.legacy { ";" } else { "" }) + .separator(if def.macro_rules { ";" } else { "" }) .trailing_separator(SeparatorTactic::Always) .preserve_newline(true); @@ -520,17 +578,12 @@ fn register_metavariable( name: &str, dollar_count: usize, ) { - let mut new_name = String::new(); - let mut old_name = String::new(); + let mut new_name = "$".repeat(dollar_count - 1); + let mut old_name = "$".repeat(dollar_count); - old_name.push('$'); - for _ in 0..(dollar_count - 1) { - new_name.push('$'); - old_name.push('$'); - } new_name.push('z'); - new_name.push_str(&name); - old_name.push_str(&name); + new_name.push_str(name); + old_name.push_str(name); result.push_str(&new_name); map.insert(old_name, new_name); @@ -579,9 +632,9 @@ fn replace_names(input: &str) -> Option<(String, HashMap)> { #[derive(Debug, Clone)] enum MacroArgKind { - /// e.g. `$x: expr`. - MetaVariable(ast::Ident, String), - /// e.g. `$($foo: expr),*` + /// e.g., `$x: expr`. + MetaVariable(Symbol, String), + /// e.g., `$($foo: expr),*` Repeat( /// `()`, `[]` or `{}`. DelimToken, @@ -592,17 +645,17 @@ enum MacroArgKind { /// The repeat token. This could be one of `*`, `+` or `?`. Token, ), - /// e.g. `[derive(Debug)]` + /// e.g., `[derive(Debug)]` Delimited(DelimToken, Vec), - /// A possible separator. e.g. `,` or `;`. + /// A possible separator. e.g., `,` or `;`. Separator(String, String), /// Other random stuff that does not fit to other kinds. - /// e.g. `== foo` in `($x: expr == foo)`. + /// e.g., `== foo` in `($x: expr == foo)`. Other(String, String), } fn delim_token_to_str( - context: &RewriteContext, + context: &RewriteContext<'_>, delim_token: DelimToken, shape: Shape, use_multiple_lines: bool, @@ -668,7 +721,7 @@ fn has_meta_var(&self) -> bool { fn rewrite( &self, - context: &RewriteContext, + context: &RewriteContext<'_>, shape: Shape, use_multiple_lines: bool, ) -> Option { @@ -688,9 +741,7 @@ fn rewrite( }; match *self { - MacroArgKind::MetaVariable(ty, ref name) => { - Some(format!("${}:{}", name, ty.name.as_str())) - } + MacroArgKind::MetaVariable(ty, ref name) => Some(format!("${}:{}", name, ty)), MacroArgKind::Repeat(delim_tok, ref args, ref another, ref tok) => { let (lhs, inner, rhs) = rewrite_delimited_inner(delim_tok, args)?; let another = another @@ -718,9 +769,9 @@ struct ParsedMacroArg { } impl ParsedMacroArg { - pub fn rewrite( + fn rewrite( &self, - context: &RewriteContext, + context: &RewriteContext<'_>, shape: Shape, use_multiple_lines: bool, ) -> Option { @@ -730,13 +781,13 @@ pub fn rewrite( /// Parses macro arguments on macro def. struct MacroArgParser { - /// Holds either a name of the next metavariable, a separator or a junk. + /// Either a name of the next metavariable, a separator, or junk. buf: String, /// The start position on the current buffer. lo: BytePos, /// The first token of the current buffer. start_tok: Token, - /// Set to true if we are parsing a metavariable or a repeat. + /// `true` if we are parsing a metavariable or a repeat. is_meta_var: bool, /// The position of the last token. hi: BytePos, @@ -748,20 +799,29 @@ struct MacroArgParser { fn last_tok(tt: &TokenTree) -> Token { match *tt { - TokenTree::Token(_, ref t) => t.clone(), - TokenTree::Delimited(_, ref d) => d.close_token(), + TokenTree::Token(ref t) => t.clone(), + TokenTree::Delimited(delim_span, delim, _) => Token { + kind: TokenKind::CloseDelim(delim), + span: delim_span.close, + }, } } impl MacroArgParser { - pub fn new() -> MacroArgParser { + fn new() -> MacroArgParser { MacroArgParser { lo: BytePos(0), hi: BytePos(0), buf: String::new(), is_meta_var: false, - last_tok: Token::Eof, - start_tok: Token::Eof, + last_tok: Token { + kind: TokenKind::Eof, + span: DUMMY_SP, + }, + start_tok: Token { + kind: TokenKind::Eof, + span: DUMMY_SP, + }, result: vec![], } } @@ -799,10 +859,13 @@ fn add_other(&mut self) { fn add_meta_variable(&mut self, iter: &mut Cursor) -> Option<()> { match iter.next() { - Some(TokenTree::Token(sp, Token::Ident(ref ident, _))) => { + Some(TokenTree::Token(Token { + kind: TokenKind::Ident(name, _), + span, + })) => { self.result.push(ParsedMacroArg { - kind: MacroArgKind::MetaVariable(*ident, self.buf.clone()), - span: mk_sp(self.lo, sp.hi()), + kind: MacroArgKind::MetaVariable(name, self.buf.clone()), + span: mk_sp(self.lo, span.hi()), }); self.buf.clear(); @@ -829,7 +892,7 @@ fn add_repeat( span: Span, ) -> Option<()> { let mut buffer = String::new(); - let mut first = false; + let mut first = true; let mut lo = span.lo(); let mut hi = span.hi(); @@ -842,14 +905,23 @@ fn add_repeat( } match tok { - TokenTree::Token(_, Token::BinOp(BinOpToken::Plus)) - | TokenTree::Token(_, Token::Question) - | TokenTree::Token(_, Token::BinOp(BinOpToken::Star)) => { + TokenTree::Token(Token { + kind: TokenKind::BinOp(BinOpToken::Plus), + .. + }) + | TokenTree::Token(Token { + kind: TokenKind::Question, + .. + }) + | TokenTree::Token(Token { + kind: TokenKind::BinOp(BinOpToken::Star), + .. + }) => { break; } - TokenTree::Token(sp, ref t) => { - buffer.push_str(&pprust::token_to_string(t)); - hi = sp.hi(); + TokenTree::Token(ref t) => { + buffer.push_str(&pprust::token_to_string(&t)); + hi = t.span.hi(); } _ => return None, } @@ -872,18 +944,18 @@ fn add_repeat( Some(()) } - fn update_buffer(&mut self, lo: BytePos, t: &Token) { + fn update_buffer(&mut self, t: &Token) { if self.buf.is_empty() { - self.lo = lo; + self.lo = t.span.lo(); self.start_tok = t.clone(); } else { - let needs_space = match next_space(&self.last_tok) { + let needs_space = match next_space(&self.last_tok.kind) { SpaceState::Ident => ident_like(t), SpaceState::Punctuation => !ident_like(t), SpaceState::Always => true, SpaceState::Never => false, }; - if force_space_before(t) || needs_space { + if force_space_before(&t.kind) || needs_space { self.buf.push(' '); } } @@ -901,12 +973,12 @@ fn need_space_prefix(&self) -> bool { if ident_like(&self.start_tok) { return true; } - if self.start_tok == Token::Colon { + if self.start_tok.kind == TokenKind::Colon { return true; } } - if force_space_before(&self.start_tok) { + if force_space_before(&self.start_tok.kind) { return true; } @@ -914,13 +986,15 @@ fn need_space_prefix(&self) -> bool { } /// Returns a collection of parsed macro def's arguments. - pub fn parse(mut self, tokens: ThinTokenStream) -> Option> { - let stream: TokenStream = tokens.into(); - let mut iter = stream.trees(); + fn parse(mut self, tokens: TokenStream) -> Option> { + let mut iter = tokens.trees(); - while let Some(ref tok) = iter.next() { + while let Some(tok) = iter.next() { match tok { - TokenTree::Token(sp, Token::Dollar) => { + TokenTree::Token(Token { + kind: TokenKind::Dollar, + span, + }) => { // We always want to add a separator before meta variables. if !self.buf.is_empty() { self.add_separator(); @@ -928,16 +1002,22 @@ pub fn parse(mut self, tokens: ThinTokenStream) -> Option> { // Start keeping the name of this metavariable in the buffer. self.is_meta_var = true; - self.lo = sp.lo(); - self.start_tok = Token::Dollar; + self.lo = span.lo(); + self.start_tok = Token { + kind: TokenKind::Dollar, + span, + }; } - TokenTree::Token(_, Token::Colon) if self.is_meta_var => { + TokenTree::Token(Token { + kind: TokenKind::Colon, + .. + }) if self.is_meta_var => { self.add_meta_variable(&mut iter)?; } - TokenTree::Token(sp, ref t) => self.update_buffer(sp.lo(), t), - TokenTree::Delimited(delimited_span, delimited) => { + TokenTree::Token(ref t) => self.update_buffer(t), + TokenTree::Delimited(delimited_span, delimited, ref tts) => { if !self.buf.is_empty() { - if next_space(&self.last_tok) == SpaceState::Always { + if next_space(&self.last_tok.kind) == SpaceState::Always { self.add_separator(); } else { self.add_other(); @@ -947,19 +1027,19 @@ pub fn parse(mut self, tokens: ThinTokenStream) -> Option> { // Parse the stuff inside delimiters. let mut parser = MacroArgParser::new(); parser.lo = delimited_span.open.lo(); - let delimited_arg = parser.parse(delimited.tts.clone())?; + let delimited_arg = parser.parse(tts.clone())?; let span = delimited_span.entire(); if self.is_meta_var { - self.add_repeat(delimited_arg, delimited.delim, &mut iter, span)?; + self.add_repeat(delimited_arg, delimited, &mut iter, span)?; self.is_meta_var = false; } else { - self.add_delimited(delimited_arg, delimited.delim, span); + self.add_delimited(delimited_arg, delimited, span); } } } - self.set_last_tok(tok); + self.set_last_tok(&tok); } // We are left with some stuff in the buffer. Since there is nothing @@ -973,7 +1053,7 @@ pub fn parse(mut self, tokens: ThinTokenStream) -> Option> { } fn wrap_macro_args( - context: &RewriteContext, + context: &RewriteContext<'_>, args: &[ParsedMacroArg], shape: Shape, ) -> Option { @@ -982,7 +1062,7 @@ fn wrap_macro_args( } fn wrap_macro_args_inner( - context: &RewriteContext, + context: &RewriteContext<'_>, args: &[ParsedMacroArg], shape: Shape, use_multiple_lines: bool, @@ -1026,19 +1106,18 @@ fn wrap_macro_args_inner( // We always try and format on one line. // FIXME: Use multi-line when every thing does not fit on one line. fn format_macro_args( - context: &RewriteContext, - toks: ThinTokenStream, + context: &RewriteContext<'_>, + token_stream: TokenStream, shape: Shape, ) -> Option { if !context.config.format_macro_matchers() { - let token_stream: TokenStream = toks.into(); let span = span_for_token_stream(&token_stream); return Some(match span { Some(span) => context.snippet(span).to_owned(), None => String::new(), }); } - let parsed_args = MacroArgParser::new().parse(toks)?; + let parsed_args = MacroArgParser::new().parse(token_stream)?; wrap_macro_args(context, &parsed_args, shape) } @@ -1055,89 +1134,93 @@ enum SpaceState { Always, } -fn force_space_before(tok: &Token) -> bool { +fn force_space_before(tok: &TokenKind) -> bool { debug!("tok: force_space_before {:?}", tok); match tok { - Token::Eq - | Token::Lt - | Token::Le - | Token::EqEq - | Token::Ne - | Token::Ge - | Token::Gt - | Token::AndAnd - | Token::OrOr - | Token::Not - | Token::Tilde - | Token::BinOpEq(_) - | Token::At - | Token::RArrow - | Token::LArrow - | Token::FatArrow - | Token::BinOp(_) - | Token::Pound - | Token::Dollar => true, + TokenKind::Eq + | TokenKind::Lt + | TokenKind::Le + | TokenKind::EqEq + | TokenKind::Ne + | TokenKind::Ge + | TokenKind::Gt + | TokenKind::AndAnd + | TokenKind::OrOr + | TokenKind::Not + | TokenKind::Tilde + | TokenKind::BinOpEq(_) + | TokenKind::At + | TokenKind::RArrow + | TokenKind::LArrow + | TokenKind::FatArrow + | TokenKind::BinOp(_) + | TokenKind::Pound + | TokenKind::Dollar => true, _ => false, } } fn ident_like(tok: &Token) -> bool { - match tok { - Token::Ident(..) | Token::Literal(..) | Token::Lifetime(_) => true, + match tok.kind { + TokenKind::Ident(..) | TokenKind::Literal(..) | TokenKind::Lifetime(_) => true, _ => false, } } -fn next_space(tok: &Token) -> SpaceState { +fn next_space(tok: &TokenKind) -> SpaceState { debug!("next_space: {:?}", tok); match tok { - Token::Not - | Token::BinOp(BinOpToken::And) - | Token::Tilde - | Token::At - | Token::Comma - | Token::Dot - | Token::DotDot - | Token::DotDotDot - | Token::DotDotEq - | Token::Question => SpaceState::Punctuation, - - Token::ModSep - | Token::Pound - | Token::Dollar - | Token::OpenDelim(_) - | Token::CloseDelim(_) - | Token::Whitespace => SpaceState::Never, - - Token::Literal(..) | Token::Ident(..) | Token::Lifetime(_) => SpaceState::Ident, + TokenKind::Not + | TokenKind::BinOp(BinOpToken::And) + | TokenKind::Tilde + | TokenKind::At + | TokenKind::Comma + | TokenKind::Dot + | TokenKind::DotDot + | TokenKind::DotDotDot + | TokenKind::DotDotEq + | TokenKind::Question => SpaceState::Punctuation, + + TokenKind::ModSep + | TokenKind::Pound + | TokenKind::Dollar + | TokenKind::OpenDelim(_) + | TokenKind::CloseDelim(_) => SpaceState::Never, + + TokenKind::Literal(..) | TokenKind::Ident(..) | TokenKind::Lifetime(_) => SpaceState::Ident, _ => SpaceState::Always, } } -/// Tries to convert a macro use into a short hand try expression. Returns None -/// when the macro is not an instance of try! (or parsing the inner expression +/// Tries to convert a macro use into a short hand try expression. Returns `None` +/// when the macro is not an instance of `try!` (or parsing the inner expression /// failed). -pub fn convert_try_mac(mac: &ast::Mac, context: &RewriteContext) -> Option { - if &mac.node.path.to_string() == "try" { - let ts: TokenStream = mac.node.tts.clone().into(); - let mut parser = new_parser_from_tts(context.parse_session, ts.trees().collect()); +pub(crate) fn convert_try_mac( + mac: &ast::MacCall, + context: &RewriteContext<'_>, +) -> Option { + let path = &pprust::path_to_string(&mac.path); + if path == "try" || path == "r#try" { + let ts = mac.args.inner_tokens(); + let mut parser = build_parser(context, ts.trees()); Some(ast::Expr { id: ast::NodeId::root(), // dummy value - node: ast::ExprKind::Try(parser.parse_expr().ok()?), - span: mac.span, // incorrect span, but shouldn't matter too much - attrs: ThinVec::new(), + kind: ast::ExprKind::Try(parser.parse_expr().ok()?), + span: mac.span(), // incorrect span, but shouldn't matter too much + attrs: ast::AttrVec::new(), + tokens: Some(LazyTokenStream::new(ts)), }) } else { None } } -fn macro_style(mac: &ast::Mac, context: &RewriteContext) -> DelimToken { - let snippet = context.snippet(mac.span); +pub(crate) fn macro_style(mac: &ast::MacCall, context: &RewriteContext<'_>) -> DelimToken { + let snippet = context.snippet(mac.span()); let paren_pos = snippet.find_uncommented("(").unwrap_or(usize::max_value()); let bracket_pos = snippet.find_uncommented("[").unwrap_or(usize::max_value()); let brace_pos = snippet.find_uncommented("{").unwrap_or(usize::max_value()); @@ -1174,16 +1257,19 @@ fn parse_branch(&mut self) -> Option { let tok = self.toks.next()?; let (lo, args_paren_kind) = match tok { TokenTree::Token(..) => return None, - TokenTree::Delimited(delimited_span, ref d) => (delimited_span.open.lo(), d.delim), + TokenTree::Delimited(delimited_span, d, _) => (delimited_span.open.lo(), d), }; - let args = tok.joint().into(); + let args = tok.joint(); match self.toks.next()? { - TokenTree::Token(_, Token::FatArrow) => {} + TokenTree::Token(Token { + kind: TokenKind::FatArrow, + .. + }) => {} _ => return None, } let (mut hi, body, whole_body) = match self.toks.next()? { TokenTree::Token(..) => return None, - TokenTree::Delimited(delimited_span, _) => { + TokenTree::Delimited(delimited_span, ..) => { let data = delimited_span.entire().data(); ( data.hi, @@ -1192,9 +1278,13 @@ fn parse_branch(&mut self) -> Option { ) } }; - if let Some(TokenTree::Token(sp, Token::Semi)) = self.toks.look_ahead(0) { + if let Some(TokenTree::Token(Token { + kind: TokenKind::Semi, + span, + })) = self.toks.look_ahead(0) + { + hi = span.hi(); self.toks.next(); - hi = sp.hi(); } Some(MacroBranch { span: mk_sp(lo, hi), @@ -1216,7 +1306,7 @@ struct Macro { struct MacroBranch { span: Span, args_paren_kind: DelimToken, - args: ThinTokenStream, + args: TokenStream, body: Span, whole_body: Span, } @@ -1224,7 +1314,7 @@ struct MacroBranch { impl MacroBranch { fn rewrite( &self, - context: &RewriteContext, + context: &RewriteContext<'_>, shape: Shape, multi_branch_style: bool, ) -> Option { @@ -1271,12 +1361,12 @@ fn rewrite( config.set().max_width(new_width); // First try to format as items, then as statements. - let new_body_snippet = match ::format_snippet(&body_str, &config) { + let new_body_snippet = match crate::format_snippet(&body_str, &config, true) { Some(new_body) => new_body, None => { let new_width = new_width + config.tab_spaces(); config.set().max_width(new_width); - match ::format_code_block(&body_str, &config) { + match crate::format_code_block(&body_str, &config, true) { Some(new_body) => new_body, None => return None, } @@ -1301,7 +1391,7 @@ fn rewrite( { s += &indent_str; } - (s + l + "\n", !kind.is_string() || l.ends_with('\\')) + (s + l + "\n", indent_next_line(kind, &l, &config)) }, ) .0; @@ -1334,7 +1424,7 @@ fn rewrite( /// /// # Expected syntax /// -/// ```ignore +/// ```text /// lazy_static! { /// [pub] static ref NAME_1: TYPE_1 = EXPR_1; /// [pub] static ref NAME_2: TYPE_2 = EXPR_2; @@ -1342,9 +1432,13 @@ fn rewrite( /// [pub] static ref NAME_N: TYPE_N = EXPR_N; /// } /// ``` -fn format_lazy_static(context: &RewriteContext, shape: Shape, ts: &TokenStream) -> Option { +fn format_lazy_static( + context: &RewriteContext<'_>, + shape: Shape, + ts: &TokenStream, +) -> Option { let mut result = String::with_capacity(1024); - let mut parser = new_parser_from_tts(context.parse_session, ts.trees().collect()); + let mut parser = build_parser(context, ts.trees()); let nested_shape = shape .block_indent(context.config.tab_spaces()) .with_max_width(context.config); @@ -1372,17 +1466,20 @@ macro_rules! parse_or { } } - while parser.token != Token::Eof { + while parser.token.kind != TokenKind::Eof { // Parse a `lazy_static!` item. - let vis = ::utils::format_visibility(context, &parse_or!(parse_visibility, false)); - parser.eat_keyword(symbol::keywords::Static); - parser.eat_keyword(symbol::keywords::Ref); + let vis = crate::utils::format_visibility( + context, + &parse_or!(parse_visibility, rustc_parse::parser::FollowedByType::No), + ); + parser.eat_keyword(kw::Static); + parser.eat_keyword(kw::Ref); let id = parse_or!(parse_ident); - parser.eat(&Token::Colon); + parser.eat(&TokenKind::Colon); let ty = parse_or!(parse_ty); - parser.eat(&Token::Eq); + parser.eat(&TokenKind::Eq); let expr = parse_or!(parse_expr); - parser.eat(&Token::Semi); + parser.eat(&TokenKind::Semi); // Rewrite as a static item. let mut stmt = String::with_capacity(128); @@ -1392,14 +1489,14 @@ macro_rules! parse_or { id, ty.rewrite(context, nested_shape)? )); - result.push_str(&::expr::rewrite_assign_rhs( + result.push_str(&crate::expr::rewrite_assign_rhs( context, stmt, &*expr, nested_shape.sub_width(1)?, )?); result.push(';'); - if parser.token != Token::Eof { + if parser.token.kind != TokenKind::Eof { result.push_str(&nested_shape.indent.to_string_with_newline(context.config)); } } @@ -1411,7 +1508,7 @@ macro_rules! parse_or { } fn rewrite_macro_with_items( - context: &RewriteContext, + context: &RewriteContext<'_>, items: &[MacroArg], macro_name: &str, shape: Shape, @@ -1451,3 +1548,65 @@ fn rewrite_macro_with_items( result.push_str(trailing_semicolon); Some(result) } + +const RUST_KW: [Symbol; 59] = [ + kw::PathRoot, + kw::DollarCrate, + kw::Underscore, + kw::As, + kw::Box, + kw::Break, + kw::Const, + kw::Continue, + kw::Crate, + kw::Else, + kw::Enum, + kw::Extern, + kw::False, + kw::Fn, + kw::For, + kw::If, + kw::Impl, + kw::In, + kw::Let, + kw::Loop, + kw::Match, + kw::Mod, + kw::Move, + kw::Mut, + kw::Pub, + kw::Ref, + kw::Return, + kw::SelfLower, + kw::SelfUpper, + kw::Static, + kw::Struct, + kw::Super, + kw::Trait, + kw::True, + kw::Type, + kw::Unsafe, + kw::Use, + kw::Where, + kw::While, + kw::Abstract, + kw::Become, + kw::Do, + kw::Final, + kw::Macro, + kw::Override, + kw::Priv, + kw::Typeof, + kw::Unsized, + kw::Virtual, + kw::Yield, + kw::Dyn, + kw::Async, + kw::Try, + kw::UnderscoreLifetime, + kw::StaticLifetime, + kw::Auto, + kw::Catch, + kw::Default, + kw::Union, +];