compiler/rustc_parse/src/lexer/tokentrees.rs

   1 use super::{StringReader, UnmatchedBrace};
   2 use rustc_ast::token::{self, Delimiter, Token};
   3 use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
   4 use rustc_ast_pretty::pprust::token_to_string;
   5 use rustc_data_structures::fx::FxHashMap;
   6 use rustc_errors::{PErr, PResult};
   7 use rustc_span::Span;
   8
   9 pub(super) struct TokenTreesReader<'a> {
  10     string_reader: StringReader<'a>,
  11     /// The "next" token, which has been obtained from the `StringReader` but
  12     /// not yet handled by the `TokenTreesReader`.
  13     token: Token,
  14     /// Stack of open delimiters and their spans. Used for error message.
  15     open_braces: Vec<(Delimiter, Span)>,
  16     unmatched_braces: Vec<UnmatchedBrace>,
  17     /// The type and spans for all braces
  18     ///
  19     /// Used only for error recovery when arriving to EOF with mismatched braces.
  20     matching_delim_spans: Vec<(Delimiter, Span, Span)>,
  21     last_unclosed_found_span: Option<Span>,
  22     /// Collect empty block spans that might have been auto-inserted by editors.
  23     last_delim_empty_block_spans: FxHashMap<Delimiter, Span>,
  24     /// Collect the spans of braces (Open, Close). Used only
  25     /// for detecting if blocks are empty and only braces.
  26     matching_block_spans: Vec<(Span, Span)>,
  27 }
  28
  29 impl<'a> TokenTreesReader<'a> {
  30     pub(super) fn parse_all_token_trees(
  31         string_reader: StringReader<'a>,
  32     ) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
  33         let mut tt_reader = TokenTreesReader {
  34             string_reader,
  35             token: Token::dummy(),
  36             open_braces: Vec::new(),
  37             unmatched_braces: Vec::new(),
  38             matching_delim_spans: Vec::new(),
  39             last_unclosed_found_span: None,
  40             last_delim_empty_block_spans: FxHashMap::default(),
  41             matching_block_spans: Vec::new(),
  42         };
  43         let res = tt_reader.parse_token_trees(/* is_delimited */ false);
  44         (res, tt_reader.unmatched_braces)
  45     }
  46
  47     // Parse a stream of tokens into a list of `TokenTree`s.
  48     fn parse_token_trees(&mut self, is_delimited: bool) -> PResult<'a, TokenStream> {
  49         self.token = self.string_reader.next_token().0;
  50         let mut buf = Vec::new();
  51         loop {
  52             match self.token.kind {
  53                 token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)),
  54                 token::CloseDelim(delim) => {
  55                     return if is_delimited {
  56                         Ok(TokenStream::new(buf))
  57                     } else {
  58                         Err(self.close_delim_err(delim))
  59                     };
  60                 }
  61                 token::Eof => {
  62                     if is_delimited {
  63                         self.eof_err().emit();
  64                     }
  65                     return Ok(TokenStream::new(buf));
  66                 }
  67                 _ => {
  68                     // Get the next normal token. This might require getting multiple adjacent
  69                     // single-char tokens and joining them together.
  70                     let (this_spacing, next_tok) = loop {
  71                         let (next_tok, is_next_tok_preceded_by_whitespace) =
  72                             self.string_reader.next_token();
  73                         if !is_next_tok_preceded_by_whitespace {
  74                             if let Some(glued) = self.token.glue(&next_tok) {
  75                                 self.token = glued;
  76                             } else {
  77                                 let this_spacing =
  78                                     if next_tok.is_op() { Spacing::Joint } else { Spacing::Alone };
  79                                 break (this_spacing, next_tok);
  80                             }
  81                         } else {
  82                             break (Spacing::Alone, next_tok);
  83                         }
  84                     };
  85                     let this_tok = std::mem::replace(&mut self.token, next_tok);
  86                     buf.push(TokenTree::Token(this_tok, this_spacing));
  87                 }
  88             }
  89         }
  90     }
  91
  92     fn eof_err(&mut self) -> PErr<'a> {
  93         let msg = "this file contains an unclosed delimiter";
  94         let mut err = self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, msg);
  95         for &(_, sp) in &self.open_braces {
  96             err.span_label(sp, "unclosed delimiter");
  97             self.unmatched_braces.push(UnmatchedBrace {
  98                 expected_delim: Delimiter::Brace,
  99                 found_delim: None,
 100                 found_span: self.token.span,
 101                 unclosed_span: Some(sp),
 102                 candidate_span: None,
 103             });
 104         }
 105
 106         if let Some((delim, _)) = self.open_braces.last() {
 107             if let Some((_, open_sp, close_sp)) =
 108                 self.matching_delim_spans.iter().find(|(d, open_sp, close_sp)| {
 109                     let sm = self.string_reader.sess.source_map();
 110                     if let Some(close_padding) = sm.span_to_margin(*close_sp) {
 111                         if let Some(open_padding) = sm.span_to_margin(*open_sp) {
 112                             return delim == d && close_padding != open_padding;
 113                         }
 114                     }
 115                     false
 116                 })
 117             // these are in reverse order as they get inserted on close, but
 118             {
 119                 // we want the last open/first close
 120                 err.span_label(*open_sp, "this delimiter might not be properly closed...");
 121                 err.span_label(*close_sp, "...as it matches this but it has different indentation");
 122             }
 123         }
 124         err
 125     }
 126
 127     fn parse_token_tree_open_delim(&mut self, open_delim: Delimiter) -> TokenTree {
 128         // The span for beginning of the delimited section
 129         let pre_span = self.token.span;
 130
 131         self.open_braces.push((open_delim, self.token.span));
 132
 133         // Parse the token trees within the delimiters.
 134         // We stop at any delimiter so we can try to recover if the user
 135         // uses an incorrect delimiter.
 136         let tts = self.parse_token_trees(/* is_delimited */ true).unwrap();
 137
 138         // Expand to cover the entire delimited token tree
 139         let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
 140
 141         match self.token.kind {
 142             // Correct delimiter.
 143             token::CloseDelim(close_delim) if close_delim == open_delim => {
 144                 let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
 145                 let close_brace_span = self.token.span;
 146
 147                 if tts.is_empty() {
 148                     let empty_block_span = open_brace_span.to(close_brace_span);
 149                     let sm = self.string_reader.sess.source_map();
 150                     if !sm.is_multiline(empty_block_span) {
 151                         // Only track if the block is in the form of `{}`, otherwise it is
 152                         // likely that it was written on purpose.
 153                         self.last_delim_empty_block_spans.insert(open_delim, empty_block_span);
 154                     }
 155                 }
 156
 157                 //only add braces
 158                 if let (Delimiter::Brace, Delimiter::Brace) = (open_brace, open_delim) {
 159                     self.matching_block_spans.push((open_brace_span, close_brace_span));
 160                 }
 161
 162                 if self.open_braces.is_empty() {
 163                     // Clear up these spans to avoid suggesting them as we've found
 164                     // properly matched delimiters so far for an entire block.
 165                     self.matching_delim_spans.clear();
 166                 } else {
 167                     self.matching_delim_spans.push((open_brace, open_brace_span, close_brace_span));
 168                 }
 169                 // Move past the closing delimiter.
 170                 self.token = self.string_reader.next_token().0;
 171             }
 172             // Incorrect delimiter.
 173             token::CloseDelim(close_delim) => {
 174                 let mut unclosed_delimiter = None;
 175                 let mut candidate = None;
 176
 177                 if self.last_unclosed_found_span != Some(self.token.span) {
 178                     // do not complain about the same unclosed delimiter multiple times
 179                     self.last_unclosed_found_span = Some(self.token.span);
 180                     // This is a conservative error: only report the last unclosed
 181                     // delimiter. The previous unclosed delimiters could actually be
 182                     // closed! The parser just hasn't gotten to them yet.
 183                     if let Some(&(_, sp)) = self.open_braces.last() {
 184                         unclosed_delimiter = Some(sp);
 185                     };
 186                     let sm = self.string_reader.sess.source_map();
 187                     if let Some(current_padding) = sm.span_to_margin(self.token.span) {
 188                         for (brace, brace_span) in &self.open_braces {
 189                             if let Some(padding) = sm.span_to_margin(*brace_span) {
 190                                 // high likelihood of these two corresponding
 191                                 if current_padding == padding && brace == &close_delim {
 192                                     candidate = Some(*brace_span);
 193                                 }
 194                             }
 195                         }
 196                     }
 197                     let (tok, _) = self.open_braces.pop().unwrap();
 198                     self.unmatched_braces.push(UnmatchedBrace {
 199                         expected_delim: tok,
 200                         found_delim: Some(close_delim),
 201                         found_span: self.token.span,
 202                         unclosed_span: unclosed_delimiter,
 203                         candidate_span: candidate,
 204                     });
 205                 } else {
 206                     self.open_braces.pop();
 207                 }
 208
 209                 // If the incorrect delimiter matches an earlier opening
 210                 // delimiter, then don't consume it (it can be used to
 211                 // close the earlier one). Otherwise, consume it.
 212                 // E.g., we try to recover from:
 213                 // fn foo() {
 214                 //     bar(baz(
 215                 // }  // Incorrect delimiter but matches the earlier `{`
 216                 if !self.open_braces.iter().any(|&(b, _)| b == close_delim) {
 217                     self.token = self.string_reader.next_token().0;
 218                 }
 219             }
 220             token::Eof => {
 221                 // Silently recover, the EOF token will be seen again
 222                 // and an error emitted then. Thus we don't pop from
 223                 // self.open_braces here.
 224             }
 225             _ => unreachable!(),
 226         }
 227
 228         TokenTree::Delimited(delim_span, open_delim, tts)
 229     }
 230
 231     fn close_delim_err(&mut self, delim: Delimiter) -> PErr<'a> {
 232         // An unexpected closing delimiter (i.e., there is no
 233         // matching opening delimiter).
 234         let token_str = token_to_string(&self.token);
 235         let msg = format!("unexpected closing delimiter: `{}`", token_str);
 236         let mut err =
 237             self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, &msg);
 238
 239         // Braces are added at the end, so the last element is the biggest block
 240         if let Some(parent) = self.matching_block_spans.last() {
 241             if let Some(span) = self.last_delim_empty_block_spans.remove(&delim) {
 242                 // Check if the (empty block) is in the last properly closed block
 243                 if (parent.0.to(parent.1)).contains(span) {
 244                     err.span_label(span, "block is empty, you might have not meant to close it");
 245                 } else {
 246                     err.span_label(parent.0, "this opening brace...");
 247                     err.span_label(parent.1, "...matches this closing brace");
 248                 }
 249             } else {
 250                 err.span_label(parent.0, "this opening brace...");
 251                 err.span_label(parent.1, "...matches this closing brace");
 252             }
 253         }
 254
 255         err.span_label(self.token.span, "unexpected closing delimiter");
 256         err
 257     }
 258 }