compiler/rustc_parse/src/lexer/tokentrees.rs

   1 use super::diagnostics::report_suspicious_mismatch_block;
   2 use super::diagnostics::same_identation_level;
   3 use super::diagnostics::TokenTreeDiagInfo;
   4 use super::{StringReader, UnmatchedBrace};
   5 use rustc_ast::token::{self, Delimiter, Token};
   6 use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
   7 use rustc_ast_pretty::pprust::token_to_string;
   8 use rustc_errors::{PErr, PResult};
   9
  10 pub(super) struct TokenTreesReader<'a> {
  11     string_reader: StringReader<'a>,
  12     /// The "next" token, which has been obtained from the `StringReader` but
  13     /// not yet handled by the `TokenTreesReader`.
  14     token: Token,
  15     diag_info: TokenTreeDiagInfo,
  16 }
  17
  18 impl<'a> TokenTreesReader<'a> {
  19     pub(super) fn parse_all_token_trees(
  20         string_reader: StringReader<'a>,
  21     ) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
  22         let mut tt_reader = TokenTreesReader {
  23             string_reader,
  24             token: Token::dummy(),
  25             diag_info: TokenTreeDiagInfo::default(),
  26         };
  27         let res = tt_reader.parse_token_trees(/* is_delimited */ false);
  28         (res, tt_reader.diag_info.unmatched_braces)
  29     }
  30
  31     // Parse a stream of tokens into a list of `TokenTree`s.
  32     fn parse_token_trees(&mut self, is_delimited: bool) -> PResult<'a, TokenStream> {
  33         self.token = self.string_reader.next_token().0;
  34         let mut buf = Vec::new();
  35         loop {
  36             match self.token.kind {
  37                 token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)),
  38                 token::CloseDelim(delim) => {
  39                     return if is_delimited {
  40                         Ok(TokenStream::new(buf))
  41                     } else {
  42                         Err(self.close_delim_err(delim))
  43                     };
  44                 }
  45                 token::Eof => {
  46                     if is_delimited {
  47                         self.eof_err().emit();
  48                     }
  49                     return Ok(TokenStream::new(buf));
  50                 }
  51                 _ => {
  52                     // Get the next normal token. This might require getting multiple adjacent
  53                     // single-char tokens and joining them together.
  54                     let (this_spacing, next_tok) = loop {
  55                         let (next_tok, is_next_tok_preceded_by_whitespace) =
  56                             self.string_reader.next_token();
  57                         if !is_next_tok_preceded_by_whitespace {
  58                             if let Some(glued) = self.token.glue(&next_tok) {
  59                                 self.token = glued;
  60                             } else {
  61                                 let this_spacing =
  62                                     if next_tok.is_op() { Spacing::Joint } else { Spacing::Alone };
  63                                 break (this_spacing, next_tok);
  64                             }
  65                         } else {
  66                             break (Spacing::Alone, next_tok);
  67                         }
  68                     };
  69                     let this_tok = std::mem::replace(&mut self.token, next_tok);
  70                     buf.push(TokenTree::Token(this_tok, this_spacing));
  71                 }
  72             }
  73         }
  74     }
  75
  76     fn eof_err(&mut self) -> PErr<'a> {
  77         let msg = "this file contains an unclosed delimiter";
  78         let mut err = self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, msg);
  79         for &(_, sp) in &self.diag_info.open_braces {
  80             err.span_label(sp, "unclosed delimiter");
  81             self.diag_info.unmatched_braces.push(UnmatchedBrace {
  82                 expected_delim: Delimiter::Brace,
  83                 found_delim: None,
  84                 found_span: self.token.span,
  85                 unclosed_span: Some(sp),
  86                 candidate_span: None,
  87             });
  88         }
  89
  90         if let Some((delim, _)) = self.diag_info.open_braces.last() {
  91             report_suspicious_mismatch_block(
  92                 &mut err,
  93                 &self.diag_info,
  94                 &self.string_reader.sess.source_map(),
  95                 *delim,
  96             )
  97         }
  98         err
  99     }
 100
 101     fn parse_token_tree_open_delim(&mut self, open_delim: Delimiter) -> TokenTree {
 102         // The span for beginning of the delimited section
 103         let pre_span = self.token.span;
 104
 105         self.diag_info.open_braces.push((open_delim, self.token.span));
 106
 107         // Parse the token trees within the delimiters.
 108         // We stop at any delimiter so we can try to recover if the user
 109         // uses an incorrect delimiter.
 110         let tts = self.parse_token_trees(/* is_delimited */ true).unwrap();
 111
 112         // Expand to cover the entire delimited token tree
 113         let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
 114         let sm = self.string_reader.sess.source_map();
 115
 116         match self.token.kind {
 117             // Correct delimiter.
 118             token::CloseDelim(close_delim) if close_delim == open_delim => {
 119                 let (open_brace, open_brace_span) = self.diag_info.open_braces.pop().unwrap();
 120                 let close_brace_span = self.token.span;
 121
 122                 if tts.is_empty() && close_delim == Delimiter::Brace {
 123                     let empty_block_span = open_brace_span.to(close_brace_span);
 124                     if !sm.is_multiline(empty_block_span) {
 125                         // Only track if the block is in the form of `{}`, otherwise it is
 126                         // likely that it was written on purpose.
 127                         self.diag_info.empty_block_spans.push(empty_block_span);
 128                     }
 129                 }
 130
 131                 // only add braces
 132                 if let (Delimiter::Brace, Delimiter::Brace) = (open_brace, open_delim) {
 133                     // Add all the matching spans, we will sort by span later
 134                     self.diag_info.matching_block_spans.push((open_brace_span, close_brace_span));
 135                 }
 136
 137                 // Move past the closing delimiter.
 138                 self.token = self.string_reader.next_token().0;
 139             }
 140             // Incorrect delimiter.
 141             token::CloseDelim(close_delim) => {
 142                 let mut unclosed_delimiter = None;
 143                 let mut candidate = None;
 144
 145                 if self.diag_info.last_unclosed_found_span != Some(self.token.span) {
 146                     // do not complain about the same unclosed delimiter multiple times
 147                     self.diag_info.last_unclosed_found_span = Some(self.token.span);
 148                     // This is a conservative error: only report the last unclosed
 149                     // delimiter. The previous unclosed delimiters could actually be
 150                     // closed! The parser just hasn't gotten to them yet.
 151                     if let Some(&(_, sp)) = self.diag_info.open_braces.last() {
 152                         unclosed_delimiter = Some(sp);
 153                     };
 154                     for (brace, brace_span) in &self.diag_info.open_braces {
 155                         if same_identation_level(&sm, self.token.span, *brace_span)
 156                             && brace == &close_delim
 157                         {
 158                             // high likelihood of these two corresponding
 159                             candidate = Some(*brace_span);
 160                         }
 161                     }
 162                     let (tok, _) = self.diag_info.open_braces.pop().unwrap();
 163                     self.diag_info.unmatched_braces.push(UnmatchedBrace {
 164                         expected_delim: tok,
 165                         found_delim: Some(close_delim),
 166                         found_span: self.token.span,
 167                         unclosed_span: unclosed_delimiter,
 168                         candidate_span: candidate,
 169                     });
 170                 } else {
 171                     self.diag_info.open_braces.pop();
 172                 }
 173
 174                 // If the incorrect delimiter matches an earlier opening
 175                 // delimiter, then don't consume it (it can be used to
 176                 // close the earlier one). Otherwise, consume it.
 177                 // E.g., we try to recover from:
 178                 // fn foo() {
 179                 //     bar(baz(
 180                 // }  // Incorrect delimiter but matches the earlier `{`
 181                 if !self.diag_info.open_braces.iter().any(|&(b, _)| b == close_delim) {
 182                     self.token = self.string_reader.next_token().0;
 183                 }
 184             }
 185             token::Eof => {
 186                 // Silently recover, the EOF token will be seen again
 187                 // and an error emitted then. Thus we don't pop from
 188                 // self.open_braces here.
 189             }
 190             _ => unreachable!(),
 191         }
 192
 193         TokenTree::Delimited(delim_span, open_delim, tts)
 194     }
 195
 196     fn close_delim_err(&mut self, delim: Delimiter) -> PErr<'a> {
 197         // An unexpected closing delimiter (i.e., there is no
 198         // matching opening delimiter).
 199         let token_str = token_to_string(&self.token);
 200         let msg = format!("unexpected closing delimiter: `{}`", token_str);
 201         let mut err =
 202             self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, &msg);
 203
 204         report_suspicious_mismatch_block(
 205             &mut err,
 206             &self.diag_info,
 207             &self.string_reader.sess.source_map(),
 208             delim,
 209         );
 210         err.span_label(self.token.span, "unexpected closing delimiter");
 211         err
 212     }
 213 }