src/librustc_parse/lexer/tokentrees.rs

   1 use super::{StringReader, UnmatchedBrace};
   2
   3 use rustc_ast::token::{self, Token};
   4 use rustc_ast::tokenstream::{
   5     DelimSpan,
   6     IsJoint::{self, *},
   7     TokenStream, TokenTree, TreeAndJoint,
   8 };
   9 use rustc_ast_pretty::pprust::token_to_string;
  10 use rustc_data_structures::fx::FxHashMap;
  11 use rustc_errors::PResult;
  12 use rustc_span::Span;
  13
  14 impl<'a> StringReader<'a> {
  15     crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
  16         let mut tt_reader = TokenTreesReader {
  17             string_reader: self,
  18             token: Token::dummy(),
  19             joint_to_prev: Joint,
  20             open_braces: Vec::new(),
  21             unmatched_braces: Vec::new(),
  22             matching_delim_spans: Vec::new(),
  23             last_unclosed_found_span: None,
  24             last_delim_empty_block_spans: FxHashMap::default(),
  25         };
  26         let res = tt_reader.parse_all_token_trees();
  27         (res, tt_reader.unmatched_braces)
  28     }
  29 }
  30
  31 struct TokenTreesReader<'a> {
  32     string_reader: StringReader<'a>,
  33     token: Token,
  34     joint_to_prev: IsJoint,
  35     /// Stack of open delimiters and their spans. Used for error message.
  36     open_braces: Vec<(token::DelimToken, Span)>,
  37     unmatched_braces: Vec<UnmatchedBrace>,
  38     /// The type and spans for all braces
  39     ///
  40     /// Used only for error recovery when arriving to EOF with mismatched braces.
  41     matching_delim_spans: Vec<(token::DelimToken, Span, Span)>,
  42     last_unclosed_found_span: Option<Span>,
  43     /// Collect empty block spans that might have been auto-inserted by editors.
  44     last_delim_empty_block_spans: FxHashMap<token::DelimToken, Span>,
  45 }
  46
  47 impl<'a> TokenTreesReader<'a> {
  48     // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
  49     fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
  50         let mut buf = TokenStreamBuilder::default();
  51
  52         self.real_token();
  53         while self.token != token::Eof {
  54             buf.push(self.parse_token_tree()?);
  55         }
  56
  57         Ok(buf.into_token_stream())
  58     }
  59
  60     // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
  61     fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
  62         let mut buf = TokenStreamBuilder::default();
  63         loop {
  64             if let token::CloseDelim(..) = self.token.kind {
  65                 return buf.into_token_stream();
  66             }
  67
  68             match self.parse_token_tree() {
  69                 Ok(tree) => buf.push(tree),
  70                 Err(mut e) => {
  71                     e.emit();
  72                     return buf.into_token_stream();
  73                 }
  74             }
  75         }
  76     }
  77
  78     fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
  79         let sm = self.string_reader.sess.source_map();
  80         match self.token.kind {
  81             token::Eof => {
  82                 let msg = "this file contains an unclosed delimiter";
  83                 let mut err =
  84                     self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, msg);
  85                 for &(_, sp) in &self.open_braces {
  86                     err.span_label(sp, "unclosed delimiter");
  87                     self.unmatched_braces.push(UnmatchedBrace {
  88                         expected_delim: token::DelimToken::Brace,
  89                         found_delim: None,
  90                         found_span: self.token.span,
  91                         unclosed_span: Some(sp),
  92                         candidate_span: None,
  93                     });
  94                 }
  95
  96                 if let Some((delim, _)) = self.open_braces.last() {
  97                     if let Some((_, open_sp, close_sp)) =
  98                         self.matching_delim_spans.iter().find(|(d, open_sp, close_sp)| {
  99                             if let Some(close_padding) = sm.span_to_margin(*close_sp) {
 100                                 if let Some(open_padding) = sm.span_to_margin(*open_sp) {
 101                                     return delim == d && close_padding != open_padding;
 102                                 }
 103                             }
 104                             false
 105                         })
 106                     // these are in reverse order as they get inserted on close, but
 107                     {
 108                         // we want the last open/first close
 109                         err.span_label(*open_sp, "this delimiter might not be properly closed...");
 110                         err.span_label(
 111                             *close_sp,
 112                             "...as it matches this but it has different indentation",
 113                         );
 114                     }
 115                 }
 116                 Err(err)
 117             }
 118             token::OpenDelim(delim) => {
 119                 // The span for beginning of the delimited section
 120                 let pre_span = self.token.span;
 121
 122                 // Parse the open delimiter.
 123                 self.open_braces.push((delim, self.token.span));
 124                 self.real_token();
 125
 126                 // Parse the token trees within the delimiters.
 127                 // We stop at any delimiter so we can try to recover if the user
 128                 // uses an incorrect delimiter.
 129                 let tts = self.parse_token_trees_until_close_delim();
 130
 131                 // Expand to cover the entire delimited token tree
 132                 let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
 133
 134                 match self.token.kind {
 135                     // Correct delimiter.
 136                     token::CloseDelim(d) if d == delim => {
 137                         let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
 138                         let close_brace_span = self.token.span;
 139
 140                         if tts.is_empty() {
 141                             let empty_block_span = open_brace_span.to(close_brace_span);
 142                             if !sm.is_multiline(empty_block_span) {
 143                                 // Only track if the block is in the form of `{}`, otherwise it is
 144                                 // likely that it was written on purpose.
 145                                 self.last_delim_empty_block_spans.insert(delim, empty_block_span);
 146                             }
 147                         }
 148
 149                         if self.open_braces.is_empty() {
 150                             // Clear up these spans to avoid suggesting them as we've found
 151                             // properly matched delimiters so far for an entire block.
 152                             self.matching_delim_spans.clear();
 153                         } else {
 154                             self.matching_delim_spans.push((
 155                                 open_brace,
 156                                 open_brace_span,
 157                                 close_brace_span,
 158                             ));
 159                         }
 160                         // Parse the closing delimiter.
 161                         self.real_token();
 162                     }
 163                     // Incorrect delimiter.
 164                     token::CloseDelim(other) => {
 165                         let mut unclosed_delimiter = None;
 166                         let mut candidate = None;
 167                         if self.last_unclosed_found_span != Some(self.token.span) {
 168                             // do not complain about the same unclosed delimiter multiple times
 169                             self.last_unclosed_found_span = Some(self.token.span);
 170                             // This is a conservative error: only report the last unclosed
 171                             // delimiter. The previous unclosed delimiters could actually be
 172                             // closed! The parser just hasn't gotten to them yet.
 173                             if let Some(&(_, sp)) = self.open_braces.last() {
 174                                 unclosed_delimiter = Some(sp);
 175                             };
 176                             if let Some(current_padding) = sm.span_to_margin(self.token.span) {
 177                                 for (brace, brace_span) in &self.open_braces {
 178                                     if let Some(padding) = sm.span_to_margin(*brace_span) {
 179                                         // high likelihood of these two corresponding
 180                                         if current_padding == padding && brace == &other {
 181                                             candidate = Some(*brace_span);
 182                                         }
 183                                     }
 184                                 }
 185                             }
 186                             let (tok, _) = self.open_braces.pop().unwrap();
 187                             self.unmatched_braces.push(UnmatchedBrace {
 188                                 expected_delim: tok,
 189                                 found_delim: Some(other),
 190                                 found_span: self.token.span,
 191                                 unclosed_span: unclosed_delimiter,
 192                                 candidate_span: candidate,
 193                             });
 194                         } else {
 195                             self.open_braces.pop();
 196                         }
 197
 198                         // If the incorrect delimiter matches an earlier opening
 199                         // delimiter, then don't consume it (it can be used to
 200                         // close the earlier one). Otherwise, consume it.
 201                         // E.g., we try to recover from:
 202                         // fn foo() {
 203                         //     bar(baz(
 204                         // }  // Incorrect delimiter but matches the earlier `{`
 205                         if !self.open_braces.iter().any(|&(b, _)| b == other) {
 206                             self.real_token();
 207                         }
 208                     }
 209                     token::Eof => {
 210                         // Silently recover, the EOF token will be seen again
 211                         // and an error emitted then. Thus we don't pop from
 212                         // self.open_braces here.
 213                     }
 214                     _ => {}
 215                 }
 216
 217                 Ok(TokenTree::Delimited(delim_span, delim, tts).into())
 218             }
 219             token::CloseDelim(delim) => {
 220                 // An unexpected closing delimiter (i.e., there is no
 221                 // matching opening delimiter).
 222                 let token_str = token_to_string(&self.token);
 223                 let msg = format!("unexpected closing delimiter: `{}`", token_str);
 224                 let mut err =
 225                     self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, &msg);
 226
 227                 if let Some(span) = self.last_delim_empty_block_spans.remove(&delim) {
 228                     err.span_label(
 229                         span,
 230                         "this block is empty, you might have not meant to close it",
 231                     );
 232                 }
 233                 err.span_label(self.token.span, "unexpected closing delimiter");
 234                 Err(err)
 235             }
 236             _ => {
 237                 let tt = TokenTree::Token(self.token.take());
 238                 self.real_token();
 239                 let is_joint = self.joint_to_prev == Joint && self.token.is_op();
 240                 Ok((tt, if is_joint { Joint } else { NonJoint }))
 241             }
 242         }
 243     }
 244
 245     fn real_token(&mut self) {
 246         self.joint_to_prev = Joint;
 247         loop {
 248             let token = self.string_reader.next_token();
 249             match token.kind {
 250                 token::Whitespace | token::Comment | token::Shebang(_) | token::Unknown(_) => {
 251                     self.joint_to_prev = NonJoint;
 252                 }
 253                 _ => {
 254                     self.token = token;
 255                     return;
 256                 }
 257             }
 258         }
 259     }
 260 }
 261
 262 #[derive(Default)]
 263 struct TokenStreamBuilder {
 264     buf: Vec<TreeAndJoint>,
 265 }
 266
 267 impl TokenStreamBuilder {
 268     fn push(&mut self, (tree, joint): TreeAndJoint) {
 269         if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() {
 270             if let TokenTree::Token(token) = &tree {
 271                 if let Some(glued) = prev_token.glue(token) {
 272                     self.buf.pop();
 273                     self.buf.push((TokenTree::Token(glued), joint));
 274                     return;
 275                 }
 276             }
 277         }
 278         self.buf.push((tree, joint))
 279     }
 280
 281     fn into_token_stream(self) -> TokenStream {
 282         TokenStream::new(self.buf)
 283     }
 284 }