src/libsyntax/parse/lexer/tokentrees.rs

   1 use rustc_data_structures::fx::FxHashMap;
   2 use syntax_pos::Span;
   3
   4 use super::{StringReader, UnmatchedBrace};
   5
   6 use crate::print::pprust::token_to_string;
   7 use crate::parse::token::{self, Token};
   8 use crate::parse::PResult;
   9 use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint};
  10
  11 impl<'a> StringReader<'a> {
  12     crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
  13         let mut tt_reader = TokenTreesReader {
  14             string_reader: self,
  15             token: Token::dummy(),
  16             joint_to_prev: Joint,
  17             open_braces: Vec::new(),
  18             unmatched_braces: Vec::new(),
  19             matching_delim_spans: Vec::new(),
  20             last_unclosed_found_span: None,
  21             last_delim_empty_block_spans: FxHashMap::default()
  22         };
  23         let res = tt_reader.parse_all_token_trees();
  24         (res, tt_reader.unmatched_braces)
  25     }
  26 }
  27
  28 struct TokenTreesReader<'a> {
  29     string_reader: StringReader<'a>,
  30     token: Token,
  31     joint_to_prev: IsJoint,
  32     /// Stack of open delimiters and their spans. Used for error message.
  33     open_braces: Vec<(token::DelimToken, Span)>,
  34     unmatched_braces: Vec<UnmatchedBrace>,
  35     /// The type and spans for all braces
  36     ///
  37     /// Used only for error recovery when arriving to EOF with mismatched braces.
  38     matching_delim_spans: Vec<(token::DelimToken, Span, Span)>,
  39     last_unclosed_found_span: Option<Span>,
  40     last_delim_empty_block_spans: FxHashMap<token::DelimToken, Span>
  41 }
  42
  43 impl<'a> TokenTreesReader<'a> {
  44     // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
  45     fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
  46         let mut buf = TokenStreamBuilder::default();
  47
  48         self.real_token();
  49         while self.token != token::Eof {
  50             buf.push(self.parse_token_tree()?);
  51         }
  52
  53         Ok(buf.into_token_stream())
  54     }
  55
  56     // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
  57     fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
  58         let mut buf = TokenStreamBuilder::default();
  59         loop {
  60             if let token::CloseDelim(..) = self.token.kind {
  61                 return buf.into_token_stream();
  62             }
  63
  64             match self.parse_token_tree() {
  65                 Ok(tree) => buf.push(tree),
  66                 Err(mut e) => {
  67                     e.emit();
  68                     return buf.into_token_stream();
  69                 }
  70             }
  71         }
  72     }
  73
  74     fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
  75         let sm = self.string_reader.sess.source_map();
  76         match self.token.kind {
  77             token::Eof => {
  78                 let msg = "this file contains an un-closed delimiter";
  79                 let mut err = self.string_reader.sess.span_diagnostic
  80                     .struct_span_err(self.token.span, msg);
  81                 for &(_, sp) in &self.open_braces {
  82                     err.span_label(sp, "un-closed delimiter");
  83                     self.unmatched_braces.push(UnmatchedBrace {
  84                         expected_delim: token::DelimToken::Brace,
  85                         found_delim: None,
  86                         found_span: self.token.span,
  87                         unclosed_span: Some(sp),
  88                         candidate_span: None,
  89                     });
  90                 }
  91
  92                 if let Some((delim, _)) = self.open_braces.last() {
  93                     if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter()
  94                         .filter(|(d, open_sp, close_sp)| {
  95                             if let Some(close_padding) = sm.span_to_margin(*close_sp) {
  96                                 if let Some(open_padding) = sm.span_to_margin(*open_sp) {
  97                                     return delim == d && close_padding != open_padding;
  98                                 }
  99                             }
 100                             false
 101                         }).next()  // these are in reverse order as they get inserted on close, but
 102                     {              // we want the last open/first close
 103                         err.span_label(
 104                             *open_sp,
 105                             "this delimiter might not be properly closed...",
 106                         );
 107                         err.span_label(
 108                             *close_sp,
 109                             "...as it matches this but it has different indentation",
 110                         );
 111                     }
 112                 }
 113                 Err(err)
 114             },
 115             token::OpenDelim(delim) => {
 116                 // The span for beginning of the delimited section
 117                 let pre_span = self.token.span;
 118
 119                 // Parse the open delimiter.
 120                 self.open_braces.push((delim, self.token.span));
 121                 self.real_token();
 122
 123                 // Parse the token trees within the delimiters.
 124                 // We stop at any delimiter so we can try to recover if the user
 125                 // uses an incorrect delimiter.
 126                 let tts = self.parse_token_trees_until_close_delim();
 127
 128                 // Expand to cover the entire delimited token tree
 129                 let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
 130
 131                 match self.token.kind {
 132                     // Correct delimiter.
 133                     token::CloseDelim(d) if d == delim => {
 134                         let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
 135                         let close_brace_span = self.token.span;
 136
 137                         if tts.is_empty() {
 138                             let empty_block_span = open_brace_span.to(close_brace_span);
 139                             self.last_delim_empty_block_spans.insert(delim, empty_block_span);
 140                         }
 141
 142                         if self.open_braces.len() == 0 {
 143                             // Clear up these spans to avoid suggesting them as we've found
 144                             // properly matched delimiters so far for an entire block.
 145                             self.matching_delim_spans.clear();
 146                         } else {
 147                             self.matching_delim_spans.push(
 148                                 (open_brace, open_brace_span, close_brace_span),
 149                             );
 150                         }
 151                         // Parse the close delimiter.
 152                         self.real_token();
 153                     }
 154                     // Incorrect delimiter.
 155                     token::CloseDelim(other) => {
 156                         let mut unclosed_delimiter = None;
 157                         let mut candidate = None;
 158                         if self.last_unclosed_found_span != Some(self.token.span) {
 159                             // do not complain about the same unclosed delimiter multiple times
 160                             self.last_unclosed_found_span = Some(self.token.span);
 161                             // This is a conservative error: only report the last unclosed
 162                             // delimiter. The previous unclosed delimiters could actually be
 163                             // closed! The parser just hasn't gotten to them yet.
 164                             if let Some(&(_, sp)) = self.open_braces.last() {
 165                                 unclosed_delimiter = Some(sp);
 166                             };
 167                             if let Some(current_padding) = sm.span_to_margin(self.token.span) {
 168                                 for (brace, brace_span) in &self.open_braces {
 169                                     if let Some(padding) = sm.span_to_margin(*brace_span) {
 170                                         // high likelihood of these two corresponding
 171                                         if current_padding == padding && brace == &other {
 172                                             candidate = Some(*brace_span);
 173                                         }
 174                                     }
 175                                 }
 176                             }
 177                             let (tok, _) = self.open_braces.pop().unwrap();
 178                             self.unmatched_braces.push(UnmatchedBrace {
 179                                 expected_delim: tok,
 180                                 found_delim: Some(other),
 181                                 found_span: self.token.span,
 182                                 unclosed_span: unclosed_delimiter,
 183                                 candidate_span: candidate,
 184                             });
 185                         } else {
 186                             self.open_braces.pop();
 187                         }
 188
 189                         // If the incorrect delimiter matches an earlier opening
 190                         // delimiter, then don't consume it (it can be used to
 191                         // close the earlier one). Otherwise, consume it.
 192                         // E.g., we try to recover from:
 193                         // fn foo() {
 194                         //     bar(baz(
 195                         // }  // Incorrect delimiter but matches the earlier `{`
 196                         if !self.open_braces.iter().any(|&(b, _)| b == other) {
 197                             self.real_token();
 198                         }
 199                     }
 200                     token::Eof => {
 201                         // Silently recover, the EOF token will be seen again
 202                         // and an error emitted then. Thus we don't pop from
 203                         // self.open_braces here.
 204                     },
 205                     _ => {}
 206                 }
 207
 208                 Ok(TokenTree::Delimited(
 209                     delim_span,
 210                     delim,
 211                     tts.into()
 212                 ).into())
 213             },
 214             token::CloseDelim(delim) => {
 215                 // An unexpected closing delimiter (i.e., there is no
 216                 // matching opening delimiter).
 217                 let token_str = token_to_string(&self.token);
 218                 let msg = format!("unexpected close delimiter: `{}`", token_str);
 219                 let mut err = self.string_reader.sess.span_diagnostic
 220                     .struct_span_err(self.token.span, &msg);
 221
 222                 if let Some(span) = self.last_delim_empty_block_spans.remove(&delim) {
 223                     err.span_label(
 224                         span,
 225                         "this block is empty, you might have not meant to close it"
 226                     );
 227                 }
 228                 err.span_label(self.token.span, "unexpected close delimiter");
 229                 Err(err)
 230             },
 231             _ => {
 232                 let tt = TokenTree::Token(self.token.take());
 233                 self.real_token();
 234                 let is_joint = self.joint_to_prev == Joint && self.token.is_op();
 235                 Ok((tt, if is_joint { Joint } else { NonJoint }))
 236             }
 237         }
 238     }
 239
 240     fn real_token(&mut self) {
 241         self.joint_to_prev = Joint;
 242         loop {
 243             let token = self.string_reader.next_token();
 244             match token.kind {
 245                 token::Whitespace | token::Comment | token::Shebang(_) | token::Unknown(_) => {
 246                     self.joint_to_prev = NonJoint;
 247                 }
 248                 _ => {
 249                     self.token = token;
 250                     return;
 251                 }
 252             }
 253         }
 254     }
 255 }
 256
 257 #[derive(Default)]
 258 struct TokenStreamBuilder {
 259     buf: Vec<TreeAndJoint>,
 260 }
 261
 262 impl TokenStreamBuilder {
 263     fn push(&mut self, (tree, joint): TreeAndJoint) {
 264         if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() {
 265             if let TokenTree::Token(token) = &tree {
 266                 if let Some(glued) = prev_token.glue(token) {
 267                     self.buf.pop();
 268                     self.buf.push((TokenTree::Token(glued), joint));
 269                     return;
 270                 }
 271             }
 272         }
 273         self.buf.push((tree, joint))
 274     }
 275
 276     fn into_token_stream(self) -> TokenStream {
 277         TokenStream::new(self.buf)
 278     }
 279 }