src/libsyntax/parse/lexer/tokentrees.rs

   1 use syntax_pos::Span;
   2
   3 use crate::print::pprust::token_to_string;
   4 use crate::parse::lexer::{StringReader, UnmatchedBrace};
   5 use crate::parse::token::{self, Token};
   6 use crate::parse::PResult;
   7 use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint};
   8
   9 impl<'a> StringReader<'a> {
  10     crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
  11         let mut tt_reader = TokenTreesReader {
  12             string_reader: self,
  13             token: Token::dummy(),
  14             joint_to_prev: Joint,
  15             open_braces: Vec::new(),
  16             unmatched_braces: Vec::new(),
  17             matching_delim_spans: Vec::new(),
  18             last_unclosed_found_span: None,
  19         };
  20         let res = tt_reader.parse_all_token_trees();
  21         (res, tt_reader.unmatched_braces)
  22     }
  23 }
  24
  25 struct TokenTreesReader<'a> {
  26     string_reader: StringReader<'a>,
  27     token: Token,
  28     joint_to_prev: IsJoint,
  29     /// Stack of open delimiters and their spans. Used for error message.
  30     open_braces: Vec<(token::DelimToken, Span)>,
  31     unmatched_braces: Vec<UnmatchedBrace>,
  32     /// The type and spans for all braces
  33     ///
  34     /// Used only for error recovery when arriving to EOF with mismatched braces.
  35     matching_delim_spans: Vec<(token::DelimToken, Span, Span)>,
  36     last_unclosed_found_span: Option<Span>,
  37 }
  38
  39 impl<'a> TokenTreesReader<'a> {
  40     // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
  41     fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
  42         let mut tts = Vec::new();
  43
  44         self.real_token();
  45         while self.token != token::Eof {
  46             tts.push(self.parse_token_tree()?);
  47         }
  48
  49         Ok(TokenStream::new(tts))
  50     }
  51
  52     // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
  53     fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
  54         let mut tts = vec![];
  55         loop {
  56             if let token::CloseDelim(..) = self.token.kind {
  57                 return TokenStream::new(tts);
  58             }
  59
  60             match self.parse_token_tree() {
  61                 Ok(tree) => tts.push(tree),
  62                 Err(mut e) => {
  63                     e.emit();
  64                     return TokenStream::new(tts);
  65                 }
  66             }
  67         }
  68     }
  69
  70     fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
  71         let sm = self.string_reader.sess.source_map();
  72         match self.token.kind {
  73             token::Eof => {
  74                 let msg = "this file contains an un-closed delimiter";
  75                 let mut err = self.string_reader.sess.span_diagnostic
  76                     .struct_span_err(self.token.span, msg);
  77                 for &(_, sp) in &self.open_braces {
  78                     err.span_label(sp, "un-closed delimiter");
  79                 }
  80
  81                 if let Some((delim, _)) = self.open_braces.last() {
  82                     if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter()
  83                         .filter(|(d, open_sp, close_sp)| {
  84                             if let Some(close_padding) = sm.span_to_margin(*close_sp) {
  85                                 if let Some(open_padding) = sm.span_to_margin(*open_sp) {
  86                                     return delim == d && close_padding != open_padding;
  87                                 }
  88                             }
  89                             false
  90                         }).next()  // these are in reverse order as they get inserted on close, but
  91                     {              // we want the last open/first close
  92                         err.span_label(
  93                             *open_sp,
  94                             "this delimiter might not be properly closed...",
  95                         );
  96                         err.span_label(
  97                             *close_sp,
  98                             "...as it matches this but it has different indentation",
  99                         );
 100                     }
 101                 }
 102                 Err(err)
 103             },
 104             token::OpenDelim(delim) => {
 105                 // The span for beginning of the delimited section
 106                 let pre_span = self.token.span;
 107
 108                 // Parse the open delimiter.
 109                 self.open_braces.push((delim, self.token.span));
 110                 self.real_token();
 111
 112                 // Parse the token trees within the delimiters.
 113                 // We stop at any delimiter so we can try to recover if the user
 114                 // uses an incorrect delimiter.
 115                 let tts = self.parse_token_trees_until_close_delim();
 116
 117                 // Expand to cover the entire delimited token tree
 118                 let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
 119
 120                 match self.token.kind {
 121                     // Correct delimiter.
 122                     token::CloseDelim(d) if d == delim => {
 123                         let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
 124                         if self.open_braces.len() == 0 {
 125                             // Clear up these spans to avoid suggesting them as we've found
 126                             // properly matched delimiters so far for an entire block.
 127                             self.matching_delim_spans.clear();
 128                         } else {
 129                             self.matching_delim_spans.push(
 130                                 (open_brace, open_brace_span, self.token.span),
 131                             );
 132                         }
 133                         // Parse the close delimiter.
 134                         self.real_token();
 135                     }
 136                     // Incorrect delimiter.
 137                     token::CloseDelim(other) => {
 138                         let mut unclosed_delimiter = None;
 139                         let mut candidate = None;
 140                         if self.last_unclosed_found_span != Some(self.token.span) {
 141                             // do not complain about the same unclosed delimiter multiple times
 142                             self.last_unclosed_found_span = Some(self.token.span);
 143                             // This is a conservative error: only report the last unclosed
 144                             // delimiter. The previous unclosed delimiters could actually be
 145                             // closed! The parser just hasn't gotten to them yet.
 146                             if let Some(&(_, sp)) = self.open_braces.last() {
 147                                 unclosed_delimiter = Some(sp);
 148                             };
 149                             if let Some(current_padding) = sm.span_to_margin(self.token.span) {
 150                                 for (brace, brace_span) in &self.open_braces {
 151                                     if let Some(padding) = sm.span_to_margin(*brace_span) {
 152                                         // high likelihood of these two corresponding
 153                                         if current_padding == padding && brace == &other {
 154                                             candidate = Some(*brace_span);
 155                                         }
 156                                     }
 157                                 }
 158                             }
 159                             let (tok, _) = self.open_braces.pop().unwrap();
 160                             self.unmatched_braces.push(UnmatchedBrace {
 161                                 expected_delim: tok,
 162                                 found_delim: other,
 163                                 found_span: self.token.span,
 164                                 unclosed_span: unclosed_delimiter,
 165                                 candidate_span: candidate,
 166                             });
 167                         } else {
 168                             self.open_braces.pop();
 169                         }
 170
 171                         // If the incorrect delimiter matches an earlier opening
 172                         // delimiter, then don't consume it (it can be used to
 173                         // close the earlier one). Otherwise, consume it.
 174                         // E.g., we try to recover from:
 175                         // fn foo() {
 176                         //     bar(baz(
 177                         // }  // Incorrect delimiter but matches the earlier `{`
 178                         if !self.open_braces.iter().any(|&(b, _)| b == other) {
 179                             self.real_token();
 180                         }
 181                     }
 182                     token::Eof => {
 183                         // Silently recover, the EOF token will be seen again
 184                         // and an error emitted then. Thus we don't pop from
 185                         // self.open_braces here.
 186                     },
 187                     _ => {}
 188                 }
 189
 190                 Ok(TokenTree::Delimited(
 191                     delim_span,
 192                     delim,
 193                     tts.into()
 194                 ).into())
 195             },
 196             token::CloseDelim(_) => {
 197                 // An unexpected closing delimiter (i.e., there is no
 198                 // matching opening delimiter).
 199                 let token_str = token_to_string(&self.token);
 200                 let msg = format!("unexpected close delimiter: `{}`", token_str);
 201                 let mut err = self.string_reader.sess.span_diagnostic
 202                     .struct_span_err(self.token.span, &msg);
 203                 err.span_label(self.token.span, "unexpected close delimiter");
 204                 Err(err)
 205             },
 206             _ => {
 207                 let tt = TokenTree::Token(self.token.take());
 208                 self.real_token();
 209                 let is_joint = self.joint_to_prev == Joint && self.token.is_op();
 210                 Ok((tt, if is_joint { Joint } else { NonJoint }))
 211             }
 212         }
 213     }
 214
 215     fn real_token(&mut self) {
 216         self.joint_to_prev = Joint;
 217         loop {
 218             let token = self.string_reader.next_token();
 219             match token.kind {
 220                 token::Whitespace | token::Comment | token::Shebang(_) => {
 221                     self.joint_to_prev = NonJoint;
 222                 }
 223                 _ => {
 224                     self.token = token;
 225                     return;
 226                 },
 227             }
 228         }
 229     }
 230 }