src/libsyntax/parse/lexer/tokentrees.rs

   1 use syntax_pos::Span;
   2
   3 use crate::print::pprust::token_to_string;
   4 use crate::parse::lexer::{StringReader, UnmatchedBrace};
   5 use crate::parse::{token, PResult};
   6 use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint};
   7
   8 impl<'a> StringReader<'a> {
   9     crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
  10         let mut tt_reader = TokenTreesReader {
  11             string_reader: self,
  12             token: token::Eof,
  13             span: syntax_pos::DUMMY_SP,
  14             open_braces: Vec::new(),
  15             unmatched_braces: Vec::new(),
  16             matching_delim_spans: Vec::new(),
  17             last_unclosed_found_span: None,
  18         };
  19         let res = tt_reader.parse_all_token_trees();
  20         (res, tt_reader.unmatched_braces)
  21     }
  22 }
  23
  24 struct TokenTreesReader<'a> {
  25     string_reader: StringReader<'a>,
  26     token: token::Token,
  27     span: Span,
  28     /// Stack of open delimiters and their spans. Used for error message.
  29     open_braces: Vec<(token::DelimToken, Span)>,
  30     unmatched_braces: Vec<UnmatchedBrace>,
  31     /// The type and spans for all braces
  32     ///
  33     /// Used only for error recovery when arriving to EOF with mismatched braces.
  34     matching_delim_spans: Vec<(token::DelimToken, Span, Span)>,
  35     last_unclosed_found_span: Option<Span>,
  36 }
  37
  38 impl<'a> TokenTreesReader<'a> {
  39     // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
  40     fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
  41         let mut tts = Vec::new();
  42
  43         self.real_token();
  44         while self.token != token::Eof {
  45             tts.push(self.parse_token_tree()?);
  46         }
  47
  48         Ok(TokenStream::new(tts))
  49     }
  50
  51     // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
  52     fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
  53         let mut tts = vec![];
  54         loop {
  55             if let token::CloseDelim(..) = self.token {
  56                 return TokenStream::new(tts);
  57             }
  58
  59             match self.parse_token_tree() {
  60                 Ok(tree) => tts.push(tree),
  61                 Err(mut e) => {
  62                     e.emit();
  63                     return TokenStream::new(tts);
  64                 }
  65             }
  66         }
  67     }
  68
  69     fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
  70         let sm = self.string_reader.sess.source_map();
  71         match self.token {
  72             token::Eof => {
  73                 let msg = "this file contains an un-closed delimiter";
  74                 let mut err = self.string_reader.sess.span_diagnostic
  75                     .struct_span_err(self.span, msg);
  76                 for &(_, sp) in &self.open_braces {
  77                     err.span_label(sp, "un-closed delimiter");
  78                 }
  79
  80                 if let Some((delim, _)) = self.open_braces.last() {
  81                     if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter()
  82                         .filter(|(d, open_sp, close_sp)| {
  83                             if let Some(close_padding) = sm.span_to_margin(*close_sp) {
  84                                 if let Some(open_padding) = sm.span_to_margin(*open_sp) {
  85                                     return delim == d && close_padding != open_padding;
  86                                 }
  87                             }
  88                             false
  89                         }).next()  // these are in reverse order as they get inserted on close, but
  90                     {              // we want the last open/first close
  91                         err.span_label(
  92                             *open_sp,
  93                             "this delimiter might not be properly closed...",
  94                         );
  95                         err.span_label(
  96                             *close_sp,
  97                             "...as it matches this but it has different indentation",
  98                         );
  99                     }
 100                 }
 101                 Err(err)
 102             },
 103             token::OpenDelim(delim) => {
 104                 // The span for beginning of the delimited section
 105                 let pre_span = self.span;
 106
 107                 // Parse the open delimiter.
 108                 self.open_braces.push((delim, self.span));
 109                 self.real_token();
 110
 111                 // Parse the token trees within the delimiters.
 112                 // We stop at any delimiter so we can try to recover if the user
 113                 // uses an incorrect delimiter.
 114                 let tts = self.parse_token_trees_until_close_delim();
 115
 116                 // Expand to cover the entire delimited token tree
 117                 let delim_span = DelimSpan::from_pair(pre_span, self.span);
 118
 119                 match self.token {
 120                     // Correct delimiter.
 121                     token::CloseDelim(d) if d == delim => {
 122                         let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
 123                         if self.open_braces.len() == 0 {
 124                             // Clear up these spans to avoid suggesting them as we've found
 125                             // properly matched delimiters so far for an entire block.
 126                             self.matching_delim_spans.clear();
 127                         } else {
 128                             self.matching_delim_spans.push(
 129                                 (open_brace, open_brace_span, self.span),
 130                             );
 131                         }
 132                         // Parse the close delimiter.
 133                         self.real_token();
 134                     }
 135                     // Incorrect delimiter.
 136                     token::CloseDelim(other) => {
 137                         let mut unclosed_delimiter = None;
 138                         let mut candidate = None;
 139                         if self.last_unclosed_found_span != Some(self.span) {
 140                             // do not complain about the same unclosed delimiter multiple times
 141                             self.last_unclosed_found_span = Some(self.span);
 142                             // This is a conservative error: only report the last unclosed
 143                             // delimiter. The previous unclosed delimiters could actually be
 144                             // closed! The parser just hasn't gotten to them yet.
 145                             if let Some(&(_, sp)) = self.open_braces.last() {
 146                                 unclosed_delimiter = Some(sp);
 147                             };
 148                             if let Some(current_padding) = sm.span_to_margin(self.span) {
 149                                 for (brace, brace_span) in &self.open_braces {
 150                                     if let Some(padding) = sm.span_to_margin(*brace_span) {
 151                                         // high likelihood of these two corresponding
 152                                         if current_padding == padding && brace == &other {
 153                                             candidate = Some(*brace_span);
 154                                         }
 155                                     }
 156                                 }
 157                             }
 158                             let (tok, _) = self.open_braces.pop().unwrap();
 159                             self.unmatched_braces.push(UnmatchedBrace {
 160                                 expected_delim: tok,
 161                                 found_delim: other,
 162                                 found_span: self.span,
 163                                 unclosed_span: unclosed_delimiter,
 164                                 candidate_span: candidate,
 165                             });
 166                         } else {
 167                             self.open_braces.pop();
 168                         }
 169
 170                         // If the incorrect delimiter matches an earlier opening
 171                         // delimiter, then don't consume it (it can be used to
 172                         // close the earlier one). Otherwise, consume it.
 173                         // E.g., we try to recover from:
 174                         // fn foo() {
 175                         //     bar(baz(
 176                         // }  // Incorrect delimiter but matches the earlier `{`
 177                         if !self.open_braces.iter().any(|&(b, _)| b == other) {
 178                             self.real_token();
 179                         }
 180                     }
 181                     token::Eof => {
 182                         // Silently recover, the EOF token will be seen again
 183                         // and an error emitted then. Thus we don't pop from
 184                         // self.open_braces here.
 185                     },
 186                     _ => {}
 187                 }
 188
 189                 Ok(TokenTree::Delimited(
 190                     delim_span,
 191                     delim,
 192                     tts.into()
 193                 ).into())
 194             },
 195             token::CloseDelim(_) => {
 196                 // An unexpected closing delimiter (i.e., there is no
 197                 // matching opening delimiter).
 198                 let token_str = token_to_string(&self.token);
 199                 let msg = format!("unexpected close delimiter: `{}`", token_str);
 200                 let mut err = self.string_reader.sess.span_diagnostic
 201                     .struct_span_err(self.span, &msg);
 202                 err.span_label(self.span, "unexpected close delimiter");
 203                 Err(err)
 204             },
 205             _ => {
 206                 let tt = TokenTree::Token(self.span, self.token.clone());
 207                 // Note that testing for joint-ness here is done via the raw
 208                 // source span as the joint-ness is a property of the raw source
 209                 // rather than wanting to take `override_span` into account.
 210                 // Additionally, we actually check if the *next* pair of tokens
 211                 // is joint, but this is equivalent to checking the current pair.
 212                 let raw = self.string_reader.peek_span_src_raw;
 213                 self.real_token();
 214                 let is_joint = raw.hi() == self.string_reader.peek_span_src_raw.lo()
 215                     && token::is_op(&self.token);
 216                 Ok((tt, if is_joint { Joint } else { NonJoint }))
 217             }
 218         }
 219     }
 220
 221     fn real_token(&mut self) {
 222         let t = self.string_reader.real_token();
 223         self.token = t.tok;
 224         self.span = t.sp;
 225     }
 226 }