src/libsyntax/parse/lexer/tokentrees.rs

   1 use crate::print::pprust::token_to_string;
   2 use crate::parse::lexer::StringReader;
   3 use crate::parse::{token, PResult};
   4 use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint};
   5
   6 impl<'a> StringReader<'a> {
   7     // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
   8     crate fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
   9         let mut tts = Vec::new();
  10
  11         while self.token != token::Eof {
  12             tts.push(self.parse_token_tree()?);
  13         }
  14
  15         Ok(TokenStream::new(tts))
  16     }
  17
  18     // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
  19     fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
  20         let mut tts = vec![];
  21         loop {
  22             if let token::CloseDelim(..) = self.token {
  23                 return TokenStream::new(tts);
  24             }
  25
  26             match self.parse_token_tree() {
  27                 Ok(tree) => tts.push(tree),
  28                 Err(mut e) => {
  29                     e.emit();
  30                     return TokenStream::new(tts);
  31                 }
  32             }
  33         }
  34     }
  35
  36     fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
  37         let sm = self.sess.source_map();
  38         match self.token {
  39             token::Eof => {
  40                 let msg = "this file contains an un-closed delimiter";
  41                 let mut err = self.sess.span_diagnostic.struct_span_err(self.span, msg);
  42                 for &(_, sp) in &self.open_braces {
  43                     err.span_label(sp, "un-closed delimiter");
  44                 }
  45
  46                 if let Some((delim, _)) = self.open_braces.last() {
  47                     if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter()
  48                         .filter(|(d, open_sp, close_sp)| {
  49
  50                         if let Some(close_padding) = sm.span_to_margin(*close_sp) {
  51                             if let Some(open_padding) = sm.span_to_margin(*open_sp) {
  52                                 return delim == d && close_padding != open_padding;
  53                             }
  54                         }
  55                         false
  56                         }).next()  // these are in reverse order as they get inserted on close, but
  57                     {              // we want the last open/first close
  58                         err.span_label(
  59                             *open_sp,
  60                             "this delimiter might not be properly closed...",
  61                         );
  62                         err.span_label(
  63                             *close_sp,
  64                             "...as it matches this but it has different indentation",
  65                         );
  66                     }
  67                 }
  68                 Err(err)
  69             },
  70             token::OpenDelim(delim) => {
  71                 // The span for beginning of the delimited section
  72                 let pre_span = self.span;
  73
  74                 // Parse the open delimiter.
  75                 self.open_braces.push((delim, self.span));
  76                 self.real_token();
  77
  78                 // Parse the token trees within the delimiters.
  79                 // We stop at any delimiter so we can try to recover if the user
  80                 // uses an incorrect delimiter.
  81                 let tts = self.parse_token_trees_until_close_delim();
  82
  83                 // Expand to cover the entire delimited token tree
  84                 let delim_span = DelimSpan::from_pair(pre_span, self.span);
  85
  86                 match self.token {
  87                     // Correct delimiter.
  88                     token::CloseDelim(d) if d == delim => {
  89                         let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
  90                         if self.open_braces.len() == 0 {
  91                             // Clear up these spans to avoid suggesting them as we've found
  92                             // properly matched delimiters so far for an entire block.
  93                             self.matching_delim_spans.clear();
  94                         } else {
  95                             self.matching_delim_spans.push(
  96                                 (open_brace, open_brace_span, self.span),
  97                             );
  98                         }
  99                         // Parse the close delimiter.
 100                         self.real_token();
 101                     }
 102                     // Incorrect delimiter.
 103                     token::CloseDelim(other) => {
 104                         let token_str = token_to_string(&self.token);
 105                         if self.last_unclosed_found_span != Some(self.span) {
 106                             // do not complain about the same unclosed delimiter multiple times
 107                             self.last_unclosed_found_span = Some(self.span);
 108                             let msg = format!("incorrect close delimiter: `{}`", token_str);
 109                             let mut err = self.sess.span_diagnostic.struct_span_err(
 110                                 self.span,
 111                                 &msg,
 112                             );
 113                             err.span_label(self.span, "incorrect close delimiter");
 114                             // This is a conservative error: only report the last unclosed
 115                             // delimiter. The previous unclosed delimiters could actually be
 116                             // closed! The parser just hasn't gotten to them yet.
 117                             if let Some(&(_, sp)) = self.open_braces.last() {
 118                                 err.span_label(sp, "un-closed delimiter");
 119                             };
 120                             if let Some(current_padding) = sm.span_to_margin(self.span) {
 121                                 for (brace, brace_span) in &self.open_braces {
 122                                     if let Some(padding) = sm.span_to_margin(*brace_span) {
 123                                         // high likelihood of these two corresponding
 124                                         if current_padding == padding && brace == &other {
 125                                             err.span_label(
 126                                                 *brace_span,
 127                                                 "close delimiter possibly meant for this",
 128                                             );
 129                                         }
 130                                     }
 131                                 }
 132                             }
 133                             err.emit();
 134                         }
 135                         self.open_braces.pop().unwrap();
 136
 137                         // If the incorrect delimiter matches an earlier opening
 138                         // delimiter, then don't consume it (it can be used to
 139                         // close the earlier one). Otherwise, consume it.
 140                         // E.g., we try to recover from:
 141                         // fn foo() {
 142                         //     bar(baz(
 143                         // }  // Incorrect delimiter but matches the earlier `{`
 144                         if !self.open_braces.iter().any(|&(b, _)| b == other) {
 145                             self.real_token();
 146                         }
 147                     }
 148                     token::Eof => {
 149                         // Silently recover, the EOF token will be seen again
 150                         // and an error emitted then. Thus we don't pop from
 151                         // self.open_braces here.
 152                     },
 153                     _ => {}
 154                 }
 155
 156                 Ok(TokenTree::Delimited(
 157                     delim_span,
 158                     delim,
 159                     tts.into()
 160                 ).into())
 161             },
 162             token::CloseDelim(_) => {
 163                 // An unexpected closing delimiter (i.e., there is no
 164                 // matching opening delimiter).
 165                 let token_str = token_to_string(&self.token);
 166                 let msg = format!("unexpected close delimiter: `{}`", token_str);
 167                 let mut err = self.sess.span_diagnostic.struct_span_err(self.span, &msg);
 168                 err.span_label(self.span, "unexpected close delimiter");
 169                 Err(err)
 170             },
 171             _ => {
 172                 let tt = TokenTree::Token(self.span, self.token.clone());
 173                 // Note that testing for joint-ness here is done via the raw
 174                 // source span as the joint-ness is a property of the raw source
 175                 // rather than wanting to take `override_span` into account.
 176                 let raw = self.span_src_raw;
 177                 self.real_token();
 178                 let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token);
 179                 Ok((tt, if is_joint { Joint } else { NonJoint }))
 180             }
 181         }
 182     }
 183 }