3 use crate::print::pprust::token_to_string;
4 use crate::parse::lexer::{StringReader, UnmatchedBrace};
5 use crate::parse::{token, PResult};
6 use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint};
8 impl<'a> StringReader<'a> {
9 crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
10 let mut tt_reader = TokenTreesReader {
13 span: syntax_pos::DUMMY_SP,
14 open_braces: Vec::new(),
15 unmatched_braces: Vec::new(),
16 matching_delim_spans: Vec::new(),
17 last_unclosed_found_span: None,
19 let res = tt_reader.parse_all_token_trees();
20 (res, tt_reader.unmatched_braces)
24 struct TokenTreesReader<'a> {
25 string_reader: StringReader<'a>,
26 token: token::TokenKind,
28 /// Stack of open delimiters and their spans. Used for error message.
29 open_braces: Vec<(token::DelimToken, Span)>,
30 unmatched_braces: Vec<UnmatchedBrace>,
31 /// The type and spans for all braces
33 /// Used only for error recovery when arriving to EOF with mismatched braces.
34 matching_delim_spans: Vec<(token::DelimToken, Span, Span)>,
35 last_unclosed_found_span: Option<Span>,
38 impl<'a> TokenTreesReader<'a> {
39 // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
40 fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
41 let mut tts = Vec::new();
44 while self.token != token::Eof {
45 tts.push(self.parse_token_tree()?);
48 Ok(TokenStream::new(tts))
51 // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
52 fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
55 if let token::CloseDelim(..) = self.token {
56 return TokenStream::new(tts);
59 match self.parse_token_tree() {
60 Ok(tree) => tts.push(tree),
63 return TokenStream::new(tts);
69 fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
70 let sm = self.string_reader.sess.source_map();
73 let msg = "this file contains an un-closed delimiter";
74 let mut err = self.string_reader.sess.span_diagnostic
75 .struct_span_err(self.span, msg);
76 for &(_, sp) in &self.open_braces {
77 err.span_label(sp, "un-closed delimiter");
80 if let Some((delim, _)) = self.open_braces.last() {
81 if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter()
82 .filter(|(d, open_sp, close_sp)| {
83 if let Some(close_padding) = sm.span_to_margin(*close_sp) {
84 if let Some(open_padding) = sm.span_to_margin(*open_sp) {
85 return delim == d && close_padding != open_padding;
89 }).next() // these are in reverse order as they get inserted on close, but
90 { // we want the last open/first close
93 "this delimiter might not be properly closed...",
97 "...as it matches this but it has different indentation",
103 token::OpenDelim(delim) => {
104 // The span for beginning of the delimited section
105 let pre_span = self.span;
107 // Parse the open delimiter.
108 self.open_braces.push((delim, self.span));
111 // Parse the token trees within the delimiters.
112 // We stop at any delimiter so we can try to recover if the user
113 // uses an incorrect delimiter.
114 let tts = self.parse_token_trees_until_close_delim();
116 // Expand to cover the entire delimited token tree
117 let delim_span = DelimSpan::from_pair(pre_span, self.span);
120 // Correct delimiter.
121 token::CloseDelim(d) if d == delim => {
122 let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
123 if self.open_braces.len() == 0 {
124 // Clear up these spans to avoid suggesting them as we've found
125 // properly matched delimiters so far for an entire block.
126 self.matching_delim_spans.clear();
128 self.matching_delim_spans.push(
129 (open_brace, open_brace_span, self.span),
132 // Parse the close delimiter.
135 // Incorrect delimiter.
136 token::CloseDelim(other) => {
137 let mut unclosed_delimiter = None;
138 let mut candidate = None;
139 if self.last_unclosed_found_span != Some(self.span) {
140 // do not complain about the same unclosed delimiter multiple times
141 self.last_unclosed_found_span = Some(self.span);
142 // This is a conservative error: only report the last unclosed
143 // delimiter. The previous unclosed delimiters could actually be
144 // closed! The parser just hasn't gotten to them yet.
145 if let Some(&(_, sp)) = self.open_braces.last() {
146 unclosed_delimiter = Some(sp);
148 if let Some(current_padding) = sm.span_to_margin(self.span) {
149 for (brace, brace_span) in &self.open_braces {
150 if let Some(padding) = sm.span_to_margin(*brace_span) {
151 // high likelihood of these two corresponding
152 if current_padding == padding && brace == &other {
153 candidate = Some(*brace_span);
158 let (tok, _) = self.open_braces.pop().unwrap();
159 self.unmatched_braces.push(UnmatchedBrace {
162 found_span: self.span,
163 unclosed_span: unclosed_delimiter,
164 candidate_span: candidate,
167 self.open_braces.pop();
170 // If the incorrect delimiter matches an earlier opening
171 // delimiter, then don't consume it (it can be used to
172 // close the earlier one). Otherwise, consume it.
173 // E.g., we try to recover from:
176 // } // Incorrect delimiter but matches the earlier `{`
177 if !self.open_braces.iter().any(|&(b, _)| b == other) {
182 // Silently recover, the EOF token will be seen again
183 // and an error emitted then. Thus we don't pop from
184 // self.open_braces here.
189 Ok(TokenTree::Delimited(
195 token::CloseDelim(_) => {
196 // An unexpected closing delimiter (i.e., there is no
197 // matching opening delimiter).
198 let token_str = token_to_string(&self.token);
199 let msg = format!("unexpected close delimiter: `{}`", token_str);
200 let mut err = self.string_reader.sess.span_diagnostic
201 .struct_span_err(self.span, &msg);
202 err.span_label(self.span, "unexpected close delimiter");
206 let tt = TokenTree::token(self.span, self.token.clone());
207 // Note that testing for joint-ness here is done via the raw
208 // source span as the joint-ness is a property of the raw source
209 // rather than wanting to take `override_span` into account.
210 // Additionally, we actually check if the *next* pair of tokens
211 // is joint, but this is equivalent to checking the current pair.
212 let raw = self.string_reader.peek_span_src_raw;
214 let is_joint = raw.hi() == self.string_reader.peek_span_src_raw.lo()
215 && token::is_op(&self.token);
216 Ok((tt, if is_joint { Joint } else { NonJoint }))
221 fn real_token(&mut self) {
222 let t = self.string_reader.real_token();