1 use super::{StringReader, UnmatchedBrace};
2 use rustc_ast::token::{self, Delimiter, Token};
3 use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
4 use rustc_ast_pretty::pprust::token_to_string;
5 use rustc_data_structures::fx::FxHashMap;
6 use rustc_errors::{PErr, PResult};
9 pub(super) struct TokenTreesReader<'a> {
10 string_reader: StringReader<'a>,
11 /// The "next" token, which has been obtained from the `StringReader` but
12 /// not yet handled by the `TokenTreesReader`.
14 /// Stack of open delimiters and their spans. Used for error message.
15 open_braces: Vec<(Delimiter, Span)>,
16 unmatched_braces: Vec<UnmatchedBrace>,
17 /// The type and spans for all braces
19 /// Used only for error recovery when arriving to EOF with mismatched braces.
20 matching_delim_spans: Vec<(Delimiter, Span, Span)>,
21 last_unclosed_found_span: Option<Span>,
22 /// Collect empty block spans that might have been auto-inserted by editors.
23 last_delim_empty_block_spans: FxHashMap<Delimiter, Span>,
24 /// Collect the spans of braces (Open, Close). Used only
25 /// for detecting if blocks are empty and only braces.
26 matching_block_spans: Vec<(Span, Span)>,
29 impl<'a> TokenTreesReader<'a> {
30 pub(super) fn parse_all_token_trees(
31 string_reader: StringReader<'a>,
32 ) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
33 let mut tt_reader = TokenTreesReader {
35 token: Token::dummy(),
36 open_braces: Vec::new(),
37 unmatched_braces: Vec::new(),
38 matching_delim_spans: Vec::new(),
39 last_unclosed_found_span: None,
40 last_delim_empty_block_spans: FxHashMap::default(),
41 matching_block_spans: Vec::new(),
43 let res = tt_reader.parse_token_trees(/* is_delimited */ false);
44 (res, tt_reader.unmatched_braces)
47 // Parse a stream of tokens into a list of `TokenTree`s.
48 fn parse_token_trees(&mut self, is_delimited: bool) -> PResult<'a, TokenStream> {
49 self.token = self.string_reader.next_token().0;
50 let mut buf = Vec::new();
52 match self.token.kind {
53 token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)),
54 token::CloseDelim(delim) => {
55 return if is_delimited {
56 Ok(TokenStream::new(buf))
58 Err(self.close_delim_err(delim))
63 self.eof_err().emit();
65 return Ok(TokenStream::new(buf));
68 // Get the next normal token. This might require getting multiple adjacent
69 // single-char tokens and joining them together.
70 let (this_spacing, next_tok) = loop {
71 let (next_tok, is_next_tok_preceded_by_whitespace) =
72 self.string_reader.next_token();
73 if !is_next_tok_preceded_by_whitespace {
74 if let Some(glued) = self.token.glue(&next_tok) {
78 if next_tok.is_op() { Spacing::Joint } else { Spacing::Alone };
79 break (this_spacing, next_tok);
82 break (Spacing::Alone, next_tok);
85 let this_tok = std::mem::replace(&mut self.token, next_tok);
86 buf.push(TokenTree::Token(this_tok, this_spacing));
92 fn eof_err(&mut self) -> PErr<'a> {
93 let msg = "this file contains an unclosed delimiter";
94 let mut err = self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, msg);
95 for &(_, sp) in &self.open_braces {
96 err.span_label(sp, "unclosed delimiter");
97 self.unmatched_braces.push(UnmatchedBrace {
98 expected_delim: Delimiter::Brace,
100 found_span: self.token.span,
101 unclosed_span: Some(sp),
102 candidate_span: None,
106 if let Some((delim, _)) = self.open_braces.last() {
107 if let Some((_, open_sp, close_sp)) =
108 self.matching_delim_spans.iter().find(|(d, open_sp, close_sp)| {
109 let sm = self.string_reader.sess.source_map();
110 if let Some(close_padding) = sm.span_to_margin(*close_sp) {
111 if let Some(open_padding) = sm.span_to_margin(*open_sp) {
112 return delim == d && close_padding != open_padding;
117 // these are in reverse order as they get inserted on close, but
119 // we want the last open/first close
120 err.span_label(*open_sp, "this delimiter might not be properly closed...");
121 err.span_label(*close_sp, "...as it matches this but it has different indentation");
127 fn parse_token_tree_open_delim(&mut self, open_delim: Delimiter) -> TokenTree {
128 // The span for beginning of the delimited section
129 let pre_span = self.token.span;
131 self.open_braces.push((open_delim, self.token.span));
133 // Parse the token trees within the delimiters.
134 // We stop at any delimiter so we can try to recover if the user
135 // uses an incorrect delimiter.
136 let tts = self.parse_token_trees(/* is_delimited */ true).unwrap();
138 // Expand to cover the entire delimited token tree
139 let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
141 match self.token.kind {
142 // Correct delimiter.
143 token::CloseDelim(close_delim) if close_delim == open_delim => {
144 let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
145 let close_brace_span = self.token.span;
148 let empty_block_span = open_brace_span.to(close_brace_span);
149 let sm = self.string_reader.sess.source_map();
150 if !sm.is_multiline(empty_block_span) {
151 // Only track if the block is in the form of `{}`, otherwise it is
152 // likely that it was written on purpose.
153 self.last_delim_empty_block_spans.insert(open_delim, empty_block_span);
158 if let (Delimiter::Brace, Delimiter::Brace) = (open_brace, open_delim) {
159 self.matching_block_spans.push((open_brace_span, close_brace_span));
162 if self.open_braces.is_empty() {
163 // Clear up these spans to avoid suggesting them as we've found
164 // properly matched delimiters so far for an entire block.
165 self.matching_delim_spans.clear();
167 self.matching_delim_spans.push((open_brace, open_brace_span, close_brace_span));
169 // Move past the closing delimiter.
170 self.token = self.string_reader.next_token().0;
172 // Incorrect delimiter.
173 token::CloseDelim(close_delim) => {
174 let mut unclosed_delimiter = None;
175 let mut candidate = None;
177 if self.last_unclosed_found_span != Some(self.token.span) {
178 // do not complain about the same unclosed delimiter multiple times
179 self.last_unclosed_found_span = Some(self.token.span);
180 // This is a conservative error: only report the last unclosed
181 // delimiter. The previous unclosed delimiters could actually be
182 // closed! The parser just hasn't gotten to them yet.
183 if let Some(&(_, sp)) = self.open_braces.last() {
184 unclosed_delimiter = Some(sp);
186 let sm = self.string_reader.sess.source_map();
187 if let Some(current_padding) = sm.span_to_margin(self.token.span) {
188 for (brace, brace_span) in &self.open_braces {
189 if let Some(padding) = sm.span_to_margin(*brace_span) {
190 // high likelihood of these two corresponding
191 if current_padding == padding && brace == &close_delim {
192 candidate = Some(*brace_span);
197 let (tok, _) = self.open_braces.pop().unwrap();
198 self.unmatched_braces.push(UnmatchedBrace {
200 found_delim: Some(close_delim),
201 found_span: self.token.span,
202 unclosed_span: unclosed_delimiter,
203 candidate_span: candidate,
206 self.open_braces.pop();
209 // If the incorrect delimiter matches an earlier opening
210 // delimiter, then don't consume it (it can be used to
211 // close the earlier one). Otherwise, consume it.
212 // E.g., we try to recover from:
215 // } // Incorrect delimiter but matches the earlier `{`
216 if !self.open_braces.iter().any(|&(b, _)| b == close_delim) {
217 self.token = self.string_reader.next_token().0;
221 // Silently recover, the EOF token will be seen again
222 // and an error emitted then. Thus we don't pop from
223 // self.open_braces here.
228 TokenTree::Delimited(delim_span, open_delim, tts)
231 fn close_delim_err(&mut self, delim: Delimiter) -> PErr<'a> {
232 // An unexpected closing delimiter (i.e., there is no
233 // matching opening delimiter).
234 let token_str = token_to_string(&self.token);
235 let msg = format!("unexpected closing delimiter: `{}`", token_str);
237 self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, &msg);
239 // Braces are added at the end, so the last element is the biggest block
240 if let Some(parent) = self.matching_block_spans.last() {
241 if let Some(span) = self.last_delim_empty_block_spans.remove(&delim) {
242 // Check if the (empty block) is in the last properly closed block
243 if (parent.0.to(parent.1)).contains(span) {
244 err.span_label(span, "block is empty, you might have not meant to close it");
246 err.span_label(parent.0, "this opening brace...");
247 err.span_label(parent.1, "...matches this closing brace");
250 err.span_label(parent.0, "this opening brace...");
251 err.span_label(parent.1, "...matches this closing brace");
255 err.span_label(self.token.span, "unexpected closing delimiter");