Error recovery in the tokeniser

Closes #31994

diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 66912abb6f5a65d2131364925b8d5b080f2fdb87..3010c040914df401a53bf24b148d191914c663ec 100644 (file)
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -268,8 +268,8 @@ pub struct Parser<'a> {
     /// Used to determine the path to externally loaded source files
     pub filename: Option<String>,
     pub mod_path_stack: Vec<InternedString>,
-    /// Stack of spans of open delimiters. Used for error message.
-    pub open_braces: Vec<Span>,
+    /// Stack of open delimiters and their spans. Used for error message.
+    pub open_braces: Vec<(token::DelimToken, Span)>,
     /// Flag if this parser "owns" the directory that it is currently parsing
     /// in. This will affect how nested files are looked up.
     pub owns_directory: bool,
@@ -895,7 +895,7 @@ pub fn parse_seq_to_before_end<T, F>(&mut self,
                                          sep: SeqSep,
                                          f: F)
                                          -> Vec<T>
-        where F: FnMut(&mut Parser<'a>) -> PResult<'a,  T>,
+        where F: FnMut(&mut Parser<'a>) -> PResult<'a,  T>
     {
         self.parse_seq_to_before_tokens(&[ket], sep, f, |mut e| e.emit())
     }
@@ -2755,8 +2755,8 @@ pub fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> {
                 let mut err: DiagnosticBuilder<'a> =
                     self.diagnostic().struct_span_err(self.span,
                                                       "this file contains an un-closed delimiter");
-                for sp in &self.open_braces {
-                    err.span_help(*sp, "did you mean to close this delimiter?");
+                for &(_, sp) in &self.open_braces {
+                    err.span_help(sp, "did you mean to close this delimiter?");
                 }
 
                 Err(err)
@@ -2766,23 +2766,66 @@ pub fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> {
                 let pre_span = self.span;
 
                 // Parse the open delimiter.
-                self.open_braces.push(self.span);
+                self.open_braces.push((delim, self.span));
                 let open_span = self.span;
                 self.bump();
 
-                // Parse the token trees within the delimiters
-                let tts = self.parse_seq_to_before_end(&token::CloseDelim(delim),
-                                                       SeqSep::none(),
-                                                       |p| p.parse_token_tree());
+                // Parse the token trees within the delimiters.
+                // We stop at any delimiter so we can try to recover if the user
+                // uses an incorrect delimiter.
+                let tts = self.parse_seq_to_before_tokens(&[&token::CloseDelim(token::Brace),
+                                                            &token::CloseDelim(token::Paren),
+                                                            &token::CloseDelim(token::Bracket)],
+                                                          SeqSep::none(),
+                                                          |p| p.parse_token_tree(),
+                                                          |mut e| e.emit());
 
-                // Parse the close delimiter.
                 let close_span = self.span;
-                self.bump();
-                self.open_braces.pop().unwrap();
-
                 // Expand to cover the entire delimited token tree
                 let span = Span { hi: close_span.hi, ..pre_span };
 
+                match self.token {
+                    // Correct delmiter.
+                    token::CloseDelim(d) if d == delim => {
+                        self.open_braces.pop().unwrap();
+
+                        // Parse the close delimiter.
+                        self.bump();
+                    }
+                    // Incorect delimiter.
+                    token::CloseDelim(other) => {
+                        let token_str = self.this_token_to_string();
+                        let mut err = self.diagnostic().struct_span_err(self.span,
+                            &format!("incorrect close delimiter: `{}`", token_str));
+                        // This is a conservative error: only report the last unclosed delimiter.
+                        // The previous unclosed delimiters could actually be closed! The parser
+                        // just hasn't gotten to them yet.
+                        if let Some(&(_, sp)) = self.open_braces.last() {
+                            err.span_note(sp, "unclosed delimiter");
+                        };
+                        err.emit();
+
+                        self.open_braces.pop().unwrap();
+
+                        // If the incorrect delimter matches an earlier opening
+                        // delimiter, then don't consume it (it can be used to
+                        // close the earlier one)Otherwise, consume it.
+                        // E.g., we try to recover from:
+                        // fn foo() {
+                        //     bar(baz(
+                        // }  // Incorrect delimiter but matches the earlier `{`
+                        if !self.open_braces.iter().any(|&(b, _)| b == other) {
+                            self.bump();
+                        }
+                    }
+                    token::Eof => {
+                        // Silently recover, the EOF token will be seen again
+                        // and an error emitted then. Thus we don't pop from
+                        // self.open_braces here.
+                    },
+                    _ => unreachable!(),
+                }
+
                 Ok(TokenTree::Delimited(span, Rc::new(Delimited {
                     delim: delim,
                     open_span: open_span,
@@ -2798,17 +2841,7 @@ pub fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> {
                 maybe_whole!(deref self, NtTT);
                 match self.token {
                     token::CloseDelim(_) => {
-                        let token_str = self.this_token_to_string();
-                        let mut err = self.diagnostic().struct_span_err(self.span,
-                            &format!("incorrect close delimiter: `{}`", token_str));
-                        // This is a conservative error: only report the last unclosed delimiter.
-                        // The previous unclosed delimiters could actually be closed! The parser
-                        // just hasn't gotten to them yet.
-                        if let Some(&sp) = self.open_braces.last() {
-                            err.span_note(sp, "unclosed delimiter");
-                        };
-
-                        Err(err)
+                        panic!("should have been caught above");
                     },
                     /* we ought to allow different depths of unquotation */
                     token::Dollar | token::SubstNt(..) if self.quote_depth > 0 => {