Auto merge of #69590 - Dylan-DPC:rollup-i3z0sic, r=Dylan-DPC

[rust.git] / src / librustc_parse / parser / mod.rs

diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs
index 75d4b3750f164ff53a344ac96237a092345a3c88..58a03ee2a74b49c8e952cbdee9f433fc27494ba3 100644 (file)
--- a/src/librustc_parse/parser/mod.rs
+++ b/src/librustc_parse/parser/mod.rs
@@ -86,23 +86,22 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath {
 #[derive(Clone)]
 pub struct Parser<'a> {
     pub sess: &'a ParseSess,
+    /// The current non-normalized token.
+    pub token: Token,
     /// The current normalized token.
     /// "Normalized" means that some interpolated tokens
     /// (`$i: ident` and `$l: lifetime` meta-variables) are replaced
     /// with non-interpolated identifier and lifetime tokens they refer to.
-    /// Use span from this token if you need an isolated span.
-    pub token: Token,
-    /// The current non-normalized token if it's different from `token`.
-    /// Use span from this token if you need to concatenate it with some neighbouring spans.
-    unnormalized_token: Token,
+    /// Use this if you need to check for `token::Ident` or `token::Lifetime` specifically,
+    /// this also includes edition checks for edition-specific keyword identifiers.
+    pub normalized_token: Token,
+    /// The previous non-normalized token.
+    pub prev_token: Token,
     /// The previous normalized token.
-    /// Use span from this token if you need an isolated span.
-    prev_token: Token,
-    /// The previous non-normalized token if it's different from `prev_token`.
-    /// Use span from this token if you need to concatenate it with some neighbouring spans.
-    unnormalized_prev_token: Token,
-    /// Equivalent to `unnormalized_prev_token.span`.
-    /// FIXME: Remove in favor of `(unnormalized_)prev_token.span`.
+    /// Use this if you need to check for `token::Ident` or `token::Lifetime` specifically,
+    /// this also includes edition checks for edition-specific keyword identifiers.
+    pub normalized_prev_token: Token,
+    /// FIXME: Remove in favor of the equivalent `prev_token.span`.
     pub prev_span: Span,
     restrictions: Restrictions,
     /// Used to determine the path to externally loaded source files.
@@ -263,8 +262,7 @@ fn next_desugared(&mut self) -> Token {
             ]
             .iter()
             .cloned()
-            .collect::<TokenStream>()
-            .into(),
+            .collect::<TokenStream>(),
         );
 
         self.stack.push(mem::replace(
@@ -376,9 +374,9 @@ pub fn new(
         let mut parser = Parser {
             sess,
             token: Token::dummy(),
-            unnormalized_token: Token::dummy(),
+            normalized_token: Token::dummy(),
             prev_token: Token::dummy(),
-            unnormalized_prev_token: Token::dummy(),
+            normalized_prev_token: Token::dummy(),
             prev_span: DUMMY_SP,
             restrictions: Restrictions::empty(),
             recurse_into_file_modules,
@@ -389,7 +387,7 @@ pub fn new(
             root_module_name: None,
             expected_tokens: Vec::new(),
             token_cursor: TokenCursor {
-                frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, &tokens.into()),
+                frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, &tokens),
                 stack: Vec::new(),
             },
             desugar_doc_comments,
@@ -483,7 +481,7 @@ pub fn parse_ident(&mut self) -> PResult<'a, ast::Ident> {
     }
 
     fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> {
-        match self.token.kind {
+        match self.normalized_token.kind {
             token::Ident(name, _) => {
                 if self.token.is_reserved_ident() {
                     let mut err = self.expected_ident_found();
@@ -493,9 +491,8 @@ fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> {
                         return Err(err);
                     }
                 }
-                let span = self.token.span;
                 self.bump();
-                Ok(Ident::new(name, span))
+                Ok(Ident::new(name, self.normalized_prev_token.span))
             }
             _ => Err(match self.prev_token.kind {
                 TokenKind::DocComment(..) => {
@@ -825,16 +822,16 @@ fn parse_paren_comma_seq<T>(
     // tokens are replaced with usual identifier and lifetime tokens,
     // so the former are never encountered during normal parsing.
     crate fn set_token(&mut self, token: Token) {
-        self.unnormalized_token = token;
-        self.token = match &self.unnormalized_token.kind {
+        self.token = token;
+        self.normalized_token = match &self.token.kind {
             token::Interpolated(nt) => match **nt {
                 token::NtIdent(ident, is_raw) => {
                     Token::new(token::Ident(ident.name, is_raw), ident.span)
                 }
                 token::NtLifetime(ident) => Token::new(token::Lifetime(ident.name), ident.span),
-                _ => self.unnormalized_token.clone(),
+                _ => self.token.clone(),
             },
-            _ => self.unnormalized_token.clone(),
+            _ => self.token.clone(),
         }
     }
 
@@ -848,11 +845,11 @@ fn bump_with(&mut self, next_token: Token) {
 
         // Update the current and previous tokens.
         self.prev_token = self.token.take();
-        self.unnormalized_prev_token = self.unnormalized_token.take();
+        self.normalized_prev_token = self.normalized_token.take();
         self.set_token(next_token);
 
         // Update fields derived from the previous token.
-        self.prev_span = self.unnormalized_prev_token.span;
+        self.prev_span = self.prev_token.span;
 
         // Diagnostics.
         self.expected_tokens.clear();
@@ -860,7 +857,7 @@ fn bump_with(&mut self, next_token: Token) {
 
     /// Advance the parser by one token.
     pub fn bump(&mut self) {
-        let next_token = self.next_tok(self.unnormalized_token.span);
+        let next_token = self.next_tok(self.token.span);
         self.bump_with(next_token);
     }
 
@@ -891,7 +888,7 @@ fn is_keyword_ahead(&self, dist: usize, kws: &[Symbol]) -> bool {
     /// Parses asyncness: `async` or nothing.
     fn parse_asyncness(&mut self) -> Async {
         if self.eat_keyword(kw::Async) {
-            let span = self.prev_span;
+            let span = self.normalized_prev_token.span;
             Async::Yes { span, closure_id: DUMMY_NODE_ID, return_impl_trait_id: DUMMY_NODE_ID }
         } else {
             Async::No
@@ -1006,7 +1003,7 @@ pub fn parse_token_tree(&mut self) -> TokenTree {
                 );
                 self.set_token(Token::new(TokenKind::CloseDelim(frame.delim), frame.span.close));
                 self.bump();
-                TokenTree::Delimited(frame.span, frame.delim, frame.tree_cursor.stream.into())
+                TokenTree::Delimited(frame.span, frame.delim, frame.tree_cursor.stream)
             }
             token::CloseDelim(_) | token::Eof => unreachable!(),
             _ => {