Make clear that `TokenTree::Token` shouldn't contain a delimiter.

diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index dd01fc8ffc506db7b26bd63a1cf94af6286e3709..ab554d70c462369d587bab710f35aa1e68fda6d2 100644 (file)
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -41,7 +41,8 @@
 /// Nothing special happens to misnamed or misplaced `SubstNt`s.
 #[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum TokenTree {
-    /// A single token.
+    /// A single token. Should never be `OpenDelim` or `CloseDelim`, because
+    /// delimiters are implicitly represented by `Delimited`.
     Token(Token, Spacing),
     /// A delimited sequence of token trees.
     Delimited(DelimSpan, Delimiter, TokenStream),

diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index 0499a56a09db46afd3be2d864170511e51c0aad5..510588f8a5f7223b53b0d2bbda03f7f8d1633557 100644 (file)
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -290,7 +290,13 @@ fn inlined_next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
                             self.frame.tree_cursor.replace_prev_and_rewind(desugared);
                             // Continue to get the first token of the desugared doc comment.
                         }
-                        _ => return (token.clone(), spacing),
+                        _ => {
+                            debug_assert!(!matches!(
+                                token.kind,
+                                token::OpenDelim(_) | token::CloseDelim(_)
+                            ));
+                            return (token.clone(), spacing);
+                        }
                     },
                     &TokenTree::Delimited(sp, delim, ref tts) => {
                         // Set `open_delim` to true here because we deal with it immediately.