Auto merge of #75912 - scottmcm:manuallydrop-vs-forget, r=Mark-Simulacrum

[rust.git] / src / librustc_lexer / src / lib.rs

diff --git a/src/librustc_lexer/src/lib.rs b/src/librustc_lexer/src/lib.rs
index f94466b03ed1ed44803a0c0f164687dbe7dd4096..b7d6194cd77cf956d16ede3087c5b8a0a1ae4517 100644 (file)
--- a/src/librustc_lexer/src/lib.rs
+++ b/src/librustc_lexer/src/lib.rs
@@ -35,6 +35,7 @@
 /// Parsed token.
 /// It doesn't contain information about data that has been parsed,
 /// only the type of the token and its size.
+#[derive(Debug)]
 pub struct Token {
     pub kind: TokenKind,
     pub len: usize,
@@ -51,12 +52,12 @@ fn new(kind: TokenKind, len: usize) -> Token {
 pub enum TokenKind {
     // Multi-char tokens:
     /// "// comment"
-    LineComment,
+    LineComment { doc_style: Option<DocStyle> },
     /// `/* block comment */`
     ///
     /// Block comments can be recursive, so the sequence like `/* /* */`
     /// will not be considered terminated and will result in a parsing error.
-    BlockComment { terminated: bool },
+    BlockComment { doc_style: Option<DocStyle>, terminated: bool },
     /// Any whitespace characters sequence.
     Whitespace,
     /// "ident" or "continue"
@@ -129,6 +130,12 @@ pub enum TokenKind {
     Unknown,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum DocStyle {
+    Outer,
+    Inner,
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum LiteralKind {
     /// "12_u8", "0o100", "0b120i99"
@@ -188,7 +195,7 @@ pub fn strip_shebang(input: &str) -> Option<usize> {
         // a doc comment (due to `TokenKind::(Line,Block)Comment` ambiguity at lexer level),
         // then it may be valid Rust code, so consider it Rust code.
         let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok|
-            !matches!(tok, TokenKind::Whitespace | TokenKind::LineComment | TokenKind::BlockComment { .. })
+            !matches!(tok, TokenKind::Whitespace | TokenKind::LineComment { .. } | TokenKind::BlockComment { .. })
         );
         if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
             // No other choice than to consider this a shebang.
@@ -410,13 +417,32 @@ fn advance_token(&mut self) -> Token {
     fn line_comment(&mut self) -> TokenKind {
         debug_assert!(self.prev() == '/' && self.first() == '/');
         self.bump();
+
+        let doc_style = match self.first() {
+            // `//!` is an inner line doc comment.
+            '!' => Some(DocStyle::Inner),
+            // `////` (more than 3 slashes) is not considered a doc comment.
+            '/' if self.second() != '/' => Some(DocStyle::Outer),
+            _ => None,
+        };
+
         self.eat_while(|c| c != '\n');
-        LineComment
+        LineComment { doc_style }
     }
 
     fn block_comment(&mut self) -> TokenKind {
         debug_assert!(self.prev() == '/' && self.first() == '*');
         self.bump();
+
+        let doc_style = match self.first() {
+            // `/*!` is an inner block doc comment.
+            '!' => Some(DocStyle::Inner),
+            // `/***` (more than 2 stars) is not considered a doc comment.
+            // `/**/` is not considered a doc comment.
+            '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
+            _ => None,
+        };
+
         let mut depth = 1usize;
         while let Some(c) = self.bump() {
             match c {
@@ -438,7 +464,7 @@ fn block_comment(&mut self) -> TokenKind {
             }
         }
 
-        BlockComment { terminated: depth == 0 }
+        BlockComment { doc_style, terminated: depth == 0 }
     }
 
     fn whitespace(&mut self) -> TokenKind {