1 //! Low-level Rust lexer.
3 //! The idea with `rustc_lexer` is to make a reusable library,
4 //! by separating out pure lexing and rustc-specific concerns, like spans,
5 //! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
6 //! produces simple tokens which are a pair of type-tag and a bit of original text,
7 //! and does not report errors, instead storing them as flags on the token.
9 //! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
10 //! For that see [`rustc_parse::lexer`], which converts this basic token stream
11 //! into wide tokens used by actual parser.
13 //! The purpose of this crate is to convert raw sources into a labeled sequence
14 //! of well-known token types, so building an actual Rust token stream will
17 //! The main entity of this crate is the [`TokenKind`] enum which represents common
20 //! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
21 #![deny(rustc::untranslatable_diagnostic)]
22 #![deny(rustc::diagnostic_outside_of_impl)]
23 // We want to be able to build this crate with a stable compiler, so no
24 // `#![feature]` attributes should be added.
32 pub use crate::cursor::Cursor;
34 use self::LiteralKind::*;
35 use self::TokenKind::*;
36 use crate::cursor::EOF_CHAR;
37 use std::convert::TryFrom;
40 /// It doesn't contain information about data that has been parsed,
41 /// only the type of the token and its size.
49 fn new(kind: TokenKind, len: u32) -> Token {
54 /// Enum representing common lexeme types.
55 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
59 LineComment { doc_style: Option<DocStyle> },
61 /// `/* block comment */`
63 /// Block comments can be recursive, so a sequence like `/* /* */`
64 /// will not be considered terminated and will result in a parsing error.
65 BlockComment { doc_style: Option<DocStyle>, terminated: bool },
67 /// Any whitespace character sequence.
70 /// "ident" or "continue"
72 /// At this step, keywords are also considered identifiers.
75 /// Like the above, but containing invalid unicode codepoints.
81 /// An unknown prefix, like `foo#`, `foo'`, `foo"`.
83 /// Note that only the
84 /// prefix (`foo`) is included in the token, not the separator (which is
85 /// lexed as its own distinct token). In Rust 2021 and later, reserved
86 /// prefixes are reported as errors; in earlier editions, they result in a
87 /// (allowed by default) lint, and are treated as regular identifier
91 /// Examples: `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
92 /// suffix, but may be present here on string and float literals. Users of
93 /// this type will need to check for and reject that case.
95 /// See [LiteralKind] for more details.
96 Literal { kind: LiteralKind, suffix_start: u32 },
99 Lifetime { starts_with_number: bool },
157 /// Unknown token, not expected by the lexer, e.g. "№"
164 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
170 // Note that the suffix is *not* considered when deciding the `LiteralKind` in
171 // this type. This means that float literals like `1f32` are classified by this
172 // type as `Int`. (Compare against `rustc_ast::token::LitKind` and
173 // `rustc_ast::ast::LitKind.)
174 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
175 pub enum LiteralKind {
176 /// "12_u8", "0o100", "0b120i99", "1f32".
177 Int { base: Base, empty_int: bool },
178 /// "12.34f32", "1e3", but not "1f32`.
179 Float { base: Base, empty_exponent: bool },
180 /// "'a'", "'\\'", "'''", "';"
181 Char { terminated: bool },
182 /// "b'a'", "b'\\'", "b'''", "b';"
183 Byte { terminated: bool },
185 Str { terminated: bool },
186 /// "b"abc"", "b"abc"
187 ByteStr { terminated: bool },
188 /// "r"abc"", "r#"abc"#", "r####"ab"###"c"####", "r#"a". `None` indicates
189 /// an invalid literal.
190 RawStr { n_hashes: Option<u8> },
191 /// "br"abc"", "br#"abc"#", "br####"ab"###"c"####", "br#"a". `None`
192 /// indicates an invalid literal.
193 RawByteStr { n_hashes: Option<u8> },
196 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
197 pub enum RawStrError {
198 /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
199 InvalidStarter { bad_char: char },
200 /// The string was not terminated, e.g. `r###"abcde"##`.
201 /// `possible_terminator_offset` is the number of characters after `r` or
202 /// `br` where they may have intended to terminate it.
203 NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
204 /// More than 255 `#`s exist.
205 TooManyDelimiters { found: u32 },
208 /// Base of numeric literal encoding according to its prefix.
209 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
211 /// Literal starts with "0b".
213 /// Literal starts with "0o".
215 /// Literal doesn't contain a prefix.
217 /// Literal starts with "0x".
221 /// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
222 /// but shebang isn't a part of rust syntax.
223 pub fn strip_shebang(input: &str) -> Option<usize> {
224 // Shebang must start with `#!` literally, without any preceding whitespace.
225 // For simplicity we consider any line starting with `#!` a shebang,
226 // regardless of restrictions put on shebangs by specific platforms.
227 if let Some(input_tail) = input.strip_prefix("#!") {
228 // Ok, this is a shebang but if the next non-whitespace token is `[`,
229 // then it may be valid Rust code, so consider it Rust code.
230 let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| {
233 TokenKind::Whitespace
234 | TokenKind::LineComment { doc_style: None }
235 | TokenKind::BlockComment { doc_style: None, .. }
238 if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
239 // No other choice than to consider this a shebang.
240 return Some(2 + input_tail.lines().next().unwrap_or_default().len());
246 /// Validates a raw string literal. Used for getting more information about a
247 /// problem with a `RawStr`/`RawByteStr` with a `None` field.
249 pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
250 debug_assert!(!input.is_empty());
251 let mut cursor = Cursor::new(input);
252 // Move past the leading `r` or `br`.
253 for _ in 0..prefix_len {
254 cursor.bump().unwrap();
256 cursor.raw_double_quoted_string(prefix_len).map(|_| ())
259 /// Creates an iterator that produces tokens from the input string.
260 pub fn tokenize(input: &str) -> impl Iterator<Item = Token> + '_ {
261 let mut cursor = Cursor::new(input);
262 std::iter::from_fn(move || {
263 let token = cursor.advance_token();
264 if token.kind != TokenKind::Eof { Some(token) } else { None }
268 /// True if `c` is considered a whitespace according to Rust language definition.
269 /// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
270 /// for definitions of these classes.
271 pub fn is_whitespace(c: char) -> bool {
272 // This is Pattern_White_Space.
274 // Note that this set is stable (ie, it doesn't change with different
275 // Unicode versions), so it's ok to just hard-code the values.
279 // Usual ASCII suspects
282 | '\u{000B}' // vertical tab
283 | '\u{000C}' // form feed
285 | '\u{0020}' // space
287 // NEXT LINE from latin1
291 | '\u{200E}' // LEFT-TO-RIGHT MARK
292 | '\u{200F}' // RIGHT-TO-LEFT MARK
294 // Dedicated whitespace characters from Unicode
295 | '\u{2028}' // LINE SEPARATOR
296 | '\u{2029}' // PARAGRAPH SEPARATOR
300 /// True if `c` is valid as a first character of an identifier.
301 /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
302 /// a formal definition of valid identifier name.
303 pub fn is_id_start(c: char) -> bool {
304 // This is XID_Start OR '_' (which formally is not a XID_Start).
305 c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
308 /// True if `c` is valid as a non-first character of an identifier.
309 /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
310 /// a formal definition of valid identifier name.
311 pub fn is_id_continue(c: char) -> bool {
312 unicode_xid::UnicodeXID::is_xid_continue(c)
315 /// The passed string is lexically an identifier.
316 pub fn is_ident(string: &str) -> bool {
317 let mut chars = string.chars();
318 if let Some(start) = chars.next() {
319 is_id_start(start) && chars.all(is_id_continue)
326 /// Parses a token from the input string.
327 pub fn advance_token(&mut self) -> Token {
328 let first_char = match self.bump() {
330 None => return Token::new(TokenKind::Eof, 0),
332 let token_kind = match first_char {
333 // Slash, comment or block comment.
334 '/' => match self.first() {
335 '/' => self.line_comment(),
336 '*' => self.block_comment(),
340 // Whitespace sequence.
341 c if is_whitespace(c) => self.whitespace(),
343 // Raw identifier, raw string literal or identifier.
344 'r' => match (self.first(), self.second()) {
345 ('#', c1) if is_id_start(c1) => self.raw_ident(),
346 ('#', _) | ('"', _) => {
347 let res = self.raw_double_quoted_string(1);
348 let suffix_start = self.pos_within_token();
350 self.eat_literal_suffix();
352 let kind = RawStr { n_hashes: res.ok() };
353 Literal { kind, suffix_start }
355 _ => self.ident_or_unknown_prefix(),
358 // Byte literal, byte string literal, raw byte string literal or identifier.
359 'b' => match (self.first(), self.second()) {
362 let terminated = self.single_quoted_string();
363 let suffix_start = self.pos_within_token();
365 self.eat_literal_suffix();
367 let kind = Byte { terminated };
368 Literal { kind, suffix_start }
372 let terminated = self.double_quoted_string();
373 let suffix_start = self.pos_within_token();
375 self.eat_literal_suffix();
377 let kind = ByteStr { terminated };
378 Literal { kind, suffix_start }
380 ('r', '"') | ('r', '#') => {
382 let res = self.raw_double_quoted_string(2);
383 let suffix_start = self.pos_within_token();
385 self.eat_literal_suffix();
387 let kind = RawByteStr { n_hashes: res.ok() };
388 Literal { kind, suffix_start }
390 _ => self.ident_or_unknown_prefix(),
393 // Identifier (this should be checked after other variant that can
394 // start as identifier).
395 c if is_id_start(c) => self.ident_or_unknown_prefix(),
399 let literal_kind = self.number(c);
400 let suffix_start = self.pos_within_token();
401 self.eat_literal_suffix();
402 TokenKind::Literal { kind: literal_kind, suffix_start }
405 // One-symbol tokens.
433 // Lifetime or character literal.
434 '\'' => self.lifetime_or_char(),
438 let terminated = self.double_quoted_string();
439 let suffix_start = self.pos_within_token();
441 self.eat_literal_suffix();
443 let kind = Str { terminated };
444 Literal { kind, suffix_start }
446 // Identifier starting with an emoji. Only lexed for graceful error recovery.
447 c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => {
448 self.fake_ident_or_unknown_prefix()
452 let res = Token::new(token_kind, self.pos_within_token());
453 self.reset_pos_within_token();
457 fn line_comment(&mut self) -> TokenKind {
458 debug_assert!(self.prev() == '/' && self.first() == '/');
461 let doc_style = match self.first() {
462 // `//!` is an inner line doc comment.
463 '!' => Some(DocStyle::Inner),
464 // `////` (more than 3 slashes) is not considered a doc comment.
465 '/' if self.second() != '/' => Some(DocStyle::Outer),
469 self.eat_while(|c| c != '\n');
470 LineComment { doc_style }
473 fn block_comment(&mut self) -> TokenKind {
474 debug_assert!(self.prev() == '/' && self.first() == '*');
477 let doc_style = match self.first() {
478 // `/*!` is an inner block doc comment.
479 '!' => Some(DocStyle::Inner),
480 // `/***` (more than 2 stars) is not considered a doc comment.
481 // `/**/` is not considered a doc comment.
482 '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
486 let mut depth = 1usize;
487 while let Some(c) = self.bump() {
489 '/' if self.first() == '*' => {
493 '*' if self.first() == '/' => {
497 // This block comment is closed, so for a construction like "/* */ */"
498 // there will be a successfully parsed block comment "/* */"
499 // and " */" will be processed separately.
507 BlockComment { doc_style, terminated: depth == 0 }
510 fn whitespace(&mut self) -> TokenKind {
511 debug_assert!(is_whitespace(self.prev()));
512 self.eat_while(is_whitespace);
516 fn raw_ident(&mut self) -> TokenKind {
517 debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
520 // Eat the identifier part of RawIdent.
521 self.eat_identifier();
525 fn ident_or_unknown_prefix(&mut self) -> TokenKind {
526 debug_assert!(is_id_start(self.prev()));
527 // Start is already eaten, eat the rest of identifier.
528 self.eat_while(is_id_continue);
529 // Known prefixes must have been handled earlier. So if
530 // we see a prefix here, it is definitely an unknown prefix.
532 '#' | '"' | '\'' => UnknownPrefix,
533 c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => {
534 self.fake_ident_or_unknown_prefix()
540 fn fake_ident_or_unknown_prefix(&mut self) -> TokenKind {
541 // Start is already eaten, eat the rest of identifier.
543 unicode_xid::UnicodeXID::is_xid_continue(c)
544 || (!c.is_ascii() && unic_emoji_char::is_emoji(c))
547 // Known prefixes must have been handled earlier. So if
548 // we see a prefix here, it is definitely an unknown prefix.
550 '#' | '"' | '\'' => UnknownPrefix,
555 fn number(&mut self, first_digit: char) -> LiteralKind {
556 debug_assert!('0' <= self.prev() && self.prev() <= '9');
557 let mut base = Base::Decimal;
558 if first_digit == '0' {
559 // Attempt to parse encoding base.
560 let has_digits = match self.first() {
564 self.eat_decimal_digits()
569 self.eat_decimal_digits()
572 base = Base::Hexadecimal;
574 self.eat_hexadecimal_digits()
576 // Not a base prefix.
577 '0'..='9' | '_' | '.' | 'e' | 'E' => {
578 self.eat_decimal_digits();
582 _ => return Int { base, empty_int: false },
584 // Base prefix was provided, but there were no digits
585 // after it, e.g. "0x".
587 return Int { base, empty_int: true };
590 // No base prefix, parse number in the usual way.
591 self.eat_decimal_digits();
595 // Don't be greedy if this is actually an
596 // integer literal followed by field/method access or a range pattern
597 // (`0..2` and `12.foo()`)
598 '.' if self.second() != '.' && !is_id_start(self.second()) => {
599 // might have stuff after the ., and if it does, it needs to start
602 let mut empty_exponent = false;
603 if self.first().is_digit(10) {
604 self.eat_decimal_digits();
608 empty_exponent = !self.eat_float_exponent();
613 Float { base, empty_exponent }
617 let empty_exponent = !self.eat_float_exponent();
618 Float { base, empty_exponent }
620 _ => Int { base, empty_int: false },
624 fn lifetime_or_char(&mut self) -> TokenKind {
625 debug_assert!(self.prev() == '\'');
627 let can_be_a_lifetime = if self.second() == '\'' {
628 // It's surely not a lifetime.
631 // If the first symbol is valid for identifier, it can be a lifetime.
632 // Also check if it's a number for a better error reporting (so '0 will
633 // be reported as invalid lifetime and not as unterminated char literal).
634 is_id_start(self.first()) || self.first().is_digit(10)
637 if !can_be_a_lifetime {
638 let terminated = self.single_quoted_string();
639 let suffix_start = self.pos_within_token();
641 self.eat_literal_suffix();
643 let kind = Char { terminated };
644 return Literal { kind, suffix_start };
647 // Either a lifetime or a character literal with
648 // length greater than 1.
650 let starts_with_number = self.first().is_digit(10);
652 // Skip the literal contents.
653 // First symbol can be a number (which isn't a valid identifier start),
654 // so skip it without any checks.
656 self.eat_while(is_id_continue);
658 // Check if after skipping literal contents we've met a closing
659 // single quote (which means that user attempted to create a
660 // string with single quotes).
661 if self.first() == '\'' {
663 let kind = Char { terminated: true };
664 Literal { kind, suffix_start: self.pos_within_token() }
666 Lifetime { starts_with_number }
670 fn single_quoted_string(&mut self) -> bool {
671 debug_assert!(self.prev() == '\'');
672 // Check if it's a one-symbol literal.
673 if self.second() == '\'' && self.first() != '\\' {
679 // Literal has more than one symbol.
681 // Parse until either quotes are terminated or error is detected.
684 // Quotes are terminated, finish parsing.
689 // Probably beginning of the comment, which we don't want to include
690 // to the error report.
692 // Newline without following '\'' means unclosed quote, stop parsing.
693 '\n' if self.second() != '\'' => break,
694 // End of file, stop parsing.
695 EOF_CHAR if self.is_eof() => break,
696 // Escaped slash is considered one character, so bump twice.
701 // Skip the character.
707 // String was not terminated.
711 /// Eats double-quoted string and returns true
712 /// if string is terminated.
713 fn double_quoted_string(&mut self) -> bool {
714 debug_assert!(self.prev() == '"');
715 while let Some(c) = self.bump() {
720 '\\' if self.first() == '\\' || self.first() == '"' => {
721 // Bump again to skip escaped character.
727 // End of file reached.
731 /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
732 fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
733 // Wrap the actual function to handle the error with too many hashes.
734 // This way, it eats the whole raw string.
735 let n_hashes = self.raw_string_unvalidated(prefix_len)?;
736 // Only up to 255 `#`s are allowed in raw strings
737 match u8::try_from(n_hashes) {
739 Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
743 fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
744 debug_assert!(self.prev() == 'r');
745 let start_pos = self.pos_within_token();
746 let mut possible_terminator_offset = None;
747 let mut max_hashes = 0;
749 // Count opening '#' symbols.
751 while self.first() == '#' {
755 let n_start_hashes = eaten;
757 // Check that string is started.
761 let c = c.unwrap_or(EOF_CHAR);
762 return Err(RawStrError::InvalidStarter { bad_char: c });
766 // Skip the string contents and on each '#' character met, check if this is
767 // a raw string termination.
769 self.eat_while(|c| c != '"');
772 return Err(RawStrError::NoTerminator {
773 expected: n_start_hashes,
775 possible_terminator_offset,
779 // Eat closing double quote.
782 // Check that amount of closing '#' symbols
783 // is equal to the amount of opening ones.
784 // Note that this will not consume extra trailing `#` characters:
785 // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
786 // followed by a `#` token.
787 let mut n_end_hashes = 0;
788 while self.first() == '#' && n_end_hashes < n_start_hashes {
793 if n_end_hashes == n_start_hashes {
794 return Ok(n_start_hashes);
795 } else if n_end_hashes > max_hashes {
796 // Keep track of possible terminators to give a hint about
797 // where there might be a missing terminator
798 possible_terminator_offset =
799 Some(self.pos_within_token() - start_pos - n_end_hashes + prefix_len);
800 max_hashes = n_end_hashes;
805 fn eat_decimal_digits(&mut self) -> bool {
806 let mut has_digits = false;
822 fn eat_hexadecimal_digits(&mut self) -> bool {
823 let mut has_digits = false;
829 '0'..='9' | 'a'..='f' | 'A'..='F' => {
839 /// Eats the float exponent. Returns true if at least one digit was met,
840 /// and returns false otherwise.
841 fn eat_float_exponent(&mut self) -> bool {
842 debug_assert!(self.prev() == 'e' || self.prev() == 'E');
843 if self.first() == '-' || self.first() == '+' {
846 self.eat_decimal_digits()
849 // Eats the suffix of the literal, e.g. "u8".
850 fn eat_literal_suffix(&mut self) {
851 self.eat_identifier();
854 // Eats the identifier. Note: succeeds on `_`, which isn't a valid
856 fn eat_identifier(&mut self) {
857 if !is_id_start(self.first()) {
862 self.eat_while(is_id_continue);