1 //! Low-level Rust lexer.
3 //! The idea with `rustc_lexer` is to make a reusable library,
4 //! by separating out pure lexing and rustc-specific concerns, like spans,
5 //! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
6 //! produces simple tokens which are a pair of type-tag and a bit of original text,
7 //! and does not report errors, instead storing them as flags on the token.
9 //! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
10 //! For that see [`rustc_parse::lexer`], which converts this basic token stream
11 //! into wide tokens used by actual parser.
13 //! The purpose of this crate is to convert raw sources into a labeled sequence
14 //! of well-known token types, so building an actual Rust token stream will
17 //! The main entity of this crate is the [`TokenKind`] enum which represents common
20 //! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
21 // We want to be able to build this crate with a stable compiler, so no
22 // `#![feature]` attributes should be added.
30 use self::LiteralKind::*;
31 use self::TokenKind::*;
32 use crate::cursor::{Cursor, EOF_CHAR};
33 use std::convert::TryFrom;
36 /// It doesn't contain information about data that has been parsed,
37 /// only the type of the token and its size.
45 fn new(kind: TokenKind, len: usize) -> Token {
50 /// Enum representing common lexeme types.
51 // perf note: Changing all `usize` to `u32` doesn't change performance. See #77629
52 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
56 LineComment { doc_style: Option<DocStyle> },
57 /// `/* block comment */`
59 /// Block comments can be recursive, so the sequence like `/* /* */`
60 /// will not be considered terminated and will result in a parsing error.
61 BlockComment { doc_style: Option<DocStyle>, terminated: bool },
62 /// Any whitespace characters sequence.
64 /// "ident" or "continue"
65 /// At this step keywords are also considered identifiers.
67 /// Like the above, but containing invalid unicode codepoints.
71 /// An unknown prefix like `foo#`, `foo'`, `foo"`. Note that only the
72 /// prefix (`foo`) is included in the token, not the separator (which is
73 /// lexed as its own distinct token). In Rust 2021 and later, reserved
74 /// prefixes are reported as errors; in earlier editions, they result in a
75 /// (allowed by default) lint, and are treated as regular identifier
78 /// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details.
79 Literal { kind: LiteralKind, suffix_start: usize },
81 Lifetime { starts_with_number: bool },
139 /// Unknown token, not expected by the lexer, e.g. "№"
143 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
149 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
150 pub enum LiteralKind {
151 /// "12_u8", "0o100", "0b120i99"
152 Int { base: Base, empty_int: bool },
153 /// "12.34f32", "0b100.100"
154 Float { base: Base, empty_exponent: bool },
155 /// "'a'", "'\\'", "'''", "';"
156 Char { terminated: bool },
157 /// "b'a'", "b'\\'", "b'''", "b';"
158 Byte { terminated: bool },
160 Str { terminated: bool },
161 /// "b"abc"", "b"abc"
162 ByteStr { terminated: bool },
163 /// "r"abc"", "r#"abc"#", "r####"ab"###"c"####", "r#"a"
164 RawStr { n_hashes: u8, err: Option<RawStrError> },
165 /// "br"abc"", "br#"abc"#", "br####"ab"###"c"####", "br#"a"
166 RawByteStr { n_hashes: u8, err: Option<RawStrError> },
169 /// Error produced validating a raw string. Represents cases like:
170 /// - `r##~"abcde"##`: `InvalidStarter`
171 /// - `r###"abcde"##`: `NoTerminator { expected: 3, found: 2, possible_terminator_offset: Some(11)`
172 /// - Too many `#`s (>255): `TooManyDelimiters`
173 // perf note: It doesn't matter that this makes `Token` 36 bytes bigger. See #77629
174 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
175 pub enum RawStrError {
176 /// Non `#` characters exist between `r` and `"` eg. `r#~"..`
177 InvalidStarter { bad_char: char },
178 /// The string was never terminated. `possible_terminator_offset` is the number of characters after `r` or `br` where they
179 /// may have intended to terminate it.
180 NoTerminator { expected: usize, found: usize, possible_terminator_offset: Option<usize> },
181 /// More than 255 `#`s exist.
182 TooManyDelimiters { found: usize },
185 /// Base of numeric literal encoding according to its prefix.
186 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
188 /// Literal starts with "0b".
190 /// Literal starts with "0o".
192 /// Literal starts with "0x".
194 /// Literal doesn't contain a prefix.
198 /// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
199 /// but shebang isn't a part of rust syntax.
200 pub fn strip_shebang(input: &str) -> Option<usize> {
201 // Shebang must start with `#!` literally, without any preceding whitespace.
202 // For simplicity we consider any line starting with `#!` a shebang,
203 // regardless of restrictions put on shebangs by specific platforms.
204 if let Some(input_tail) = input.strip_prefix("#!") {
205 // Ok, this is a shebang but if the next non-whitespace token is `[`,
206 // then it may be valid Rust code, so consider it Rust code.
207 let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| {
210 TokenKind::Whitespace
211 | TokenKind::LineComment { doc_style: None }
212 | TokenKind::BlockComment { doc_style: None, .. }
215 if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
216 // No other choice than to consider this a shebang.
217 return Some(2 + input_tail.lines().next().unwrap_or_default().len());
223 /// Parses the first token from the provided input string.
224 pub fn first_token(input: &str) -> Token {
225 debug_assert!(!input.is_empty());
226 Cursor::new(input).advance_token()
229 /// Creates an iterator that produces tokens from the input string.
230 pub fn tokenize(input: &str) -> impl Iterator<Item = Token> + '_ {
231 let mut cursor = Cursor::new(input);
232 std::iter::from_fn(move || {
236 cursor.reset_len_consumed();
237 Some(cursor.advance_token())
242 /// True if `c` is considered a whitespace according to Rust language definition.
243 /// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
244 /// for definitions of these classes.
245 pub fn is_whitespace(c: char) -> bool {
246 // This is Pattern_White_Space.
248 // Note that this set is stable (ie, it doesn't change with different
249 // Unicode versions), so it's ok to just hard-code the values.
253 // Usual ASCII suspects
256 | '\u{000B}' // vertical tab
257 | '\u{000C}' // form feed
259 | '\u{0020}' // space
261 // NEXT LINE from latin1
265 | '\u{200E}' // LEFT-TO-RIGHT MARK
266 | '\u{200F}' // RIGHT-TO-LEFT MARK
268 // Dedicated whitespace characters from Unicode
269 | '\u{2028}' // LINE SEPARATOR
270 | '\u{2029}' // PARAGRAPH SEPARATOR
274 /// True if `c` is valid as a first character of an identifier.
275 /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
276 /// a formal definition of valid identifier name.
277 pub fn is_id_start(c: char) -> bool {
278 // This is XID_Start OR '_' (which formally is not a XID_Start).
279 c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
282 /// True if `c` is valid as a non-first character of an identifier.
283 /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
284 /// a formal definition of valid identifier name.
285 pub fn is_id_continue(c: char) -> bool {
286 unicode_xid::UnicodeXID::is_xid_continue(c)
289 /// The passed string is lexically an identifier.
290 pub fn is_ident(string: &str) -> bool {
291 let mut chars = string.chars();
292 if let Some(start) = chars.next() {
293 is_id_start(start) && chars.all(is_id_continue)
300 /// Parses a token from the input string.
301 fn advance_token(&mut self) -> Token {
302 let first_char = self.bump().unwrap();
303 let token_kind = match first_char {
304 // Slash, comment or block comment.
305 '/' => match self.first() {
306 '/' => self.line_comment(),
307 '*' => self.block_comment(),
311 // Whitespace sequence.
312 c if is_whitespace(c) => self.whitespace(),
314 // Raw identifier, raw string literal or identifier.
315 'r' => match (self.first(), self.second()) {
316 ('#', c1) if is_id_start(c1) => self.raw_ident(),
317 ('#', _) | ('"', _) => {
318 let (n_hashes, err) = self.raw_double_quoted_string(1);
319 let suffix_start = self.len_consumed();
321 self.eat_literal_suffix();
323 let kind = RawStr { n_hashes, err };
324 Literal { kind, suffix_start }
326 _ => self.ident_or_unknown_prefix(),
329 // Byte literal, byte string literal, raw byte string literal or identifier.
330 'b' => match (self.first(), self.second()) {
333 let terminated = self.single_quoted_string();
334 let suffix_start = self.len_consumed();
336 self.eat_literal_suffix();
338 let kind = Byte { terminated };
339 Literal { kind, suffix_start }
343 let terminated = self.double_quoted_string();
344 let suffix_start = self.len_consumed();
346 self.eat_literal_suffix();
348 let kind = ByteStr { terminated };
349 Literal { kind, suffix_start }
351 ('r', '"') | ('r', '#') => {
353 let (n_hashes, err) = self.raw_double_quoted_string(2);
354 let suffix_start = self.len_consumed();
356 self.eat_literal_suffix();
358 let kind = RawByteStr { n_hashes, err };
359 Literal { kind, suffix_start }
361 _ => self.ident_or_unknown_prefix(),
364 // Identifier (this should be checked after other variant that can
365 // start as identifier).
366 c if is_id_start(c) => self.ident_or_unknown_prefix(),
370 let literal_kind = self.number(c);
371 let suffix_start = self.len_consumed();
372 self.eat_literal_suffix();
373 TokenKind::Literal { kind: literal_kind, suffix_start }
376 // One-symbol tokens.
404 // Lifetime or character literal.
405 '\'' => self.lifetime_or_char(),
409 let terminated = self.double_quoted_string();
410 let suffix_start = self.len_consumed();
412 self.eat_literal_suffix();
414 let kind = Str { terminated };
415 Literal { kind, suffix_start }
417 // Identifier starting with an emoji. Only lexed for graceful error recovery.
418 c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => {
419 self.fake_ident_or_unknown_prefix()
423 Token::new(token_kind, self.len_consumed())
426 fn line_comment(&mut self) -> TokenKind {
427 debug_assert!(self.prev() == '/' && self.first() == '/');
430 let doc_style = match self.first() {
431 // `//!` is an inner line doc comment.
432 '!' => Some(DocStyle::Inner),
433 // `////` (more than 3 slashes) is not considered a doc comment.
434 '/' if self.second() != '/' => Some(DocStyle::Outer),
438 self.eat_while(|c| c != '\n');
439 LineComment { doc_style }
442 fn block_comment(&mut self) -> TokenKind {
443 debug_assert!(self.prev() == '/' && self.first() == '*');
446 let doc_style = match self.first() {
447 // `/*!` is an inner block doc comment.
448 '!' => Some(DocStyle::Inner),
449 // `/***` (more than 2 stars) is not considered a doc comment.
450 // `/**/` is not considered a doc comment.
451 '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
455 let mut depth = 1usize;
456 while let Some(c) = self.bump() {
458 '/' if self.first() == '*' => {
462 '*' if self.first() == '/' => {
466 // This block comment is closed, so for a construction like "/* */ */"
467 // there will be a successfully parsed block comment "/* */"
468 // and " */" will be processed separately.
476 BlockComment { doc_style, terminated: depth == 0 }
479 fn whitespace(&mut self) -> TokenKind {
480 debug_assert!(is_whitespace(self.prev()));
481 self.eat_while(is_whitespace);
485 fn raw_ident(&mut self) -> TokenKind {
486 debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
489 // Eat the identifier part of RawIdent.
490 self.eat_identifier();
494 fn ident_or_unknown_prefix(&mut self) -> TokenKind {
495 debug_assert!(is_id_start(self.prev()));
496 // Start is already eaten, eat the rest of identifier.
497 self.eat_while(is_id_continue);
498 // Known prefixes must have been handled earlier. So if
499 // we see a prefix here, it is definitely an unknown prefix.
501 '#' | '"' | '\'' => UnknownPrefix,
502 c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => {
503 self.fake_ident_or_unknown_prefix()
509 fn fake_ident_or_unknown_prefix(&mut self) -> TokenKind {
510 // Start is already eaten, eat the rest of identifier.
512 unicode_xid::UnicodeXID::is_xid_continue(c)
513 || (!c.is_ascii() && unic_emoji_char::is_emoji(c))
516 // Known prefixes must have been handled earlier. So if
517 // we see a prefix here, it is definitely an unknown prefix.
519 '#' | '"' | '\'' => UnknownPrefix,
524 fn number(&mut self, first_digit: char) -> LiteralKind {
525 debug_assert!('0' <= self.prev() && self.prev() <= '9');
526 let mut base = Base::Decimal;
527 if first_digit == '0' {
528 // Attempt to parse encoding base.
529 let has_digits = match self.first() {
533 self.eat_decimal_digits()
538 self.eat_decimal_digits()
541 base = Base::Hexadecimal;
543 self.eat_hexadecimal_digits()
545 // Not a base prefix.
546 '0'..='9' | '_' | '.' | 'e' | 'E' => {
547 self.eat_decimal_digits();
551 _ => return Int { base, empty_int: false },
553 // Base prefix was provided, but there were no digits
554 // after it, e.g. "0x".
556 return Int { base, empty_int: true };
559 // No base prefix, parse number in the usual way.
560 self.eat_decimal_digits();
564 // Don't be greedy if this is actually an
565 // integer literal followed by field/method access or a range pattern
566 // (`0..2` and `12.foo()`)
567 '.' if self.second() != '.' && !is_id_start(self.second()) => {
568 // might have stuff after the ., and if it does, it needs to start
571 let mut empty_exponent = false;
572 if self.first().is_digit(10) {
573 self.eat_decimal_digits();
577 empty_exponent = !self.eat_float_exponent();
582 Float { base, empty_exponent }
586 let empty_exponent = !self.eat_float_exponent();
587 Float { base, empty_exponent }
589 _ => Int { base, empty_int: false },
593 fn lifetime_or_char(&mut self) -> TokenKind {
594 debug_assert!(self.prev() == '\'');
596 let can_be_a_lifetime = if self.second() == '\'' {
597 // It's surely not a lifetime.
600 // If the first symbol is valid for identifier, it can be a lifetime.
601 // Also check if it's a number for a better error reporting (so '0 will
602 // be reported as invalid lifetime and not as unterminated char literal).
603 is_id_start(self.first()) || self.first().is_digit(10)
606 if !can_be_a_lifetime {
607 let terminated = self.single_quoted_string();
608 let suffix_start = self.len_consumed();
610 self.eat_literal_suffix();
612 let kind = Char { terminated };
613 return Literal { kind, suffix_start };
616 // Either a lifetime or a character literal with
617 // length greater than 1.
619 let starts_with_number = self.first().is_digit(10);
621 // Skip the literal contents.
622 // First symbol can be a number (which isn't a valid identifier start),
623 // so skip it without any checks.
625 self.eat_while(is_id_continue);
627 // Check if after skipping literal contents we've met a closing
628 // single quote (which means that user attempted to create a
629 // string with single quotes).
630 if self.first() == '\'' {
632 let kind = Char { terminated: true };
633 Literal { kind, suffix_start: self.len_consumed() }
635 Lifetime { starts_with_number }
639 fn single_quoted_string(&mut self) -> bool {
640 debug_assert!(self.prev() == '\'');
641 // Check if it's a one-symbol literal.
642 if self.second() == '\'' && self.first() != '\\' {
648 // Literal has more than one symbol.
650 // Parse until either quotes are terminated or error is detected.
653 // Quotes are terminated, finish parsing.
658 // Probably beginning of the comment, which we don't want to include
659 // to the error report.
661 // Newline without following '\'' means unclosed quote, stop parsing.
662 '\n' if self.second() != '\'' => break,
663 // End of file, stop parsing.
664 EOF_CHAR if self.is_eof() => break,
665 // Escaped slash is considered one character, so bump twice.
670 // Skip the character.
676 // String was not terminated.
680 /// Eats double-quoted string and returns true
681 /// if string is terminated.
682 fn double_quoted_string(&mut self) -> bool {
683 debug_assert!(self.prev() == '"');
684 while let Some(c) = self.bump() {
689 '\\' if self.first() == '\\' || self.first() == '"' => {
690 // Bump again to skip escaped character.
696 // End of file reached.
700 /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
701 fn raw_double_quoted_string(&mut self, prefix_len: usize) -> (u8, Option<RawStrError>) {
702 // Wrap the actual function to handle the error with too many hashes.
703 // This way, it eats the whole raw string.
704 let (n_hashes, err) = self.raw_string_unvalidated(prefix_len);
705 // Only up to 255 `#`s are allowed in raw strings
706 match u8::try_from(n_hashes) {
707 Ok(num) => (num, err),
708 // We lie about the number of hashes here :P
709 Err(_) => (0, Some(RawStrError::TooManyDelimiters { found: n_hashes })),
713 fn raw_string_unvalidated(&mut self, prefix_len: usize) -> (usize, Option<RawStrError>) {
714 debug_assert!(self.prev() == 'r');
715 let start_pos = self.len_consumed();
716 let mut possible_terminator_offset = None;
717 let mut max_hashes = 0;
719 // Count opening '#' symbols.
721 while self.first() == '#' {
725 let n_start_hashes = eaten;
727 // Check that string is started.
731 let c = c.unwrap_or(EOF_CHAR);
732 return (n_start_hashes, Some(RawStrError::InvalidStarter { bad_char: c }));
736 // Skip the string contents and on each '#' character met, check if this is
737 // a raw string termination.
739 self.eat_while(|c| c != '"');
744 Some(RawStrError::NoTerminator {
745 expected: n_start_hashes,
747 possible_terminator_offset,
752 // Eat closing double quote.
755 // Check that amount of closing '#' symbols
756 // is equal to the amount of opening ones.
757 // Note that this will not consume extra trailing `#` characters:
758 // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
759 // followed by a `#` token.
760 let mut n_end_hashes = 0;
761 while self.first() == '#' && n_end_hashes < n_start_hashes {
766 if n_end_hashes == n_start_hashes {
767 return (n_start_hashes, None);
768 } else if n_end_hashes > max_hashes {
769 // Keep track of possible terminators to give a hint about
770 // where there might be a missing terminator
771 possible_terminator_offset =
772 Some(self.len_consumed() - start_pos - n_end_hashes + prefix_len);
773 max_hashes = n_end_hashes;
778 fn eat_decimal_digits(&mut self) -> bool {
779 let mut has_digits = false;
795 fn eat_hexadecimal_digits(&mut self) -> bool {
796 let mut has_digits = false;
802 '0'..='9' | 'a'..='f' | 'A'..='F' => {
812 /// Eats the float exponent. Returns true if at least one digit was met,
813 /// and returns false otherwise.
814 fn eat_float_exponent(&mut self) -> bool {
815 debug_assert!(self.prev() == 'e' || self.prev() == 'E');
816 if self.first() == '-' || self.first() == '+' {
819 self.eat_decimal_digits()
822 // Eats the suffix of the literal, e.g. "_u8".
823 fn eat_literal_suffix(&mut self) {
824 self.eat_identifier();
827 // Eats the identifier.
828 fn eat_identifier(&mut self) {
829 if !is_id_start(self.first()) {
834 self.eat_while(is_id_continue);