1 //! Low-level Rust lexer.
3 //! The idea with `librustc_lexer` is to make a reusable library,
4 //! by separating out pure lexing and rustc-specific concerns, like spans,
5 //! error reporting an interning. So, rustc_lexer operates directly on `&str`,
6 //! produces simple tokens which are a pair of type-tag and a bit of original text,
7 //! and does not report errors, instead storing them as flags on the token.
9 //! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
10 //! For that see [`librustc_parse::lexer`], which converts this basic token stream
11 //! into wide tokens used by actual parser.
13 //! The purpose of this crate is to convert raw sources into a labeled sequence
14 //! of well-known token types, so building an actual Rust token stream will
17 //! The main entity of this crate is the [`TokenKind`] enum which represents common
20 //! [`librustc_parse::lexer`]: ../rustc_parse/lexer/index.html
21 // We want to be able to build this crate with a stable compiler, so no
22 // `#![feature]` attributes should be added.
30 use self::LiteralKind::*;
31 use self::TokenKind::*;
32 use crate::cursor::{Cursor, EOF_CHAR};
33 use std::convert::TryFrom;
36 /// It doesn't contain information about data that has been parsed,
37 /// only the type of the token and its size.
44 fn new(kind: TokenKind, len: usize) -> Token {
49 /// Enum representing common lexeme types.
50 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
54 LineComment { doc_style: Option<DocStyle> },
55 /// `/* block comment */`
57 /// Block comments can be recursive, so the sequence like `/* /* */`
58 /// will not be considered terminated and will result in a parsing error.
59 BlockComment { doc_style: Option<DocStyle>, terminated: bool },
60 /// Any whitespace characters sequence.
62 /// "ident" or "continue"
63 /// At this step keywords are also considered identifiers.
67 /// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details.
68 Literal { kind: LiteralKind, suffix_start: usize },
70 Lifetime { starts_with_number: bool },
128 /// Unknown token, not expected by the lexer, e.g. "№"
132 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
138 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
139 pub enum LiteralKind {
140 /// "12_u8", "0o100", "0b120i99"
141 Int { base: Base, empty_int: bool },
142 /// "12.34f32", "0b100.100"
143 Float { base: Base, empty_exponent: bool },
144 /// "'a'", "'\\'", "'''", "';"
145 Char { terminated: bool },
146 /// "b'a'", "b'\\'", "b'''", "b';"
147 Byte { terminated: bool },
149 Str { terminated: bool },
150 /// "b"abc"", "b"abc"
151 ByteStr { terminated: bool },
152 /// "r"abc"", "r#"abc"#", "r####"ab"###"c"####", "r#"a"
153 RawStr { n_hashes: u16, err: Option<RawStrError> },
154 /// "br"abc"", "br#"abc"#", "br####"ab"###"c"####", "br#"a"
155 RawByteStr { n_hashes: u16, err: Option<RawStrError> },
158 /// Error produced validating a raw string. Represents cases like:
159 /// - `r##~"abcde"##`: `InvalidStarter`
160 /// - `r###"abcde"##`: `NoTerminator { expected: 3, found: 2, possible_terminator_offset: Some(11)`
161 /// - Too many `#`s (>65535): `TooManyDelimiters`
162 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
163 pub enum RawStrError {
164 /// Non `#` characters exist between `r` and `"` eg. `r#~"..`
165 InvalidStarter { bad_char: char },
166 /// The string was never terminated. `possible_terminator_offset` is the number of characters after `r` or `br` where they
167 /// may have intended to terminate it.
168 NoTerminator { expected: usize, found: usize, possible_terminator_offset: Option<usize> },
169 /// More than 65535 `#`s exist.
170 TooManyDelimiters { found: usize },
173 /// Base of numeric literal encoding according to its prefix.
174 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
176 /// Literal starts with "0b".
178 /// Literal starts with "0o".
180 /// Literal starts with "0x".
182 /// Literal doesn't contain a prefix.
186 /// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
187 /// but shebang isn't a part of rust syntax.
188 pub fn strip_shebang(input: &str) -> Option<usize> {
189 // Shebang must start with `#!` literally, without any preceding whitespace.
190 // For simplicity we consider any line starting with `#!` a shebang,
191 // regardless of restrictions put on shebangs by specific platforms.
192 if let Some(input_tail) = input.strip_prefix("#!") {
193 // Ok, this is a shebang but if the next non-whitespace token is `[` or maybe
194 // a doc comment (due to `TokenKind::(Line,Block)Comment` ambiguity at lexer level),
195 // then it may be valid Rust code, so consider it Rust code.
196 let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok|
197 !matches!(tok, TokenKind::Whitespace | TokenKind::LineComment { .. } | TokenKind::BlockComment { .. })
199 if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
200 // No other choice than to consider this a shebang.
201 return Some(2 + input_tail.lines().next().unwrap_or_default().len());
207 /// Parses the first token from the provided input string.
208 pub fn first_token(input: &str) -> Token {
209 debug_assert!(!input.is_empty());
210 Cursor::new(input).advance_token()
213 /// Creates an iterator that produces tokens from the input string.
214 pub fn tokenize(mut input: &str) -> impl Iterator<Item = Token> + '_ {
215 std::iter::from_fn(move || {
216 if input.is_empty() {
219 let token = first_token(input);
220 input = &input[token.len..];
225 /// True if `c` is considered a whitespace according to Rust language definition.
226 /// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
227 /// for definitions of these classes.
228 pub fn is_whitespace(c: char) -> bool {
229 // This is Pattern_White_Space.
231 // Note that this set is stable (ie, it doesn't change with different
232 // Unicode versions), so it's ok to just hard-code the values.
235 // Usual ASCII suspects
238 | '\u{000B}' // vertical tab
239 | '\u{000C}' // form feed
241 | '\u{0020}' // space
243 // NEXT LINE from latin1
247 | '\u{200E}' // LEFT-TO-RIGHT MARK
248 | '\u{200F}' // RIGHT-TO-LEFT MARK
250 // Dedicated whitespace characters from Unicode
251 | '\u{2028}' // LINE SEPARATOR
252 | '\u{2029}' // PARAGRAPH SEPARATOR
258 /// True if `c` is valid as a first character of an identifier.
259 /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
260 /// a formal definition of valid identifier name.
261 pub fn is_id_start(c: char) -> bool {
262 // This is XID_Start OR '_' (which formally is not a XID_Start).
263 // We also add fast-path for ascii idents
264 ('a' <= c && c <= 'z')
265 || ('A' <= c && c <= 'Z')
267 || (c > '\x7f' && unicode_xid::UnicodeXID::is_xid_start(c))
270 /// True if `c` is valid as a non-first character of an identifier.
271 /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
272 /// a formal definition of valid identifier name.
273 pub fn is_id_continue(c: char) -> bool {
274 // This is exactly XID_Continue.
275 // We also add fast-path for ascii idents
276 ('a' <= c && c <= 'z')
277 || ('A' <= c && c <= 'Z')
278 || ('0' <= c && c <= '9')
280 || (c > '\x7f' && unicode_xid::UnicodeXID::is_xid_continue(c))
283 /// The passed string is lexically an identifier.
284 pub fn is_ident(string: &str) -> bool {
285 let mut chars = string.chars();
286 if let Some(start) = chars.next() {
287 is_id_start(start) && chars.all(is_id_continue)
294 /// Parses a token from the input string.
295 fn advance_token(&mut self) -> Token {
296 let first_char = self.bump().unwrap();
297 let token_kind = match first_char {
298 // Slash, comment or block comment.
299 '/' => match self.first() {
300 '/' => self.line_comment(),
301 '*' => self.block_comment(),
305 // Whitespace sequence.
306 c if is_whitespace(c) => self.whitespace(),
308 // Raw identifier, raw string literal or identifier.
309 'r' => match (self.first(), self.second()) {
310 ('#', c1) if is_id_start(c1) => self.raw_ident(),
311 ('#', _) | ('"', _) => {
312 let (n_hashes, err) = self.raw_double_quoted_string(1);
313 let suffix_start = self.len_consumed();
315 self.eat_literal_suffix();
317 let kind = RawStr { n_hashes, err };
318 Literal { kind, suffix_start }
323 // Byte literal, byte string literal, raw byte string literal or identifier.
324 'b' => match (self.first(), self.second()) {
327 let terminated = self.single_quoted_string();
328 let suffix_start = self.len_consumed();
330 self.eat_literal_suffix();
332 let kind = Byte { terminated };
333 Literal { kind, suffix_start }
337 let terminated = self.double_quoted_string();
338 let suffix_start = self.len_consumed();
340 self.eat_literal_suffix();
342 let kind = ByteStr { terminated };
343 Literal { kind, suffix_start }
345 ('r', '"') | ('r', '#') => {
347 let (n_hashes, err) = self.raw_double_quoted_string(2);
348 let suffix_start = self.len_consumed();
350 self.eat_literal_suffix();
352 let kind = RawByteStr { n_hashes, err };
353 Literal { kind, suffix_start }
358 // Identifier (this should be checked after other variant that can
359 // start as identifier).
360 c if is_id_start(c) => self.ident(),
364 let literal_kind = self.number(c);
365 let suffix_start = self.len_consumed();
366 self.eat_literal_suffix();
367 TokenKind::Literal { kind: literal_kind, suffix_start }
370 // One-symbol tokens.
398 // Lifetime or character literal.
399 '\'' => self.lifetime_or_char(),
403 let terminated = self.double_quoted_string();
404 let suffix_start = self.len_consumed();
406 self.eat_literal_suffix();
408 let kind = Str { terminated };
409 Literal { kind, suffix_start }
413 Token::new(token_kind, self.len_consumed())
416 fn line_comment(&mut self) -> TokenKind {
417 debug_assert!(self.prev() == '/' && self.first() == '/');
420 let doc_style = match self.first() {
421 // `//!` is an inner line doc comment.
422 '!' => Some(DocStyle::Inner),
423 // `////` (more than 3 slashes) is not considered a doc comment.
424 '/' if self.second() != '/' => Some(DocStyle::Outer),
428 self.eat_while(|c| c != '\n');
429 LineComment { doc_style }
432 fn block_comment(&mut self) -> TokenKind {
433 debug_assert!(self.prev() == '/' && self.first() == '*');
436 let doc_style = match self.first() {
437 // `/*!` is an inner block doc comment.
438 '!' => Some(DocStyle::Inner),
439 // `/***` (more than 2 stars) is not considered a doc comment.
440 // `/**/` is not considered a doc comment.
441 '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
445 let mut depth = 1usize;
446 while let Some(c) = self.bump() {
448 '/' if self.first() == '*' => {
452 '*' if self.first() == '/' => {
456 // This block comment is closed, so for a construction like "/* */ */"
457 // there will be a successfully parsed block comment "/* */"
458 // and " */" will be processed separately.
466 BlockComment { doc_style, terminated: depth == 0 }
469 fn whitespace(&mut self) -> TokenKind {
470 debug_assert!(is_whitespace(self.prev()));
471 self.eat_while(is_whitespace);
475 fn raw_ident(&mut self) -> TokenKind {
476 debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
479 // Eat the identifier part of RawIdent.
480 self.eat_identifier();
484 fn ident(&mut self) -> TokenKind {
485 debug_assert!(is_id_start(self.prev()));
486 // Start is already eaten, eat the rest of identifier.
487 self.eat_while(is_id_continue);
491 fn number(&mut self, first_digit: char) -> LiteralKind {
492 debug_assert!('0' <= self.prev() && self.prev() <= '9');
493 let mut base = Base::Decimal;
494 if first_digit == '0' {
495 // Attempt to parse encoding base.
496 let has_digits = match self.first() {
500 self.eat_decimal_digits()
505 self.eat_decimal_digits()
508 base = Base::Hexadecimal;
510 self.eat_hexadecimal_digits()
512 // Not a base prefix.
513 '0'..='9' | '_' | '.' | 'e' | 'E' => {
514 self.eat_decimal_digits();
518 _ => return Int { base, empty_int: false },
520 // Base prefix was provided, but there were no digits
521 // after it, e.g. "0x".
523 return Int { base, empty_int: true };
526 // No base prefix, parse number in the usual way.
527 self.eat_decimal_digits();
531 // Don't be greedy if this is actually an
532 // integer literal followed by field/method access or a range pattern
533 // (`0..2` and `12.foo()`)
534 '.' if self.second() != '.' && !is_id_start(self.second()) => {
535 // might have stuff after the ., and if it does, it needs to start
538 let mut empty_exponent = false;
539 if self.first().is_digit(10) {
540 self.eat_decimal_digits();
544 empty_exponent = !self.eat_float_exponent();
549 Float { base, empty_exponent }
553 let empty_exponent = !self.eat_float_exponent();
554 Float { base, empty_exponent }
556 _ => Int { base, empty_int: false },
560 fn lifetime_or_char(&mut self) -> TokenKind {
561 debug_assert!(self.prev() == '\'');
563 let can_be_a_lifetime = if self.second() == '\'' {
564 // It's surely not a lifetime.
567 // If the first symbol is valid for identifier, it can be a lifetime.
568 // Also check if it's a number for a better error reporting (so '0 will
569 // be reported as invalid lifetime and not as unterminated char literal).
570 is_id_start(self.first()) || self.first().is_digit(10)
573 if !can_be_a_lifetime {
574 let terminated = self.single_quoted_string();
575 let suffix_start = self.len_consumed();
577 self.eat_literal_suffix();
579 let kind = Char { terminated };
580 return Literal { kind, suffix_start };
583 // Either a lifetime or a character literal with
584 // length greater than 1.
586 let starts_with_number = self.first().is_digit(10);
588 // Skip the literal contents.
589 // First symbol can be a number (which isn't a valid identifier start),
590 // so skip it without any checks.
592 self.eat_while(is_id_continue);
594 // Check if after skipping literal contents we've met a closing
595 // single quote (which means that user attempted to create a
596 // string with single quotes).
597 if self.first() == '\'' {
599 let kind = Char { terminated: true };
600 Literal { kind, suffix_start: self.len_consumed() }
602 Lifetime { starts_with_number }
606 fn single_quoted_string(&mut self) -> bool {
607 debug_assert!(self.prev() == '\'');
608 // Check if it's a one-symbol literal.
609 if self.second() == '\'' && self.first() != '\\' {
615 // Literal has more than one symbol.
617 // Parse until either quotes are terminated or error is detected.
620 // Quotes are terminated, finish parsing.
625 // Probably beginning of the comment, which we don't want to include
626 // to the error report.
628 // Newline without following '\'' means unclosed quote, stop parsing.
629 '\n' if self.second() != '\'' => break,
630 // End of file, stop parsing.
631 EOF_CHAR if self.is_eof() => break,
632 // Escaped slash is considered one character, so bump twice.
637 // Skip the character.
643 // String was not terminated.
647 /// Eats double-quoted string and returns true
648 /// if string is terminated.
649 fn double_quoted_string(&mut self) -> bool {
650 debug_assert!(self.prev() == '"');
651 while let Some(c) = self.bump() {
656 '\\' if self.first() == '\\' || self.first() == '"' => {
657 // Bump again to skip escaped character.
663 // End of file reached.
667 /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
668 fn raw_double_quoted_string(&mut self, prefix_len: usize) -> (u16, Option<RawStrError>) {
669 // Wrap the actual function to handle the error with too many hashes.
670 // This way, it eats the whole raw string.
671 let (n_hashes, err) = self.raw_string_unvalidated(prefix_len);
672 // Only up to 65535 `#`s are allowed in raw strings
673 match u16::try_from(n_hashes) {
674 Ok(num) => (num, err),
675 // We lie about the number of hashes here :P
676 Err(_) => (0, Some(RawStrError::TooManyDelimiters { found: n_hashes })),
680 fn raw_string_unvalidated(&mut self, prefix_len: usize) -> (usize, Option<RawStrError>) {
681 debug_assert!(self.prev() == 'r');
682 let start_pos = self.len_consumed();
683 let mut possible_terminator_offset = None;
684 let mut max_hashes = 0;
686 // Count opening '#' symbols.
687 let n_start_hashes = self.eat_while(|c| c == '#');
689 // Check that string is started.
693 let c = c.unwrap_or(EOF_CHAR);
694 return (n_start_hashes, Some(RawStrError::InvalidStarter { bad_char: c }));
698 // Skip the string contents and on each '#' character met, check if this is
699 // a raw string termination.
701 self.eat_while(|c| c != '"');
706 Some(RawStrError::NoTerminator {
707 expected: n_start_hashes,
709 possible_terminator_offset,
714 // Eat closing double quote.
717 // Check that amount of closing '#' symbols
718 // is equal to the amount of opening ones.
719 // Note that this will not consume extra trailing `#` characters:
720 // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
721 // followed by a `#` token.
722 let mut hashes_left = n_start_hashes;
723 let is_closing_hash = |c| {
724 if c == '#' && hashes_left != 0 {
731 let n_end_hashes = self.eat_while(is_closing_hash);
733 if n_end_hashes == n_start_hashes {
734 return (n_start_hashes, None);
735 } else if n_end_hashes > max_hashes {
736 // Keep track of possible terminators to give a hint about
737 // where there might be a missing terminator
738 possible_terminator_offset =
739 Some(self.len_consumed() - start_pos - n_end_hashes + prefix_len);
740 max_hashes = n_end_hashes;
745 fn eat_decimal_digits(&mut self) -> bool {
746 let mut has_digits = false;
762 fn eat_hexadecimal_digits(&mut self) -> bool {
763 let mut has_digits = false;
769 '0'..='9' | 'a'..='f' | 'A'..='F' => {
779 /// Eats the float exponent. Returns true if at least one digit was met,
780 /// and returns false otherwise.
781 fn eat_float_exponent(&mut self) -> bool {
782 debug_assert!(self.prev() == 'e' || self.prev() == 'E');
783 if self.first() == '-' || self.first() == '+' {
786 self.eat_decimal_digits()
789 // Eats the suffix of the literal, e.g. "_u8".
790 fn eat_literal_suffix(&mut self) {
791 self.eat_identifier();
794 // Eats the identifier.
795 fn eat_identifier(&mut self) {
796 if !is_id_start(self.first()) {
801 self.eat_while(is_id_continue);
804 /// Eats symbols while predicate returns true or until the end of file is reached.
805 /// Returns amount of eaten symbols.
806 fn eat_while<F>(&mut self, mut predicate: F) -> usize
808 F: FnMut(char) -> bool,
810 let mut eaten: usize = 0;
811 while predicate(self.first()) && !self.is_eof() {