3 pub use Nonterminal::*;
9 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
10 use rustc_data_structures::sync::Lrc;
11 use rustc_macros::HashStable_Generic;
12 use rustc_span::symbol::{kw, sym};
13 use rustc_span::symbol::{Ident, Symbol};
14 use rustc_span::{self, edition::Edition, Span, DUMMY_SP};
18 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
19 pub enum CommentKind {
24 #[derive(Clone, PartialEq, Encodable, Decodable, Hash, Debug, Copy)]
25 #[derive(HashStable_Generic)]
39 /// Describes how a sequence of token trees is delimited.
40 /// Cannot use `proc_macro::Delimiter` directly because this
41 /// structure should implement some additional traits.
42 /// The `None` variant is also renamed to `Invisible` to be
43 /// less confusing and better convey the semantics.
44 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
45 #[derive(Encodable, Decodable, Hash, HashStable_Generic)]
54 /// An invisible delimiter, that may, for example, appear around tokens coming from a
55 /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
56 /// `$var * 3` where `$var` is `1 + 2`.
57 /// Invisible delimiters might not survive roundtrip of a token stream through a string.
61 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
63 Bool, // AST only, must never appear in a `Token`
69 StrRaw(u8), // raw string delimited by `n` hash symbols
71 ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols
76 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
80 pub suffix: Option<Symbol>,
83 impl fmt::Display for Lit {
84 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
85 let Lit { kind, symbol, suffix } = *self;
87 Byte => write!(f, "b'{}'", symbol)?,
88 Char => write!(f, "'{}'", symbol)?,
89 Str => write!(f, "\"{}\"", symbol)?,
92 "r{delim}\"{string}\"{delim}",
93 delim = "#".repeat(n as usize),
96 ByteStr => write!(f, "b\"{}\"", symbol)?,
97 ByteStrRaw(n) => write!(
99 "br{delim}\"{string}\"{delim}",
100 delim = "#".repeat(n as usize),
103 Integer | Float | Bool | Err => write!(f, "{}", symbol)?,
106 if let Some(suffix) = suffix {
107 write!(f, "{}", suffix)?;
115 /// An English article for the literal token kind.
116 pub fn article(self) -> &'static str {
118 Integer | Err => "an",
123 pub fn descr(self) -> &'static str {
125 Bool => panic!("literal token contains `Lit::Bool`"),
128 Integer => "integer",
130 Str | StrRaw(..) => "string",
131 ByteStr | ByteStrRaw(..) => "byte string",
136 pub(crate) fn may_have_suffix(self) -> bool {
137 matches!(self, Integer | Float | Err)
142 pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
143 Lit { kind, symbol, suffix }
147 pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
148 let ident_token = Token::new(Ident(name, is_raw), span);
150 !ident_token.is_reserved_ident()
151 || ident_token.is_path_segment_keyword()
177 fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool {
178 let ident_token = Token::new(Ident(name, is_raw), span);
180 !ident_token.is_reserved_ident()
181 || ident_token.is_path_segment_keyword()
182 || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn]
186 #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
188 /* Expression-operator symbols. */
203 /* Structural symbols */
219 /// Used by proc macros for representing lifetimes, not generated by lexer right now.
221 /// An opening delimiter (e.g., `{`).
222 OpenDelim(Delimiter),
223 /// A closing delimiter (e.g., `}`).
224 CloseDelim(Delimiter),
229 /// Identifier token.
230 /// Do not forget about `NtIdent` when you want to match on identifiers.
231 /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
232 /// treat regular and interpolated identifiers in the same way.
233 Ident(Symbol, /* is_raw */ bool),
234 /// Lifetime identifier token.
235 /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
236 /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
237 /// treat regular and interpolated lifetime identifiers in the same way.
240 /// An embedded AST node, as produced by a macro. This only exists for
241 /// historical reasons. We'd like to get rid of it, for multiple reasons.
242 /// - It's conceptually very strange. Saying a token can contain an AST
243 /// node is like saying, in natural language, that a word can contain a
245 /// - It requires special handling in a bunch of places in the parser.
246 /// - It prevents `Token` from implementing `Copy`.
247 /// It adds complexity and likely slows things down. Please don't add new
248 /// occurrences of this token kind!
249 Interpolated(Lrc<Nonterminal>),
251 /// A doc comment token.
252 /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
253 /// similarly to symbols in string literal tokens.
254 DocComment(CommentKind, ast::AttrStyle, Symbol),
259 // `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger.
260 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
261 rustc_data_structures::static_assert_size!(TokenKind, 16);
263 #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
270 pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind {
271 Literal(Lit::new(kind, symbol, suffix))
274 // An approximation to proc-macro-style single-character operators used by rustc parser.
275 // If the operator token can be broken into two tokens, the first of which is single-character,
276 // then this function performs that operation, otherwise it returns `None`.
277 pub fn break_two_token_op(&self) -> Option<(TokenKind, TokenKind)> {
283 AndAnd => (BinOp(And), BinOp(And)),
284 OrOr => (BinOp(Or), BinOp(Or)),
285 BinOp(Shl) => (Lt, Lt),
286 BinOp(Shr) => (Gt, Gt),
287 BinOpEq(Plus) => (BinOp(Plus), Eq),
288 BinOpEq(Minus) => (BinOp(Minus), Eq),
289 BinOpEq(Star) => (BinOp(Star), Eq),
290 BinOpEq(Slash) => (BinOp(Slash), Eq),
291 BinOpEq(Percent) => (BinOp(Percent), Eq),
292 BinOpEq(Caret) => (BinOp(Caret), Eq),
293 BinOpEq(And) => (BinOp(And), Eq),
294 BinOpEq(Or) => (BinOp(Or), Eq),
295 BinOpEq(Shl) => (Lt, Le),
296 BinOpEq(Shr) => (Gt, Ge),
297 DotDot => (Dot, Dot),
298 DotDotDot => (Dot, DotDot),
299 ModSep => (Colon, Colon),
300 RArrow => (BinOp(Minus), Gt),
301 LArrow => (Lt, BinOp(Minus)),
302 FatArrow => (Eq, Gt),
307 /// Returns tokens that are likely to be typed accidentally instead of the current token.
308 /// Enables better error recovery when the wrong token is found.
309 pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> {
311 Comma => Some(vec![Dot, Lt, Semi]),
312 Semi => Some(vec![Colon, Comma]),
313 FatArrow => Some(vec![Eq, RArrow]),
318 pub fn should_end_const_arg(&self) -> bool {
319 matches!(self, Gt | Ge | BinOp(Shr) | BinOpEq(Shr))
324 pub fn new(kind: TokenKind, span: Span) -> Self {
328 /// Some token that will be thrown away later.
329 pub fn dummy() -> Self {
330 Token::new(TokenKind::Question, DUMMY_SP)
333 /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary.
334 pub fn from_ast_ident(ident: Ident) -> Self {
335 Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
338 /// Return this token by value and leave a dummy token in its place.
339 pub fn take(&mut self) -> Self {
340 mem::replace(self, Token::dummy())
343 /// For interpolated tokens, returns a span of the fragment to which the interpolated
344 /// token refers. For all other tokens this is just a regular span.
345 /// It is particularly important to use this for identifiers and lifetimes
346 /// for which spans affect name resolution and edition checks.
347 /// Note that keywords are also identifiers, so they should use this
348 /// if they keep spans or perform edition checks.
349 pub fn uninterpolated_span(&self) -> Span {
351 Interpolated(nt) => nt.span(),
356 pub fn is_op(&self) -> bool {
370 pub fn is_like_plus(&self) -> bool {
371 matches!(self.kind, BinOp(Plus) | BinOpEq(Plus))
374 /// Returns `true` if the token can appear at the start of an expression.
375 pub fn can_begin_expr(&self) -> bool {
376 match self.uninterpolate().kind {
377 Ident(name, is_raw) =>
378 ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
379 OpenDelim(..) | // tuple, array or block
380 Literal(..) | // literal
381 Not | // operator not
382 BinOp(Minus) | // unary minus
383 BinOp(Star) | // dereference
384 BinOp(Or) | OrOr | // closure
385 BinOp(And) | // reference
386 AndAnd | // double reference
387 // DotDotDot is no longer supported, but we need some way to display the error
388 DotDot | DotDotDot | DotDotEq | // range notation
389 Lt | BinOp(Shl) | // associated path
390 ModSep | // global path
391 Lifetime(..) | // labeled loop
392 Pound => true, // expression attributes
393 Interpolated(ref nt) => matches!(**nt, NtLiteral(..) |
401 /// Returns `true` if the token can appear at the start of a type.
402 pub fn can_begin_type(&self) -> bool {
403 match self.uninterpolate().kind {
404 Ident(name, is_raw) =>
405 ident_can_begin_type(name, self.span, is_raw), // type name or keyword
406 OpenDelim(Delimiter::Parenthesis) | // tuple
407 OpenDelim(Delimiter::Bracket) | // array
409 BinOp(Star) | // raw pointer
410 BinOp(And) | // reference
411 AndAnd | // double reference
412 Question | // maybe bound in trait object
413 Lifetime(..) | // lifetime bound in trait object
414 Lt | BinOp(Shl) | // associated path
415 ModSep => true, // global path
416 Interpolated(ref nt) => matches!(**nt, NtTy(..) | NtPath(..)),
421 /// Returns `true` if the token can appear at the start of a const param.
422 pub fn can_begin_const_arg(&self) -> bool {
424 OpenDelim(Delimiter::Brace) => true,
425 Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
426 _ => self.can_begin_literal_maybe_minus(),
430 /// Returns `true` if the token can appear at the start of a generic bound.
431 pub fn can_begin_bound(&self) -> bool {
433 || self.is_lifetime()
434 || self.is_keyword(kw::For)
436 || self == &OpenDelim(Delimiter::Parenthesis)
439 /// Returns `true` if the token is any literal.
440 pub fn is_lit(&self) -> bool {
441 matches!(self.kind, Literal(..))
444 /// Returns `true` if the token is any literal, a minus (which can prefix a literal,
445 /// for example a '-42', or one of the boolean idents).
447 /// In other words, would this token be a valid start of `parse_literal_maybe_minus`?
449 /// Keep this in sync with and `Lit::from_token`, excluding unary negation.
450 pub fn can_begin_literal_maybe_minus(&self) -> bool {
451 match self.uninterpolate().kind {
452 Literal(..) | BinOp(Minus) => true,
453 Ident(name, false) if name.is_bool_lit() => true,
454 Interpolated(ref nt) => match &**nt {
455 NtLiteral(_) => true,
456 NtExpr(e) => match &e.kind {
457 ast::ExprKind::Lit(_) => true,
458 ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
459 matches!(&e.kind, ast::ExprKind::Lit(_))
469 // A convenience function for matching on identifiers during parsing.
470 // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
471 // into the regular identifier or lifetime token it refers to,
472 // otherwise returns the original token.
473 pub fn uninterpolate(&self) -> Cow<'_, Token> {
475 Interpolated(nt) => match **nt {
476 NtIdent(ident, is_raw) => {
477 Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
479 NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
480 _ => Cow::Borrowed(self),
482 _ => Cow::Borrowed(self),
486 /// Returns an identifier if this token is an identifier.
488 pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> {
489 // We avoid using `Token::uninterpolate` here because it's slow.
491 &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
492 Interpolated(nt) => match **nt {
493 NtIdent(ident, is_raw) => Some((ident, is_raw)),
500 /// Returns a lifetime identifier if this token is a lifetime.
502 pub fn lifetime(&self) -> Option<Ident> {
503 // We avoid using `Token::uninterpolate` here because it's slow.
505 &Lifetime(name) => Some(Ident::new(name, self.span)),
506 Interpolated(nt) => match **nt {
507 NtLifetime(ident) => Some(ident),
514 /// Returns `true` if the token is an identifier.
515 pub fn is_ident(&self) -> bool {
516 self.ident().is_some()
519 /// Returns `true` if the token is a lifetime.
520 pub fn is_lifetime(&self) -> bool {
521 self.lifetime().is_some()
524 /// Returns `true` if the token is an identifier whose name is the given
526 pub fn is_ident_named(&self, name: Symbol) -> bool {
527 self.ident().map_or(false, |(ident, _)| ident.name == name)
530 /// Returns `true` if the token is an interpolated path.
531 fn is_path(&self) -> bool {
532 if let Interpolated(ref nt) = self.kind && let NtPath(..) = **nt {
538 /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`?
539 /// That is, is this a pre-parsed expression dropped into the token stream
540 /// (which happens while parsing the result of macro expansion)?
541 pub fn is_whole_expr(&self) -> bool {
542 if let Interpolated(ref nt) = self.kind
543 && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt
551 // Is the token an interpolated block (`$b:block`)?
552 pub fn is_whole_block(&self) -> bool {
553 if let Interpolated(ref nt) = self.kind && let NtBlock(..) = **nt {
559 /// Returns `true` if the token is either the `mut` or `const` keyword.
560 pub fn is_mutability(&self) -> bool {
561 self.is_keyword(kw::Mut) || self.is_keyword(kw::Const)
564 pub fn is_qpath_start(&self) -> bool {
565 self == &Lt || self == &BinOp(Shl)
568 pub fn is_path_start(&self) -> bool {
570 || self.is_qpath_start()
572 || self.is_path_segment_keyword()
573 || self.is_ident() && !self.is_reserved_ident()
576 /// Returns `true` if the token is a given keyword, `kw`.
577 pub fn is_keyword(&self, kw: Symbol) -> bool {
578 self.is_non_raw_ident_where(|id| id.name == kw)
581 pub fn is_path_segment_keyword(&self) -> bool {
582 self.is_non_raw_ident_where(Ident::is_path_segment_keyword)
585 // Returns true for reserved identifiers used internally for elided lifetimes,
586 // unnamed method parameters, crate root module, error recovery etc.
587 pub fn is_special_ident(&self) -> bool {
588 self.is_non_raw_ident_where(Ident::is_special)
591 /// Returns `true` if the token is a keyword used in the language.
592 pub fn is_used_keyword(&self) -> bool {
593 self.is_non_raw_ident_where(Ident::is_used_keyword)
596 /// Returns `true` if the token is a keyword reserved for possible future use.
597 pub fn is_unused_keyword(&self) -> bool {
598 self.is_non_raw_ident_where(Ident::is_unused_keyword)
601 /// Returns `true` if the token is either a special identifier or a keyword.
602 pub fn is_reserved_ident(&self) -> bool {
603 self.is_non_raw_ident_where(Ident::is_reserved)
606 /// Returns `true` if the token is the identifier `true` or `false`.
607 pub fn is_bool_lit(&self) -> bool {
608 self.is_non_raw_ident_where(|id| id.name.is_bool_lit())
611 pub fn is_numeric_lit(&self) -> bool {
614 Literal(Lit { kind: LitKind::Integer, .. }) | Literal(Lit { kind: LitKind::Float, .. })
618 /// Returns `true` if the token is a non-raw identifier for which `pred` holds.
619 pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
621 Some((id, false)) => pred(id),
626 pub fn glue(&self, joint: &Token) -> Option<Token> {
627 let kind = match self.kind {
628 Eq => match joint.kind {
633 Lt => match joint.kind {
637 BinOp(Minus) => LArrow,
640 Gt => match joint.kind {
646 Not => match joint.kind {
650 BinOp(op) => match joint.kind {
652 BinOp(And) if op == And => AndAnd,
653 BinOp(Or) if op == Or => OrOr,
654 Gt if op == Minus => RArrow,
657 Dot => match joint.kind {
662 DotDot => match joint.kind {
667 Colon => match joint.kind {
671 SingleQuote => match joint.kind {
672 Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))),
676 Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
677 | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
678 | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
679 | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
682 Some(Token::new(kind, self.span.to(joint.span)))
686 impl PartialEq<TokenKind> for Token {
687 fn eq(&self, rhs: &TokenKind) -> bool {
692 #[derive(Clone, Encodable, Decodable)]
693 /// For interpolation during macro expansion.
694 pub enum Nonterminal {
695 NtItem(P<ast::Item>),
696 NtBlock(P<ast::Block>),
697 NtStmt(P<ast::Stmt>),
699 NtExpr(P<ast::Expr>),
701 NtIdent(Ident, /* is_raw */ bool),
703 NtLiteral(P<ast::Expr>),
704 /// Stuff inside brackets for attributes
705 NtMeta(P<ast::AttrItem>),
706 NtPath(P<ast::Path>),
707 NtVis(P<ast::Visibility>),
710 // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
711 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
712 rustc_data_structures::static_assert_size!(Nonterminal, 16);
714 #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
715 pub enum NonterminalKind {
720 /// Keep track of whether the user used `:pat_param` or `:pat` and we inferred it from the
721 /// edition of the span. This is used for diagnostics.
736 impl NonterminalKind {
737 /// The `edition` closure is used to get the edition for the given symbol. Doing
738 /// `span.edition()` is expensive, so we do it lazily.
741 edition: impl FnOnce() -> Edition,
742 ) -> Option<NonterminalKind> {
744 sym::item => NonterminalKind::Item,
745 sym::block => NonterminalKind::Block,
746 sym::stmt => NonterminalKind::Stmt,
747 sym::pat => match edition() {
748 Edition::Edition2015 | Edition::Edition2018 => {
749 NonterminalKind::PatParam { inferred: true }
751 Edition::Edition2021 | Edition::Edition2024 => NonterminalKind::PatWithOr,
753 sym::pat_param => NonterminalKind::PatParam { inferred: false },
754 sym::expr => NonterminalKind::Expr,
755 sym::ty => NonterminalKind::Ty,
756 sym::ident => NonterminalKind::Ident,
757 sym::lifetime => NonterminalKind::Lifetime,
758 sym::literal => NonterminalKind::Literal,
759 sym::meta => NonterminalKind::Meta,
760 sym::path => NonterminalKind::Path,
761 sym::vis => NonterminalKind::Vis,
762 sym::tt => NonterminalKind::TT,
766 fn symbol(self) -> Symbol {
768 NonterminalKind::Item => sym::item,
769 NonterminalKind::Block => sym::block,
770 NonterminalKind::Stmt => sym::stmt,
771 NonterminalKind::PatParam { inferred: false } => sym::pat_param,
772 NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat,
773 NonterminalKind::Expr => sym::expr,
774 NonterminalKind::Ty => sym::ty,
775 NonterminalKind::Ident => sym::ident,
776 NonterminalKind::Lifetime => sym::lifetime,
777 NonterminalKind::Literal => sym::literal,
778 NonterminalKind::Meta => sym::meta,
779 NonterminalKind::Path => sym::path,
780 NonterminalKind::Vis => sym::vis,
781 NonterminalKind::TT => sym::tt,
786 impl fmt::Display for NonterminalKind {
787 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
788 write!(f, "{}", self.symbol())
793 pub fn span(&self) -> Span {
795 NtItem(item) => item.span,
796 NtBlock(block) => block.span,
797 NtStmt(stmt) => stmt.span,
798 NtPat(pat) => pat.span,
799 NtExpr(expr) | NtLiteral(expr) => expr.span,
801 NtIdent(ident, _) | NtLifetime(ident) => ident.span,
802 NtMeta(attr_item) => attr_item.span(),
803 NtPath(path) => path.span,
804 NtVis(vis) => vis.span,
809 impl PartialEq for Nonterminal {
810 fn eq(&self, rhs: &Self) -> bool {
812 (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
813 ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
815 (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
816 // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
817 // correctly based on data from AST. This will prevent them from matching each other
818 // in macros. The comparison will become possible only when each nonterminal has an
819 // attached token stream from which it was parsed.
825 impl fmt::Debug for Nonterminal {
826 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
828 NtItem(..) => f.pad("NtItem(..)"),
829 NtBlock(..) => f.pad("NtBlock(..)"),
830 NtStmt(..) => f.pad("NtStmt(..)"),
831 NtPat(..) => f.pad("NtPat(..)"),
832 NtExpr(..) => f.pad("NtExpr(..)"),
833 NtTy(..) => f.pad("NtTy(..)"),
834 NtIdent(..) => f.pad("NtIdent(..)"),
835 NtLiteral(..) => f.pad("NtLiteral(..)"),
836 NtMeta(..) => f.pad("NtMeta(..)"),
837 NtPath(..) => f.pad("NtPath(..)"),
838 NtVis(..) => f.pad("NtVis(..)"),
839 NtLifetime(..) => f.pad("NtLifetime(..)"),
844 impl<CTX> HashStable<CTX> for Nonterminal
846 CTX: crate::HashStableContext,
848 fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
849 panic!("interpolated tokens should not be present in the HIR")