4 pub use Nonterminal::*;
9 use crate::tokenstream::TokenTree;
11 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
12 use rustc_data_structures::sync::Lrc;
13 use rustc_macros::HashStable_Generic;
14 use rustc_span::symbol::kw;
15 use rustc_span::symbol::Symbol;
16 use rustc_span::{self, Span, DUMMY_SP};
20 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
21 #[derive(HashStable_Generic)]
35 /// A delimiter token.
36 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
37 #[derive(HashStable_Generic)]
39 /// A round parenthesis (i.e., `(` or `)`).
41 /// A square bracket (i.e., `[` or `]`).
43 /// A curly brace (i.e., `{` or `}`).
45 /// An empty delimiter.
50 pub fn len(self) -> usize {
51 if self == NoDelim { 0 } else { 1 }
54 pub fn is_empty(self) -> bool {
59 #[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
61 Bool, // AST only, must never appear in a `Token`
67 StrRaw(u16), // raw string delimited by `n` hash symbols
69 ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols
74 #[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
78 pub suffix: Option<Symbol>,
81 impl fmt::Display for Lit {
82 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
83 let Lit { kind, symbol, suffix } = *self;
85 Byte => write!(f, "b'{}'", symbol)?,
86 Char => write!(f, "'{}'", symbol)?,
87 Str => write!(f, "\"{}\"", symbol)?,
90 "r{delim}\"{string}\"{delim}",
91 delim = "#".repeat(n as usize),
94 ByteStr => write!(f, "b\"{}\"", symbol)?,
95 ByteStrRaw(n) => write!(
97 "br{delim}\"{string}\"{delim}",
98 delim = "#".repeat(n as usize),
101 Integer | Float | Bool | Err => write!(f, "{}", symbol)?,
104 if let Some(suffix) = suffix {
105 write!(f, "{}", suffix)?;
113 /// An English article for the literal token kind.
114 pub fn article(self) -> &'static str {
116 Integer | Err => "an",
121 pub fn descr(self) -> &'static str {
123 Bool => panic!("literal token contains `Lit::Bool`"),
126 Integer => "integer",
128 Str | StrRaw(..) => "string",
129 ByteStr | ByteStrRaw(..) => "byte string",
134 crate fn may_have_suffix(self) -> bool {
136 Integer | Float | Err => true,
143 pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
144 Lit { kind, symbol, suffix }
148 pub fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool {
149 let ident_token = Token::new(Ident(name, is_raw), span);
151 !ident_token.is_reserved_ident()
152 || ident_token.is_path_segment_keyword()
176 fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool {
177 let ident_token = Token::new(Ident(name, is_raw), span);
179 !ident_token.is_reserved_ident()
180 || ident_token.is_path_segment_keyword()
181 || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn]
185 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
187 /* Expression-operator symbols. */
202 /* Structural symbols */
218 /// Used by proc macros for representing lifetimes, not generated by lexer right now.
220 /// An opening delimiter (e.g., `{`).
221 OpenDelim(DelimToken),
222 /// A closing delimiter (e.g., `}`).
223 CloseDelim(DelimToken),
228 /// Identifier token.
229 /// Do not forget about `NtIdent` when you want to match on identifiers.
230 /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
231 /// treat regular and interpolated identifiers in the same way.
232 Ident(ast::Name, /* is_raw */ bool),
233 /// Lifetime identifier token.
234 /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
235 /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
236 /// treat regular and interpolated lifetime identifiers in the same way.
239 Interpolated(Lrc<Nonterminal>),
241 // Can be expanded into several tokens.
243 DocComment(ast::Name),
245 // Junk. These carry no data because we don't really care about the data
246 // they *would* carry, and don't really want to allocate a new ident for
247 // them. Instead, users could extract that from the associated span.
253 /// A completely invalid token which should be skipped.
259 // `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger.
260 #[cfg(target_arch = "x86_64")]
261 rustc_data_structures::static_assert_size!(TokenKind, 16);
263 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
270 pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind {
271 Literal(Lit::new(kind, symbol, suffix))
274 // An approximation to proc-macro-style single-character operators used by rustc parser.
275 // If the operator token can be broken into two tokens, the first of which is single-character,
276 // then this function performs that operation, otherwise it returns `None`.
277 pub fn break_two_token_op(&self) -> Option<(TokenKind, TokenKind)> {
283 AndAnd => (BinOp(And), BinOp(And)),
284 OrOr => (BinOp(Or), BinOp(Or)),
285 BinOp(Shl) => (Lt, Lt),
286 BinOp(Shr) => (Gt, Gt),
287 BinOpEq(Plus) => (BinOp(Plus), Eq),
288 BinOpEq(Minus) => (BinOp(Minus), Eq),
289 BinOpEq(Star) => (BinOp(Star), Eq),
290 BinOpEq(Slash) => (BinOp(Slash), Eq),
291 BinOpEq(Percent) => (BinOp(Percent), Eq),
292 BinOpEq(Caret) => (BinOp(Caret), Eq),
293 BinOpEq(And) => (BinOp(And), Eq),
294 BinOpEq(Or) => (BinOp(Or), Eq),
295 BinOpEq(Shl) => (Lt, Le),
296 BinOpEq(Shr) => (Gt, Ge),
297 DotDot => (Dot, Dot),
298 DotDotDot => (Dot, DotDot),
299 ModSep => (Colon, Colon),
300 RArrow => (BinOp(Minus), Gt),
301 LArrow => (Lt, BinOp(Minus)),
302 FatArrow => (Eq, Gt),
307 /// Returns tokens that are likely to be typed accidentally instead of the current token.
308 /// Enables better error recovery when the wrong token is found.
309 pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> {
311 Comma => Some(vec![Dot, Lt, Semi]),
312 Semi => Some(vec![Colon, Comma]),
319 pub fn new(kind: TokenKind, span: Span) -> Self {
323 /// Some token that will be thrown away later.
324 pub fn dummy() -> Self {
325 Token::new(TokenKind::Whitespace, DUMMY_SP)
328 /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary.
329 pub fn from_ast_ident(ident: ast::Ident) -> Self {
330 Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
333 /// Return this token by value and leave a dummy token in its place.
334 pub fn take(&mut self) -> Self {
335 mem::replace(self, Token::dummy())
338 /// For interpolated tokens, returns a span of the fragment to which the interpolated
339 /// token refers. For all other tokens this is just a regular span.
340 /// It is particularly important to use this for identifiers and lifetimes
341 /// for which spans affect name resolution and edition checks.
342 /// Note that keywords are also identifiers, so they should use this
343 /// if they keep spans or perform edition checks.
344 pub fn uninterpolated_span(&self) -> Span {
346 Interpolated(nt) => nt.span(),
351 pub fn is_op(&self) -> bool {
353 OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
354 | Lifetime(..) | Interpolated(..) | Whitespace | Comment | Shebang(..) | Eof => false,
359 pub fn is_like_plus(&self) -> bool {
361 BinOp(Plus) | BinOpEq(Plus) => true,
366 /// Returns `true` if the token can appear at the start of an expression.
367 pub fn can_begin_expr(&self) -> bool {
368 match self.uninterpolate().kind {
369 Ident(name, is_raw) =>
370 ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
371 OpenDelim(..) | // tuple, array or block
372 Literal(..) | // literal
373 Not | // operator not
374 BinOp(Minus) | // unary minus
375 BinOp(Star) | // dereference
376 BinOp(Or) | OrOr | // closure
377 BinOp(And) | // reference
378 AndAnd | // double reference
379 // DotDotDot is no longer supported, but we need some way to display the error
380 DotDot | DotDotDot | DotDotEq | // range notation
381 Lt | BinOp(Shl) | // associated path
382 ModSep | // global path
383 Lifetime(..) | // labeled loop
384 Pound => true, // expression attributes
385 Interpolated(ref nt) => match **nt {
396 /// Returns `true` if the token can appear at the start of a type.
397 pub fn can_begin_type(&self) -> bool {
398 match self.uninterpolate().kind {
399 Ident(name, is_raw) =>
400 ident_can_begin_type(name, self.span, is_raw), // type name or keyword
401 OpenDelim(Paren) | // tuple
402 OpenDelim(Bracket) | // array
404 BinOp(Star) | // raw pointer
405 BinOp(And) | // reference
406 AndAnd | // double reference
407 Question | // maybe bound in trait object
408 Lifetime(..) | // lifetime bound in trait object
409 Lt | BinOp(Shl) | // associated path
410 ModSep => true, // global path
411 Interpolated(ref nt) => match **nt {
412 NtTy(..) | NtPath(..) => true,
419 /// Returns `true` if the token can appear at the start of a const param.
420 pub fn can_begin_const_arg(&self) -> bool {
422 OpenDelim(Brace) => true,
423 Interpolated(ref nt) => match **nt {
424 NtExpr(..) | NtBlock(..) | NtLiteral(..) => true,
427 _ => self.can_begin_literal_maybe_minus(),
431 /// Returns `true` if the token can appear at the start of a generic bound.
432 pub fn can_begin_bound(&self) -> bool {
434 || self.is_lifetime()
435 || self.is_keyword(kw::For)
437 || self == &OpenDelim(Paren)
440 /// Returns `true` if the token is any literal
441 pub fn is_lit(&self) -> bool {
448 /// Returns `true` if the token is any literal, a minus (which can prefix a literal,
449 /// for example a '-42', or one of the boolean idents).
451 /// In other words, would this token be a valid start of `parse_literal_maybe_minus`?
453 /// Keep this in sync with and `Lit::from_token`, excluding unary negation.
454 pub fn can_begin_literal_maybe_minus(&self) -> bool {
455 match self.uninterpolate().kind {
456 Literal(..) | BinOp(Minus) => true,
457 Ident(name, false) if name.is_bool_lit() => true,
458 Interpolated(ref nt) => match &**nt {
459 NtLiteral(_) => true,
460 NtExpr(e) => match &e.kind {
461 ast::ExprKind::Lit(_) => true,
462 ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
463 matches!(&e.kind, ast::ExprKind::Lit(_))
473 // A convenience function for matching on identifiers during parsing.
474 // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
475 // into the regular identifier or lifetime token it refers to,
476 // otherwise returns the original token.
477 pub fn uninterpolate(&self) -> Cow<'_, Token> {
479 Interpolated(nt) => match **nt {
480 NtIdent(ident, is_raw) => {
481 Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
483 NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
484 _ => Cow::Borrowed(self),
486 _ => Cow::Borrowed(self),
490 /// Returns an identifier if this token is an identifier.
491 pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
492 let token = self.uninterpolate();
494 Ident(name, is_raw) => Some((ast::Ident::new(name, token.span), is_raw)),
499 /// Returns a lifetime identifier if this token is a lifetime.
500 pub fn lifetime(&self) -> Option<ast::Ident> {
501 let token = self.uninterpolate();
503 Lifetime(name) => Some(ast::Ident::new(name, token.span)),
508 /// Returns `true` if the token is an identifier.
509 pub fn is_ident(&self) -> bool {
510 self.ident().is_some()
513 /// Returns `true` if the token is a lifetime.
514 pub fn is_lifetime(&self) -> bool {
515 self.lifetime().is_some()
518 /// Returns `true` if the token is a identifier whose name is the given
520 pub fn is_ident_named(&self, name: Symbol) -> bool {
521 self.ident().map_or(false, |(ident, _)| ident.name == name)
524 /// Returns `true` if the token is an interpolated path.
525 fn is_path(&self) -> bool {
526 if let Interpolated(ref nt) = self.kind {
527 if let NtPath(..) = **nt {
534 /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`?
535 /// That is, is this a pre-parsed expression dropped into the token stream
536 /// (which happens while parsing the result of macro expansion)?
537 pub fn is_whole_expr(&self) -> bool {
538 if let Interpolated(ref nt) = self.kind {
539 if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt {
547 // Is the token an interpolated block (`$b:block`)?
548 pub fn is_whole_block(&self) -> bool {
549 if let Interpolated(ref nt) = self.kind {
550 if let NtBlock(..) = **nt {
557 /// Returns `true` if the token is either the `mut` or `const` keyword.
558 pub fn is_mutability(&self) -> bool {
559 self.is_keyword(kw::Mut) || self.is_keyword(kw::Const)
562 pub fn is_qpath_start(&self) -> bool {
563 self == &Lt || self == &BinOp(Shl)
566 pub fn is_path_start(&self) -> bool {
568 || self.is_qpath_start()
570 || self.is_path_segment_keyword()
571 || self.is_ident() && !self.is_reserved_ident()
574 /// Returns `true` if the token is a given keyword, `kw`.
575 pub fn is_keyword(&self, kw: Symbol) -> bool {
576 self.is_non_raw_ident_where(|id| id.name == kw)
579 pub fn is_path_segment_keyword(&self) -> bool {
580 self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword)
583 // Returns true for reserved identifiers used internally for elided lifetimes,
584 // unnamed method parameters, crate root module, error recovery etc.
585 pub fn is_special_ident(&self) -> bool {
586 self.is_non_raw_ident_where(ast::Ident::is_special)
589 /// Returns `true` if the token is a keyword used in the language.
590 pub fn is_used_keyword(&self) -> bool {
591 self.is_non_raw_ident_where(ast::Ident::is_used_keyword)
594 /// Returns `true` if the token is a keyword reserved for possible future use.
595 pub fn is_unused_keyword(&self) -> bool {
596 self.is_non_raw_ident_where(ast::Ident::is_unused_keyword)
599 /// Returns `true` if the token is either a special identifier or a keyword.
600 pub fn is_reserved_ident(&self) -> bool {
601 self.is_non_raw_ident_where(ast::Ident::is_reserved)
604 /// Returns `true` if the token is the identifier `true` or `false`.
605 pub fn is_bool_lit(&self) -> bool {
606 self.is_non_raw_ident_where(|id| id.name.is_bool_lit())
609 /// Returns `true` if the token is a non-raw identifier for which `pred` holds.
610 pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool {
612 Some((id, false)) => pred(id),
617 pub fn glue(&self, joint: &Token) -> Option<Token> {
618 let kind = match self.kind {
619 Eq => match joint.kind {
624 Lt => match joint.kind {
628 BinOp(Minus) => LArrow,
631 Gt => match joint.kind {
637 Not => match joint.kind {
641 BinOp(op) => match joint.kind {
643 BinOp(And) if op == And => AndAnd,
644 BinOp(Or) if op == Or => OrOr,
645 Gt if op == Minus => RArrow,
648 Dot => match joint.kind {
653 DotDot => match joint.kind {
658 Colon => match joint.kind {
662 SingleQuote => match joint.kind {
663 Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))),
667 Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
668 | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
669 | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
670 | Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment
671 | Shebang(..) | Unknown(..) | Eof => return None,
674 Some(Token::new(kind, self.span.to(joint.span)))
677 // See comments in `Nonterminal::to_tokenstream` for why we care about
678 // *probably* equal here rather than actual equality
679 crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool {
680 if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) {
683 match (&self.kind, &other.kind) {
698 | (&DotDotDot, &DotDotDot)
699 | (&DotDotEq, &DotDotEq)
706 | (&FatArrow, &FatArrow)
709 | (&Question, &Question)
710 | (&Whitespace, &Whitespace)
711 | (&Comment, &Comment)
712 | (&Eof, &Eof) => true,
714 (&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b,
716 (&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b,
718 (&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b,
720 (&Literal(a), &Literal(b)) => a == b,
722 (&Lifetime(a), &Lifetime(b)) => a == b,
723 (&Ident(a, b), &Ident(c, d)) => {
724 b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate)
727 (&Interpolated(_), &Interpolated(_)) => false,
729 _ => panic!("forgot to add a token?"),
734 impl PartialEq<TokenKind> for Token {
735 fn eq(&self, rhs: &TokenKind) -> bool {
740 #[derive(Clone, RustcEncodable, RustcDecodable)]
741 /// For interpolation during macro expansion.
742 pub enum Nonterminal {
743 NtItem(P<ast::Item>),
744 NtBlock(P<ast::Block>),
747 NtExpr(P<ast::Expr>),
749 NtIdent(ast::Ident, /* is_raw */ bool),
750 NtLifetime(ast::Ident),
751 NtLiteral(P<ast::Expr>),
752 /// Stuff inside brackets for attributes
753 NtMeta(P<ast::AttrItem>),
755 NtVis(ast::Visibility),
759 // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
760 #[cfg(target_arch = "x86_64")]
761 rustc_data_structures::static_assert_size!(Nonterminal, 40);
764 fn span(&self) -> Span {
766 NtItem(item) => item.span,
767 NtBlock(block) => block.span,
768 NtStmt(stmt) => stmt.span,
769 NtPat(pat) => pat.span,
770 NtExpr(expr) | NtLiteral(expr) => expr.span,
772 NtIdent(ident, _) | NtLifetime(ident) => ident.span,
773 NtMeta(attr_item) => attr_item.span(),
774 NtPath(path) => path.span,
775 NtVis(vis) => vis.span,
776 NtTT(tt) => tt.span(),
781 impl PartialEq for Nonterminal {
782 fn eq(&self, rhs: &Self) -> bool {
784 (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
785 ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
787 (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
788 (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs,
789 // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
790 // correctly based on data from AST. This will prevent them from matching each other
791 // in macros. The comparison will become possible only when each nonterminal has an
792 // attached token stream from which it was parsed.
798 impl fmt::Debug for Nonterminal {
799 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
801 NtItem(..) => f.pad("NtItem(..)"),
802 NtBlock(..) => f.pad("NtBlock(..)"),
803 NtStmt(..) => f.pad("NtStmt(..)"),
804 NtPat(..) => f.pad("NtPat(..)"),
805 NtExpr(..) => f.pad("NtExpr(..)"),
806 NtTy(..) => f.pad("NtTy(..)"),
807 NtIdent(..) => f.pad("NtIdent(..)"),
808 NtLiteral(..) => f.pad("NtLiteral(..)"),
809 NtMeta(..) => f.pad("NtMeta(..)"),
810 NtPath(..) => f.pad("NtPath(..)"),
811 NtTT(..) => f.pad("NtTT(..)"),
812 NtVis(..) => f.pad("NtVis(..)"),
813 NtLifetime(..) => f.pad("NtLifetime(..)"),
818 impl<CTX> HashStable<CTX> for Nonterminal
820 CTX: crate::HashStableContext,
822 fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
823 panic!("interpolated tokens should not be present in the HIR")