4 pub use Nonterminal::*;
10 use crate::tokenstream::TokenTree;
12 use rustc_span::symbol::Symbol;
13 use rustc_span::{self, Span, DUMMY_SP};
15 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
16 use rustc_data_structures::sync::Lrc;
17 use rustc_macros::HashStable_Generic;
21 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
22 #[derive(HashStable_Generic)]
36 /// A delimiter token.
37 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
38 #[derive(HashStable_Generic)]
40 /// A round parenthesis (i.e., `(` or `)`).
42 /// A square bracket (i.e., `[` or `]`).
44 /// A curly brace (i.e., `{` or `}`).
46 /// An empty delimiter.
51 pub fn len(self) -> usize {
52 if self == NoDelim { 0 } else { 1 }
55 pub fn is_empty(self) -> bool {
60 #[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
62 Bool, // AST only, must never appear in a `Token`
68 StrRaw(u16), // raw string delimited by `n` hash symbols
70 ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols
75 #[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
79 pub suffix: Option<Symbol>,
82 impl fmt::Display for Lit {
83 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
84 let Lit { kind, symbol, suffix } = *self;
86 Byte => write!(f, "b'{}'", symbol)?,
87 Char => write!(f, "'{}'", symbol)?,
88 Str => write!(f, "\"{}\"", symbol)?,
91 "r{delim}\"{string}\"{delim}",
92 delim = "#".repeat(n as usize),
95 ByteStr => write!(f, "b\"{}\"", symbol)?,
96 ByteStrRaw(n) => write!(
98 "br{delim}\"{string}\"{delim}",
99 delim = "#".repeat(n as usize),
102 Integer | Float | Bool | Err => write!(f, "{}", symbol)?,
105 if let Some(suffix) = suffix {
106 write!(f, "{}", suffix)?;
114 /// An English article for the literal token kind.
115 pub fn article(self) -> &'static str {
117 Integer | Err => "an",
122 pub fn descr(self) -> &'static str {
124 Bool => panic!("literal token contains `Lit::Bool`"),
127 Integer => "integer",
129 Str | StrRaw(..) => "string",
130 ByteStr | ByteStrRaw(..) => "byte string",
135 crate fn may_have_suffix(self) -> bool {
137 Integer | Float | Err => true,
144 pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
145 Lit { kind, symbol, suffix }
149 pub fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool {
150 let ident_token = Token::new(Ident(name, is_raw), span);
151 token_can_begin_expr(&ident_token)
154 pub fn token_can_begin_expr(ident_token: &Token) -> bool {
155 !ident_token.is_reserved_ident()
156 || ident_token.is_path_segment_keyword()
157 || match ident_token.kind {
158 TokenKind::Ident(ident, _) => [
183 fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool {
184 let ident_token = Token::new(Ident(name, is_raw), span);
186 !ident_token.is_reserved_ident()
187 || ident_token.is_path_segment_keyword()
188 || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn]
192 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
194 /* Expression-operator symbols. */
209 /* Structural symbols */
225 /// Used by proc macros for representing lifetimes, not generated by lexer right now.
227 /// An opening delimiter (e.g., `{`).
228 OpenDelim(DelimToken),
229 /// A closing delimiter (e.g., `}`).
230 CloseDelim(DelimToken),
235 /* Name components */
236 Ident(ast::Name, /* is_raw */ bool),
239 Interpolated(Lrc<Nonterminal>),
241 // Can be expanded into several tokens.
243 DocComment(ast::Name),
245 // Junk. These carry no data because we don't really care about the data
246 // they *would* carry, and don't really want to allocate a new ident for
247 // them. Instead, users could extract that from the associated span.
253 /// A completely invalid token which should be skipped.
259 // `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger.
260 #[cfg(target_arch = "x86_64")]
261 rustc_data_structures::static_assert_size!(TokenKind, 16);
263 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
270 pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind {
271 Literal(Lit::new(kind, symbol, suffix))
274 /// Returns tokens that are likely to be typed accidentally instead of the current token.
275 /// Enables better error recovery when the wrong token is found.
276 pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> {
278 Comma => Some(vec![Dot, Lt, Semi]),
279 Semi => Some(vec![Colon, Comma]),
286 pub fn new(kind: TokenKind, span: Span) -> Self {
290 /// Some token that will be thrown away later.
291 pub fn dummy() -> Self {
292 Token::new(TokenKind::Whitespace, DUMMY_SP)
295 /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary.
296 pub fn from_ast_ident(ident: ast::Ident) -> Self {
297 Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
300 /// Return this token by value and leave a dummy token in its place.
301 pub fn take(&mut self) -> Self {
302 mem::replace(self, Token::dummy())
305 pub fn is_op(&self) -> bool {
307 OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
308 | Lifetime(..) | Interpolated(..) | Whitespace | Comment | Shebang(..) | Eof => false,
313 pub fn is_like_plus(&self) -> bool {
315 BinOp(Plus) | BinOpEq(Plus) => true,
320 /// Returns `true` if the token can appear at the start of an expression.
321 pub fn can_begin_expr(&self) -> bool {
323 Ident(name, is_raw) =>
324 ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
325 OpenDelim(..) | // tuple, array or block
326 Literal(..) | // literal
327 Not | // operator not
328 BinOp(Minus) | // unary minus
329 BinOp(Star) | // dereference
330 BinOp(Or) | OrOr | // closure
331 BinOp(And) | // reference
332 AndAnd | // double reference
333 // DotDotDot is no longer supported, but we need some way to display the error
334 DotDot | DotDotDot | DotDotEq | // range notation
335 Lt | BinOp(Shl) | // associated path
336 ModSep | // global path
337 Lifetime(..) | // labeled loop
338 Pound => true, // expression attributes
339 Interpolated(ref nt) => match **nt {
345 NtLifetime(..) => true,
352 /// Returns `true` if the token can appear at the start of a type.
353 pub fn can_begin_type(&self) -> bool {
355 Ident(name, is_raw) =>
356 ident_can_begin_type(name, self.span, is_raw), // type name or keyword
357 OpenDelim(Paren) | // tuple
358 OpenDelim(Bracket) | // array
360 BinOp(Star) | // raw pointer
361 BinOp(And) | // reference
362 AndAnd | // double reference
363 Question | // maybe bound in trait object
364 Lifetime(..) | // lifetime bound in trait object
365 Lt | BinOp(Shl) | // associated path
366 ModSep => true, // global path
367 Interpolated(ref nt) => match **nt {
368 NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true,
375 /// Returns `true` if the token can appear at the start of a const param.
376 pub fn can_begin_const_arg(&self) -> bool {
378 OpenDelim(Brace) => true,
379 Interpolated(ref nt) => match **nt {
380 NtExpr(..) | NtBlock(..) | NtLiteral(..) => true,
383 _ => self.can_begin_literal_or_bool(),
387 /// Returns `true` if the token can appear at the start of a generic bound.
388 pub fn can_begin_bound(&self) -> bool {
390 || self.is_lifetime()
391 || self.is_keyword(kw::For)
393 || self == &OpenDelim(Paren)
396 /// Returns `true` if the token is any literal
397 pub fn is_lit(&self) -> bool {
404 /// Returns `true` if the token is any literal, a minus (which can prefix a literal,
405 /// for example a '-42', or one of the boolean idents).
406 pub fn can_begin_literal_or_bool(&self) -> bool {
408 Literal(..) | BinOp(Minus) => true,
409 Ident(name, false) if name.is_bool_lit() => true,
410 Interpolated(ref nt) => match **nt {
411 NtLiteral(..) => true,
418 /// Returns an identifier if this token is an identifier.
419 pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
421 Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)),
422 Interpolated(ref nt) => match **nt {
423 NtIdent(ident, is_raw) => Some((ident, is_raw)),
430 /// Returns a lifetime identifier if this token is a lifetime.
431 pub fn lifetime(&self) -> Option<ast::Ident> {
433 Lifetime(name) => Some(ast::Ident::new(name, self.span)),
434 Interpolated(ref nt) => match **nt {
435 NtLifetime(ident) => Some(ident),
442 /// Returns `true` if the token is an identifier.
443 pub fn is_ident(&self) -> bool {
444 self.ident().is_some()
447 /// Returns `true` if the token is a lifetime.
448 pub fn is_lifetime(&self) -> bool {
449 self.lifetime().is_some()
452 /// Returns `true` if the token is a identifier whose name is the given
454 pub fn is_ident_named(&self, name: Symbol) -> bool {
455 self.ident().map_or(false, |(ident, _)| ident.name == name)
458 /// Returns `true` if the token is an interpolated path.
459 fn is_path(&self) -> bool {
460 if let Interpolated(ref nt) = self.kind {
461 if let NtPath(..) = **nt {
468 /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`?
469 /// That is, is this a pre-parsed expression dropped into the token stream
470 /// (which happens while parsing the result of macro expansion)?
471 pub fn is_whole_expr(&self) -> bool {
472 if let Interpolated(ref nt) = self.kind {
473 if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt {
481 /// Returns `true` if the token is either the `mut` or `const` keyword.
482 pub fn is_mutability(&self) -> bool {
483 self.is_keyword(kw::Mut) || self.is_keyword(kw::Const)
486 pub fn is_qpath_start(&self) -> bool {
487 self == &Lt || self == &BinOp(Shl)
490 pub fn is_path_start(&self) -> bool {
492 || self.is_qpath_start()
494 || self.is_path_segment_keyword()
495 || self.is_ident() && !self.is_reserved_ident()
498 /// Returns `true` if the token is a given keyword, `kw`.
499 pub fn is_keyword(&self, kw: Symbol) -> bool {
500 self.is_non_raw_ident_where(|id| id.name == kw)
503 pub fn is_path_segment_keyword(&self) -> bool {
504 self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword)
507 // Returns true for reserved identifiers used internally for elided lifetimes,
508 // unnamed method parameters, crate root module, error recovery etc.
509 pub fn is_special_ident(&self) -> bool {
510 self.is_non_raw_ident_where(ast::Ident::is_special)
513 /// Returns `true` if the token is a keyword used in the language.
514 pub fn is_used_keyword(&self) -> bool {
515 self.is_non_raw_ident_where(ast::Ident::is_used_keyword)
518 /// Returns `true` if the token is a keyword reserved for possible future use.
519 pub fn is_unused_keyword(&self) -> bool {
520 self.is_non_raw_ident_where(ast::Ident::is_unused_keyword)
523 /// Returns `true` if the token is either a special identifier or a keyword.
524 pub fn is_reserved_ident(&self) -> bool {
525 self.is_non_raw_ident_where(ast::Ident::is_reserved)
528 /// Returns `true` if the token is the identifier `true` or `false`.
529 pub fn is_bool_lit(&self) -> bool {
530 self.is_non_raw_ident_where(|id| id.name.is_bool_lit())
533 /// Returns `true` if the token is a non-raw identifier for which `pred` holds.
534 fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool {
536 Some((id, false)) => pred(id),
541 pub fn glue(&self, joint: &Token) -> Option<Token> {
542 let kind = match self.kind {
543 Eq => match joint.kind {
548 Lt => match joint.kind {
552 BinOp(Minus) => LArrow,
555 Gt => match joint.kind {
561 Not => match joint.kind {
565 BinOp(op) => match joint.kind {
567 BinOp(And) if op == And => AndAnd,
568 BinOp(Or) if op == Or => OrOr,
569 Gt if op == Minus => RArrow,
572 Dot => match joint.kind {
577 DotDot => match joint.kind {
582 Colon => match joint.kind {
586 SingleQuote => match joint.kind {
587 Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))),
591 Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
592 | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
593 | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
594 | Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment
595 | Shebang(..) | Unknown(..) | Eof => return None,
598 Some(Token::new(kind, self.span.to(joint.span)))
601 // See comments in `Nonterminal::to_tokenstream` for why we care about
602 // *probably* equal here rather than actual equality
603 crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool {
604 if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) {
607 match (&self.kind, &other.kind) {
622 | (&DotDotDot, &DotDotDot)
623 | (&DotDotEq, &DotDotEq)
630 | (&FatArrow, &FatArrow)
633 | (&Question, &Question)
634 | (&Whitespace, &Whitespace)
635 | (&Comment, &Comment)
636 | (&Eof, &Eof) => true,
638 (&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b,
640 (&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b,
642 (&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b,
644 (&Literal(a), &Literal(b)) => a == b,
646 (&Lifetime(a), &Lifetime(b)) => a == b,
647 (&Ident(a, b), &Ident(c, d)) => {
648 b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate)
651 (&Interpolated(_), &Interpolated(_)) => false,
653 _ => panic!("forgot to add a token?"),
658 impl PartialEq<TokenKind> for Token {
659 fn eq(&self, rhs: &TokenKind) -> bool {
664 #[derive(Clone, RustcEncodable, RustcDecodable)]
665 /// For interpolation during macro expansion.
666 pub enum Nonterminal {
667 NtItem(P<ast::Item>),
668 NtBlock(P<ast::Block>),
671 NtExpr(P<ast::Expr>),
673 NtIdent(ast::Ident, /* is_raw */ bool),
674 NtLifetime(ast::Ident),
675 NtLiteral(P<ast::Expr>),
676 /// Stuff inside brackets for attributes
677 NtMeta(ast::AttrItem),
679 NtVis(ast::Visibility),
681 // Used only for passing items to proc macro attributes (they are not
682 // strictly necessary for that, `Annotatable` can be converted into
683 // tokens directly, but doing that naively regresses pretty-printing).
684 NtTraitItem(ast::AssocItem),
685 NtImplItem(ast::AssocItem),
686 NtForeignItem(ast::ForeignItem),
689 impl PartialEq for Nonterminal {
690 fn eq(&self, rhs: &Self) -> bool {
692 (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
693 ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
695 (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
696 (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs,
697 // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
698 // correctly based on data from AST. This will prevent them from matching each other
699 // in macros. The comparison will become possible only when each nonterminal has an
700 // attached token stream from which it was parsed.
706 impl fmt::Debug for Nonterminal {
707 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
709 NtItem(..) => f.pad("NtItem(..)"),
710 NtBlock(..) => f.pad("NtBlock(..)"),
711 NtStmt(..) => f.pad("NtStmt(..)"),
712 NtPat(..) => f.pad("NtPat(..)"),
713 NtExpr(..) => f.pad("NtExpr(..)"),
714 NtTy(..) => f.pad("NtTy(..)"),
715 NtIdent(..) => f.pad("NtIdent(..)"),
716 NtLiteral(..) => f.pad("NtLiteral(..)"),
717 NtMeta(..) => f.pad("NtMeta(..)"),
718 NtPath(..) => f.pad("NtPath(..)"),
719 NtTT(..) => f.pad("NtTT(..)"),
720 NtImplItem(..) => f.pad("NtImplItem(..)"),
721 NtTraitItem(..) => f.pad("NtTraitItem(..)"),
722 NtForeignItem(..) => f.pad("NtForeignItem(..)"),
723 NtVis(..) => f.pad("NtVis(..)"),
724 NtLifetime(..) => f.pad("NtLifetime(..)"),
729 impl<CTX> HashStable<CTX> for Nonterminal
731 CTX: crate::HashStableContext,
733 fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
734 panic!("interpolated tokens should not be present in the HIR")