1 // Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 pub use self::BinOpToken::*;
12 pub use self::Nonterminal::*;
13 pub use self::DelimToken::*;
14 pub use self::IdentStyle::*;
16 pub use self::Token::*;
21 use util::interner::{RcStr, StrInterner};
24 use serialize::{Decodable, Decoder, Encodable, Encoder};
29 #[allow(non_camel_case_types)]
30 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
45 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
47 /// A round parenthesis: `(` or `)`
49 /// A square bracket: `[` or `]`
51 /// A curly brace: `{` or `}`
55 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
57 /// `::` follows the identifier with no whitespace in-between.
62 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
63 pub enum SpecialMacroVar {
64 /// `$crate` will be filled in with the name of the crate a macro was
65 /// imported from, if any.
69 impl SpecialMacroVar {
70 pub fn as_str(self) -> &'static str {
72 SpecialMacroVar::CrateMacroVar => "crate",
77 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
84 StrRaw(ast::Name, usize), /* raw str delimited by n hash symbols */
86 BinaryRaw(ast::Name, usize), /* raw binary str delimited by n hash symbols */
90 pub fn short_name(&self) -> &'static str {
94 Integer(_) => "integer",
96 Str_(_) | StrRaw(..) => "str",
97 Binary(_) | BinaryRaw(..) => "binary str"
102 #[allow(non_camel_case_types)]
103 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)]
105 /* Expression-operator symbols. */
120 /* Structural symbols */
135 /// An opening delimiter, eg. `{`
136 OpenDelim(DelimToken),
137 /// A closing delimiter, eg. `}`
138 CloseDelim(DelimToken),
141 Literal(Lit, Option<ast::Name>),
143 /* Name components */
144 Ident(ast::Ident, IdentStyle),
146 Lifetime(ast::Ident),
148 /* For interpolation */
149 Interpolated(Nonterminal),
150 // Can be expanded into several tokens.
152 DocComment(ast::Name),
153 // In left-hand-sides of MBE macros:
154 /// Parse a nonterminal (name to bind, name of NT, styles of their idents)
155 MatchNt(ast::Ident, ast::Ident, IdentStyle, IdentStyle),
156 // In right-hand-sides of MBE macros:
157 /// A syntactic variable that will be filled in by macro expansion.
158 SubstNt(ast::Ident, IdentStyle),
159 /// A macro variable with special meaning.
160 SpecialVarNt(SpecialMacroVar),
162 // Junk. These carry no data because we don't really care about the data
163 // they *would* carry, and don't really want to allocate a new ident for
164 // them. Instead, users could extract that from the associated span.
176 /// Returns `true` if the token starts with '>'.
177 pub fn is_like_gt(&self) -> bool {
179 BinOp(Shr) | BinOpEq(Shr) | Gt | Ge => true,
184 /// Returns `true` if the token can appear at the start of an expression.
185 pub fn can_begin_expr(&self) -> bool {
187 OpenDelim(_) => true,
191 Literal(_, _) => true,
193 BinOp(Minus) => true,
196 BinOp(Or) => true, // in lambda syntax
197 OrOr => true, // in lambda syntax
198 AndAnd => true, // double borrow
199 DotDot => true, // range notation
201 Interpolated(NtExpr(..)) => true,
202 Interpolated(NtIdent(..)) => true,
203 Interpolated(NtBlock(..)) => true,
204 Interpolated(NtPath(..)) => true,
209 /// Returns `true` if the token is any literal
210 pub fn is_lit(&self) -> bool {
212 Literal(_, _) => true,
217 /// Returns `true` if the token is an identifier.
218 pub fn is_ident(&self) -> bool {
225 /// Returns `true` if the token is an interpolated path.
226 pub fn is_path(&self) -> bool {
228 Interpolated(NtPath(..)) => true,
233 /// Returns `true` if the token is a path that is not followed by a `::`
235 #[allow(non_upper_case_globals)]
236 pub fn is_plain_ident(&self) -> bool {
238 Ident(_, Plain) => true,
243 /// Returns `true` if the token is a lifetime.
244 pub fn is_lifetime(&self) -> bool {
246 Lifetime(..) => true,
251 /// Returns `true` if the token is either the `mut` or `const` keyword.
252 pub fn is_mutability(&self) -> bool {
253 self.is_keyword(keywords::Mut) ||
254 self.is_keyword(keywords::Const)
257 /// Maps a token to its corresponding binary operator.
258 pub fn to_binop(&self) -> Option<ast::BinOp_> {
260 BinOp(Star) => Some(ast::BiMul),
261 BinOp(Slash) => Some(ast::BiDiv),
262 BinOp(Percent) => Some(ast::BiRem),
263 BinOp(Plus) => Some(ast::BiAdd),
264 BinOp(Minus) => Some(ast::BiSub),
265 BinOp(Shl) => Some(ast::BiShl),
266 BinOp(Shr) => Some(ast::BiShr),
267 BinOp(And) => Some(ast::BiBitAnd),
268 BinOp(Caret) => Some(ast::BiBitXor),
269 BinOp(Or) => Some(ast::BiBitOr),
270 Lt => Some(ast::BiLt),
271 Le => Some(ast::BiLe),
272 Ge => Some(ast::BiGe),
273 Gt => Some(ast::BiGt),
274 EqEq => Some(ast::BiEq),
275 Ne => Some(ast::BiNe),
276 AndAnd => Some(ast::BiAnd),
277 OrOr => Some(ast::BiOr),
282 /// Returns `true` if the token is a given keyword, `kw`.
283 #[allow(non_upper_case_globals)]
284 pub fn is_keyword(&self, kw: keywords::Keyword) -> bool {
286 Ident(sid, Plain) => kw.to_name() == sid.name,
291 pub fn is_keyword_allow_following_colon(&self, kw: keywords::Keyword) -> bool {
293 Ident(sid, _) => { kw.to_name() == sid.name }
298 /// Returns `true` if the token is either a special identifier, or a strict
299 /// or reserved keyword.
300 #[allow(non_upper_case_globals)]
301 pub fn is_any_keyword(&self) -> bool {
303 Ident(sid, Plain) => {
306 n == SELF_KEYWORD_NAME
307 || n == STATIC_KEYWORD_NAME
308 || n == SUPER_KEYWORD_NAME
309 || n == SELF_TYPE_KEYWORD_NAME
310 || STRICT_KEYWORD_START <= n
311 && n <= RESERVED_KEYWORD_FINAL
317 /// Returns `true` if the token may not appear as an identifier.
318 #[allow(non_upper_case_globals)]
319 pub fn is_strict_keyword(&self) -> bool {
321 Ident(sid, Plain) => {
324 n == SELF_KEYWORD_NAME
325 || n == STATIC_KEYWORD_NAME
326 || n == SUPER_KEYWORD_NAME
327 || n == SELF_TYPE_KEYWORD_NAME
328 || STRICT_KEYWORD_START <= n
329 && n <= STRICT_KEYWORD_FINAL
331 Ident(sid, ModName) => {
334 n != SELF_KEYWORD_NAME
335 && n != SUPER_KEYWORD_NAME
336 && STRICT_KEYWORD_START <= n
337 && n <= STRICT_KEYWORD_FINAL
343 /// Returns `true` if the token is a keyword that has been reserved for
344 /// possible future use.
345 #[allow(non_upper_case_globals)]
346 pub fn is_reserved_keyword(&self) -> bool {
348 Ident(sid, Plain) => {
351 RESERVED_KEYWORD_START <= n
352 && n <= RESERVED_KEYWORD_FINAL
358 /// Hygienic identifier equality comparison.
360 /// See `styntax::ext::mtwt`.
361 pub fn mtwt_eq(&self, other : &Token) -> bool {
362 match (self, other) {
363 (&Ident(id1,_), &Ident(id2,_)) | (&Lifetime(id1), &Lifetime(id2)) =>
364 mtwt::resolve(id1) == mtwt::resolve(id2),
370 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
371 /// For interpolation during macro expansion.
372 pub enum Nonterminal {
373 NtItem(P<ast::Item>),
374 NtBlock(P<ast::Block>),
375 NtStmt(P<ast::Stmt>),
377 NtExpr(P<ast::Expr>),
379 NtIdent(Box<ast::Ident>, IdentStyle),
380 /// Stuff inside brackets for attributes
381 NtMeta(P<ast::MetaItem>),
382 NtPath(Box<ast::Path>),
383 NtTT(P<ast::TokenTree>), // needs P'ed to break a circularity
384 // These is not exposed to macros, but is used by quasiquote.
386 NtImplItem(P<ast::ImplItem>),
387 NtTraitItem(P<ast::TraitItem>),
388 NtGenerics(ast::Generics),
389 NtWhereClause(ast::WhereClause),
392 impl fmt::Debug for Nonterminal {
393 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
395 NtItem(..) => f.pad("NtItem(..)"),
396 NtBlock(..) => f.pad("NtBlock(..)"),
397 NtStmt(..) => f.pad("NtStmt(..)"),
398 NtPat(..) => f.pad("NtPat(..)"),
399 NtExpr(..) => f.pad("NtExpr(..)"),
400 NtTy(..) => f.pad("NtTy(..)"),
401 NtIdent(..) => f.pad("NtIdent(..)"),
402 NtMeta(..) => f.pad("NtMeta(..)"),
403 NtPath(..) => f.pad("NtPath(..)"),
404 NtTT(..) => f.pad("NtTT(..)"),
405 NtArm(..) => f.pad("NtArm(..)"),
406 NtImplItem(..) => f.pad("NtImplItem(..)"),
407 NtTraitItem(..) => f.pad("NtTraitItem(..)"),
408 NtGenerics(..) => f.pad("NtGenerics(..)"),
409 NtWhereClause(..) => f.pad("NtWhereClause(..)"),
415 // Get the first "argument"
417 ( $first:expr, $( $remainder:expr, )* ) => ( $first )
420 // Get the last "argument" (has to be done recursively to avoid phoney local ambiguity error)
422 ( $first:expr, $( $remainder:expr, )+ ) => ( last!( $( $remainder, )+ ) );
423 ( $first:expr, ) => ( $first )
426 // In this macro, there is the requirement that the name (the number) must be monotonically
427 // increasing by one in the special identifiers, starting at 0; the same holds for the keywords,
428 // except starting from the next number instead of zero, and with the additional exception that
429 // special identifiers are *also* allowed (they are deduplicated in the important place, the
430 // interner), an exception which is demonstrated by "static" and "self".
431 macro_rules! declare_special_idents_and_keywords {(
432 // So now, in these rules, why is each definition parenthesised?
433 // Answer: otherwise we get a spurious local ambiguity bug on the "}"
434 pub mod special_idents {
435 $( ($si_name:expr, $si_static:ident, $si_str:expr); )*
440 $( ($sk_name:expr, $sk_variant:ident, $sk_str:expr); )*
442 $( ($rk_name:expr, $rk_variant:ident, $rk_str:expr); )*
445 const STRICT_KEYWORD_START: ast::Name = first!($( ast::Name($sk_name), )*);
446 const STRICT_KEYWORD_FINAL: ast::Name = last!($( ast::Name($sk_name), )*);
447 const RESERVED_KEYWORD_START: ast::Name = first!($( ast::Name($rk_name), )*);
448 const RESERVED_KEYWORD_FINAL: ast::Name = last!($( ast::Name($rk_name), )*);
450 pub mod special_idents {
453 #[allow(non_upper_case_globals)]
454 pub const $si_static: ast::Ident = ast::Ident {
455 name: ast::Name($si_name),
461 pub mod special_names {
464 #[allow(non_upper_case_globals)]
465 pub const $si_static: ast::Name = ast::Name($si_name);
469 /// All the valid words that have meaning in the Rust language.
471 /// Rust keywords are either 'strict' or 'reserved'. Strict keywords may not
472 /// appear as identifiers at all. Reserved keywords are not used anywhere in
473 /// the language and may not appear as identifiers.
475 pub use self::Keyword::*;
478 #[derive(Copy, Clone, PartialEq, Eq)]
485 pub fn to_name(&self) -> ast::Name {
487 $( $sk_variant => ast::Name($sk_name), )*
488 $( $rk_variant => ast::Name($rk_name), )*
494 fn mk_fresh_ident_interner() -> IdentInterner {
495 // The indices here must correspond to the numbers in
496 // special_idents, in Keyword to_name(), and in static
498 let mut init_vec = Vec::new();
499 $(init_vec.push($si_str);)*
500 $(init_vec.push($sk_str);)*
501 $(init_vec.push($rk_str);)*
502 interner::StrInterner::prefill(&init_vec[..])
506 // If the special idents get renumbered, remember to modify these two as appropriate
507 pub const SELF_KEYWORD_NAME: ast::Name = ast::Name(SELF_KEYWORD_NAME_NUM);
508 const STATIC_KEYWORD_NAME: ast::Name = ast::Name(STATIC_KEYWORD_NAME_NUM);
509 const SUPER_KEYWORD_NAME: ast::Name = ast::Name(SUPER_KEYWORD_NAME_NUM);
510 const SELF_TYPE_KEYWORD_NAME: ast::Name = ast::Name(SELF_TYPE_KEYWORD_NAME_NUM);
512 pub const SELF_KEYWORD_NAME_NUM: u32 = 1;
513 const STATIC_KEYWORD_NAME_NUM: u32 = 2;
514 const SUPER_KEYWORD_NAME_NUM: u32 = 3;
515 const SELF_TYPE_KEYWORD_NAME_NUM: u32 = 10;
517 // NB: leaving holes in the ident table is bad! a different ident will get
518 // interned with the id from the hole, but it will be between the min and max
519 // of the reserved words, and thus tagged as "reserved".
521 declare_special_idents_and_keywords! {
522 pub mod special_idents {
523 // These ones are statics
525 (super::SELF_KEYWORD_NAME_NUM, self_, "self");
526 (super::STATIC_KEYWORD_NAME_NUM, statik, "static");
527 (super::SUPER_KEYWORD_NAME_NUM, super_, "super");
528 (4, static_lifetime, "'static");
532 (6, matchers, "matchers");
534 // outside of libsyntax
535 (7, clownshoe_abi, "__rust_abi");
536 (8, opaque, "<opaque>");
537 (9, unnamed_field, "<unnamed_field>");
538 (super::SELF_TYPE_KEYWORD_NAME_NUM, type_self, "Self");
539 (11, prelude_import, "prelude_import");
543 // These ones are variants of the Keyword enum
547 (13, Break, "break");
548 (14, Crate, "crate");
551 (17, Extern, "extern");
552 (18, False, "false");
560 (26, Match, "match");
566 (32, Return, "return");
567 // Static and Self are also special idents (prefill de-dupes)
568 (super::STATIC_KEYWORD_NAME_NUM, Static, "static");
569 (super::SELF_KEYWORD_NAME_NUM, SelfValue, "self");
570 (super::SELF_TYPE_KEYWORD_NAME_NUM, SelfType, "Self");
571 (33, Struct, "struct");
572 (super::SUPER_KEYWORD_NAME_NUM, Super, "super");
574 (35, Trait, "trait");
576 (37, Unsafe, "unsafe");
578 (39, Virtual, "virtual");
579 (40, While, "while");
580 (41, Continue, "continue");
582 (43, Const, "const");
583 (44, Where, "where");
586 (46, Alignof, "alignof");
587 (47, Become, "become");
588 (48, Offsetof, "offsetof");
591 (51, Sizeof, "sizeof");
592 (52, Typeof, "typeof");
593 (53, Unsized, "unsized");
594 (54, Yield, "yield");
596 (56, Abstract, "abstract");
597 (57, Final, "final");
598 (58, Override, "override");
599 (59, Macro, "macro");
603 // looks like we can get rid of this completely...
604 pub type IdentInterner = StrInterner;
606 // if an interner exists in TLS, return it. Otherwise, prepare a
608 // FIXME(eddyb) #8726 This should probably use a thread-local reference.
609 pub fn get_ident_interner() -> Rc<IdentInterner> {
610 thread_local!(static KEY: Rc<::parse::token::IdentInterner> = {
611 Rc::new(mk_fresh_ident_interner())
613 KEY.with(|k| k.clone())
616 /// Reset the ident interner to its initial state.
617 pub fn reset_ident_interner() {
618 let interner = get_ident_interner();
619 interner.reset(mk_fresh_ident_interner());
622 /// Represents a string stored in the thread-local interner. Because the
623 /// interner lives for the life of the thread, this can be safely treated as an
624 /// immortal string, as long as it never crosses between threads.
626 /// FIXME(pcwalton): You must be careful about what you do in the destructors
627 /// of objects stored in TLS, because they may run after the interner is
628 /// destroyed. In particular, they must not access string contents. This can
629 /// be fixed in the future by just leaking all strings until thread death
631 #[derive(Clone, PartialEq, Hash, PartialOrd, Eq, Ord)]
632 pub struct InternedString {
636 impl InternedString {
638 pub fn new(string: &'static str) -> InternedString {
640 string: RcStr::new(string),
645 fn new_from_rc_str(string: RcStr) -> InternedString {
652 pub fn new_from_name(name: ast::Name) -> InternedString {
653 let interner = get_ident_interner();
654 InternedString::new_from_rc_str(interner.get(name))
658 impl Deref for InternedString {
661 fn deref(&self) -> &str { &*self.string }
664 impl fmt::Debug for InternedString {
665 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
666 fmt::Debug::fmt(&self.string, f)
670 impl fmt::Display for InternedString {
671 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
672 fmt::Display::fmt(&self.string, f)
676 impl<'a> PartialEq<&'a str> for InternedString {
678 fn eq(&self, other: & &'a str) -> bool {
679 PartialEq::eq(&self.string[..], *other)
682 fn ne(&self, other: & &'a str) -> bool {
683 PartialEq::ne(&self.string[..], *other)
687 impl<'a> PartialEq<InternedString> for &'a str {
689 fn eq(&self, other: &InternedString) -> bool {
690 PartialEq::eq(*self, &other.string[..])
693 fn ne(&self, other: &InternedString) -> bool {
694 PartialEq::ne(*self, &other.string[..])
698 impl Decodable for InternedString {
699 fn decode<D: Decoder>(d: &mut D) -> Result<InternedString, D::Error> {
700 Ok(intern(try!(d.read_str()).as_ref()).as_str())
704 impl Encodable for InternedString {
705 fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
706 s.emit_str(&self.string)
710 /// Interns and returns the string contents of an identifier, using the
711 /// thread-local interner.
713 pub fn intern_and_get_ident(s: &str) -> InternedString {
717 /// Maps a string to its interned representation.
719 pub fn intern(s: &str) -> ast::Name {
720 get_ident_interner().intern(s)
723 /// gensym's a new usize, using the current interner.
725 pub fn gensym(s: &str) -> ast::Name {
726 get_ident_interner().gensym(s)
729 /// Maps a string to an identifier with an empty syntax context.
731 pub fn str_to_ident(s: &str) -> ast::Ident {
732 ast::Ident::new(intern(s))
735 /// Maps a string to a gensym'ed identifier.
737 pub fn gensym_ident(s: &str) -> ast::Ident {
738 ast::Ident::new(gensym(s))
741 // create a fresh name that maps to the same string as the old one.
742 // note that this guarantees that str_ptr_eq(ident_to_string(src),interner_get(fresh_name(src)));
743 // that is, that the new name and the old one are connected to ptr_eq strings.
744 pub fn fresh_name(src: &ast::Ident) -> ast::Name {
745 let interner = get_ident_interner();
746 interner.gensym_copy(src.name)
747 // following: debug version. Could work in final except that it's incompatible with
748 // good error messages and uses of struct names in ambiguous could-be-binding
749 // locations. Also definitely destroys the guarantee given above about ptr_eq.
750 /*let num = rand::thread_rng().gen_uint_range(0,0xffff);
751 gensym(format!("{}_{}",ident_to_string(src),num))*/
754 // create a fresh mark.
755 pub fn fresh_mark() -> ast::Mrk {
756 gensym("mark").usize() as u32
765 fn mark_ident(id : ast::Ident, m : ast::Mrk) -> ast::Ident {
766 ast::Ident { name: id.name, ctxt:mtwt::apply_mark(m, id.ctxt) }
769 #[test] fn mtwt_token_eq_test() {
770 assert!(Gt.mtwt_eq(&Gt));
771 let a = str_to_ident("bac");
772 let a1 = mark_ident(a,92);
773 assert!(Ident(a, ModName).mtwt_eq(&Ident(a1, Plain)));