1 // Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
12 use ast::{P, Ident, Name, Mrk};
15 use util::interner::{RcStr, StrInterner};
18 use serialize::{Decodable, Decoder, Encodable, Encoder};
23 use std::path::BytesContainer;
26 #[allow(non_camel_case_types)]
27 #[deriving(Clone, Encodable, Decodable, Eq, Hash, Show)]
41 #[allow(non_camel_case_types)]
42 #[deriving(Clone, Encodable, Decodable, Eq, Hash, Show)]
44 /* Expression-operator symbols. */
59 /* Structural symbols */
83 LIT_INT(i64, ast::IntTy),
84 LIT_UINT(u64, ast::UintTy),
85 LIT_INT_UNSUFFIXED(i64),
86 LIT_FLOAT(ast::Ident, ast::FloatTy),
87 LIT_FLOAT_UNSUFFIXED(ast::Ident),
89 LIT_STR_RAW(ast::Ident, uint), /* raw str delimited by n hash symbols */
92 // an identifier contains an "is_mod_name" boolean,
93 // indicating whether :: follows this token with no
94 // whitespace in between.
95 IDENT(ast::Ident, bool),
99 /* For interpolation */
100 INTERPOLATED(Nonterminal),
102 DOC_COMMENT(ast::Ident),
106 #[deriving(Clone, Encodable, Decodable, Eq, Hash)]
107 /// For interpolation during macro expansion.
108 pub enum Nonterminal {
110 NtBlock(P<ast::Block>),
115 NtIdent(~ast::Ident, bool),
116 NtAttr(@ast::Attribute), // #[foo]
118 NtTT( @ast::TokenTree), // needs @ed to break a circularity
119 NtMatchers(Vec<ast::Matcher> )
122 impl fmt::Show for Nonterminal {
123 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
125 NtItem(..) => f.pad("NtItem(..)"),
126 NtBlock(..) => f.pad("NtBlock(..)"),
127 NtStmt(..) => f.pad("NtStmt(..)"),
128 NtPat(..) => f.pad("NtPat(..)"),
129 NtExpr(..) => f.pad("NtExpr(..)"),
130 NtTy(..) => f.pad("NtTy(..)"),
131 NtIdent(..) => f.pad("NtIdent(..)"),
132 NtAttr(..) => f.pad("NtAttr(..)"),
133 NtPath(..) => f.pad("NtPath(..)"),
134 NtTT(..) => f.pad("NtTT(..)"),
135 NtMatchers(..) => f.pad("NtMatchers(..)"),
140 pub fn binop_to_str(o: BinOp) -> ~str {
155 pub fn to_str(t: &Token) -> ~str {
168 BINOP(op) => binop_to_str(op),
169 BINOPEQ(op) => binop_to_str(op) + "=",
171 /* Structural symbols */
196 char::from_u32(c).unwrap().escape_default(|c| {
203 i.to_str() + ast_util::int_ty_to_str(t)
206 u.to_str() + ast_util::uint_ty_to_str(t)
208 LIT_INT_UNSUFFIXED(i) => { i.to_str() }
210 let mut body = get_ident(s).get().to_str();
211 if body.ends_with(".") {
212 body.push_char('0'); // `10.f` is not a float literal
214 body + ast_util::float_ty_to_str(t)
216 LIT_FLOAT_UNSUFFIXED(s) => {
217 let mut body = get_ident(s).get().to_str();
218 if body.ends_with(".") {
219 body.push_char('0'); // `10.f` is not a float literal
224 format!("\"{}\"", get_ident(s).get().escape_default())
226 LIT_STR_RAW(s, n) => {
227 format!("r{delim}\"{string}\"{delim}",
228 delim="#".repeat(n), string=get_ident(s))
231 /* Name components */
232 IDENT(s, _) => get_ident(s).get().to_str(),
234 format!("'{}", get_ident(s))
239 DOC_COMMENT(s) => get_ident(s).get().to_str(),
241 INTERPOLATED(ref nt) => {
243 &NtExpr(e) => ::print::pprust::expr_to_str(e),
244 &NtAttr(e) => ::print::pprust::attribute_to_str(e),
246 ~"an interpolated " +
248 NtItem(..) => ~"item",
249 NtBlock(..) => ~"block",
250 NtStmt(..) => ~"statement",
251 NtPat(..) => ~"pattern",
252 NtAttr(..) => fail!("should have been handled"),
253 NtExpr(..) => fail!("should have been handled above"),
255 NtIdent(..) => ~"identifier",
256 NtPath(..) => ~"path",
258 NtMatchers(..) => ~"matcher sequence"
266 pub fn can_begin_expr(t: &Token) -> bool {
275 LIT_INT(_, _) => true,
276 LIT_UINT(_, _) => true,
277 LIT_INT_UNSUFFIXED(_) => true,
278 LIT_FLOAT(_, _) => true,
279 LIT_FLOAT_UNSUFFIXED(_) => true,
281 LIT_STR_RAW(_, _) => true,
285 BINOP(MINUS) => true,
288 BINOP(OR) => true, // in lambda syntax
289 OROR => true, // in lambda syntax
291 INTERPOLATED(NtExpr(..))
292 | INTERPOLATED(NtIdent(..))
293 | INTERPOLATED(NtBlock(..))
294 | INTERPOLATED(NtPath(..)) => true,
299 /// what's the opposite delimiter?
300 pub fn flip_delimiter(t: &token::Token) -> token::Token {
304 LBRACKET => RBRACKET,
307 RBRACKET => LBRACKET,
314 pub fn is_lit(t: &Token) -> bool {
317 LIT_INT(_, _) => true,
318 LIT_UINT(_, _) => true,
319 LIT_INT_UNSUFFIXED(_) => true,
320 LIT_FLOAT(_, _) => true,
321 LIT_FLOAT_UNSUFFIXED(_) => true,
323 LIT_STR_RAW(_, _) => true,
328 pub fn is_ident(t: &Token) -> bool {
329 match *t { IDENT(_, _) => true, _ => false }
332 pub fn is_ident_or_path(t: &Token) -> bool {
334 IDENT(_, _) | INTERPOLATED(NtPath(..)) => true,
339 pub fn is_plain_ident(t: &Token) -> bool {
340 match *t { IDENT(_, false) => true, _ => false }
343 pub fn is_bar(t: &Token) -> bool {
344 match *t { BINOP(OR) | OROR => true, _ => false }
347 // Get the first "argument"
349 ( $first:expr, $( $remainder:expr, )* ) => ( $first )
352 // Get the last "argument" (has to be done recursively to avoid phoney local ambiguity error)
354 ( $first:expr, $( $remainder:expr, )+ ) => ( last!( $( $remainder, )+ ) );
355 ( $first:expr, ) => ( $first )
358 // In this macro, there is the requirement that the name (the number) must be monotonically
359 // increasing by one in the special identifiers, starting at 0; the same holds for the keywords,
360 // except starting from the next number instead of zero, and with the additional exception that
361 // special identifiers are *also* allowed (they are deduplicated in the important place, the
362 // interner), an exception which is demonstrated by "static" and "self".
363 macro_rules! declare_special_idents_and_keywords {(
364 // So now, in these rules, why is each definition parenthesised?
365 // Answer: otherwise we get a spurious local ambiguity bug on the "}"
366 pub mod special_idents {
367 $( ($si_name:expr, $si_static:ident, $si_str:expr); )*
372 $( ($sk_name:expr, $sk_variant:ident, $sk_str:expr); )*
374 $( ($rk_name:expr, $rk_variant:ident, $rk_str:expr); )*
377 static STRICT_KEYWORD_START: Name = first!($( $sk_name, )*);
378 static STRICT_KEYWORD_FINAL: Name = last!($( $sk_name, )*);
379 static RESERVED_KEYWORD_START: Name = first!($( $rk_name, )*);
380 static RESERVED_KEYWORD_FINAL: Name = last!($( $rk_name, )*);
382 pub mod special_idents {
384 $( pub static $si_static: Ident = Ident { name: $si_name, ctxt: 0 }; )*
388 * All the valid words that have meaning in the Rust language.
390 * Rust keywords are either 'strict' or 'reserved'. Strict keywords may not
391 * appear as identifiers at all. Reserved keywords are not used anywhere in
392 * the language and may not appear as identifiers.
403 pub fn to_ident(&self) -> Ident {
405 $( $sk_variant => Ident { name: $sk_name, ctxt: 0 }, )*
406 $( $rk_variant => Ident { name: $rk_name, ctxt: 0 }, )*
412 fn mk_fresh_ident_interner() -> IdentInterner {
413 // The indices here must correspond to the numbers in
414 // special_idents, in Keyword to_ident(), and in static
416 let mut init_vec = Vec::new();
417 $(init_vec.push($si_str);)*
418 $(init_vec.push($sk_str);)*
419 $(init_vec.push($rk_str);)*
420 interner::StrInterner::prefill(init_vec.as_slice())
424 // If the special idents get renumbered, remember to modify these two as appropriate
425 static SELF_KEYWORD_NAME: Name = 1;
426 static STATIC_KEYWORD_NAME: Name = 2;
428 declare_special_idents_and_keywords! {
429 pub mod special_idents {
430 // These ones are statics
432 (super::SELF_KEYWORD_NAME, self_, "self");
433 (super::STATIC_KEYWORD_NAME, statik, "static");
437 (4, matchers, "matchers");
439 // outside of libsyntax
440 (5, clownshoe_abi, "__rust_abi");
441 (6, opaque, "<opaque>");
442 (7, unnamed_field, "<unnamed_field>");
443 (8, type_self, "Self");
447 // These ones are variants of the Keyword enum
451 (10, Break, "break");
452 (11, Const, "const");
453 (12, Crate, "crate");
456 (15, Extern, "extern");
457 (16, False, "false");
464 (23, __LogLevel, "__log_level");
466 (25, Match, "match");
473 (32, Return, "return");
474 // Static and Self are also special idents (prefill de-dupes)
475 (super::STATIC_KEYWORD_NAME, Static, "static");
476 (super::SELF_KEYWORD_NAME, Self, "self");
477 (33, Struct, "struct");
478 (34, Super, "super");
480 (36, Trait, "trait");
482 (38, Unsafe, "unsafe");
484 (40, While, "while");
485 (41, Continue, "continue");
490 (44, Alignof, "alignof");
492 (46, Offsetof, "offsetof");
494 (48, Sizeof, "sizeof");
495 (49, Typeof, "typeof");
496 (50, Unsized, "unsized");
497 (51, Yield, "yield");
503 * Maps a token to a record specifying the corresponding binary
506 pub fn token_to_binop(tok: &Token) -> Option<ast::BinOp> {
508 BINOP(STAR) => Some(ast::BiMul),
509 BINOP(SLASH) => Some(ast::BiDiv),
510 BINOP(PERCENT) => Some(ast::BiRem),
511 BINOP(PLUS) => Some(ast::BiAdd),
512 BINOP(MINUS) => Some(ast::BiSub),
513 BINOP(SHL) => Some(ast::BiShl),
514 BINOP(SHR) => Some(ast::BiShr),
515 BINOP(AND) => Some(ast::BiBitAnd),
516 BINOP(CARET) => Some(ast::BiBitXor),
517 BINOP(OR) => Some(ast::BiBitOr),
518 LT => Some(ast::BiLt),
519 LE => Some(ast::BiLe),
520 GE => Some(ast::BiGe),
521 GT => Some(ast::BiGt),
522 EQEQ => Some(ast::BiEq),
523 NE => Some(ast::BiNe),
524 ANDAND => Some(ast::BiAnd),
525 OROR => Some(ast::BiOr),
530 // looks like we can get rid of this completely...
531 pub type IdentInterner = StrInterner;
533 // if an interner exists in TLS, return it. Otherwise, prepare a
535 pub fn get_ident_interner() -> @IdentInterner {
536 local_data_key!(key: @::parse::token::IdentInterner)
537 match local_data::get(key, |k| k.map(|k| *k)) {
538 Some(interner) => interner,
540 let interner = @mk_fresh_ident_interner();
541 local_data::set(key, interner);
547 /// Represents a string stored in the task-local interner. Because the
548 /// interner lives for the life of the task, this can be safely treated as an
549 /// immortal string, as long as it never crosses between tasks.
551 /// FIXME(pcwalton): You must be careful about what you do in the destructors
552 /// of objects stored in TLS, because they may run after the interner is
553 /// destroyed. In particular, they must not access string contents. This can
554 /// be fixed in the future by just leaking all strings until task death
556 #[deriving(Clone, Eq, Hash, Ord, TotalEq, TotalOrd)]
557 pub struct InternedString {
561 impl InternedString {
563 pub fn new(string: &'static str) -> InternedString {
565 string: RcStr::new(string),
570 fn new_from_rc_str(string: RcStr) -> InternedString {
577 pub fn get<'a>(&'a self) -> &'a str {
578 self.string.as_slice()
582 impl BytesContainer for InternedString {
583 fn container_as_bytes<'a>(&'a self) -> &'a [u8] {
584 // FIXME(pcwalton): This is a workaround for the incorrect signature
585 // of `BytesContainer`, which is itself a workaround for the lack of
588 let this = self.get();
589 cast::transmute(this.container_as_bytes())
594 impl fmt::Show for InternedString {
595 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
596 write!(f.buf, "{}", self.string.as_slice())
600 impl<'a> Equiv<&'a str> for InternedString {
601 fn equiv(&self, other: & &'a str) -> bool {
602 (*other) == self.string.as_slice()
606 impl<D:Decoder> Decodable<D> for InternedString {
607 fn decode(d: &mut D) -> InternedString {
608 get_name(get_ident_interner().intern(d.read_str()))
612 impl<E:Encoder> Encodable<E> for InternedString {
613 fn encode(&self, e: &mut E) {
614 e.emit_str(self.string.as_slice())
618 /// Returns the string contents of a name, using the task-local interner.
620 pub fn get_name(name: Name) -> InternedString {
621 let interner = get_ident_interner();
622 InternedString::new_from_rc_str(interner.get(name))
625 /// Returns the string contents of an identifier, using the task-local
628 pub fn get_ident(ident: Ident) -> InternedString {
632 /// Interns and returns the string contents of an identifier, using the
633 /// task-local interner.
635 pub fn intern_and_get_ident(s: &str) -> InternedString {
639 /// Maps a string to its interned representation.
641 pub fn intern(s: &str) -> Name {
642 get_ident_interner().intern(s)
645 /// gensym's a new uint, using the current interner.
647 pub fn gensym(s: &str) -> Name {
648 get_ident_interner().gensym(s)
651 /// Maps a string to an identifier with an empty syntax context.
653 pub fn str_to_ident(s: &str) -> ast::Ident {
654 ast::Ident::new(intern(s))
657 /// Maps a string to a gensym'ed identifier.
659 pub fn gensym_ident(s: &str) -> ast::Ident {
660 ast::Ident::new(gensym(s))
663 // create a fresh name that maps to the same string as the old one.
664 // note that this guarantees that str_ptr_eq(ident_to_str(src),interner_get(fresh_name(src)));
665 // that is, that the new name and the old one are connected to ptr_eq strings.
666 pub fn fresh_name(src: &ast::Ident) -> Name {
667 let interner = get_ident_interner();
668 interner.gensym_copy(src.name)
669 // following: debug version. Could work in final except that it's incompatible with
670 // good error messages and uses of struct names in ambiguous could-be-binding
671 // locations. Also definitely destroys the guarantee given above about ptr_eq.
672 /*let num = rand::rng().gen_uint_range(0,0xffff);
673 gensym(format!("{}_{}",ident_to_str(src),num))*/
676 // create a fresh mark.
677 pub fn fresh_mark() -> Mrk {
681 // See the macro above about the types of keywords
683 pub fn is_keyword(kw: keywords::Keyword, tok: &Token) -> bool {
685 token::IDENT(sid, false) => { kw.to_ident().name == sid.name }
690 pub fn is_any_keyword(tok: &Token) -> bool {
692 token::IDENT(sid, false) => match sid.name {
693 SELF_KEYWORD_NAME | STATIC_KEYWORD_NAME |
694 STRICT_KEYWORD_START .. RESERVED_KEYWORD_FINAL => true,
701 pub fn is_strict_keyword(tok: &Token) -> bool {
703 token::IDENT(sid, false) => match sid.name {
704 SELF_KEYWORD_NAME | STATIC_KEYWORD_NAME |
705 STRICT_KEYWORD_START .. STRICT_KEYWORD_FINAL => true,
712 pub fn is_reserved_keyword(tok: &Token) -> bool {
714 token::IDENT(sid, false) => match sid.name {
715 RESERVED_KEYWORD_START .. RESERVED_KEYWORD_FINAL => true,
722 pub fn mtwt_token_eq(t1 : &Token, t2 : &Token) -> bool {
724 (&IDENT(id1,_),&IDENT(id2,_)) | (&LIFETIME(id1),&LIFETIME(id2)) =>
725 ast_util::mtwt_resolve(id1) == ast_util::mtwt_resolve(id2),
737 fn mark_ident(id : ast::Ident, m : ast::Mrk) -> ast::Ident {
738 ast::Ident{name:id.name,ctxt:ast_util::new_mark(m,id.ctxt)}
741 #[test] fn mtwt_token_eq_test() {
742 assert!(mtwt_token_eq(>,>));
743 let a = str_to_ident("bac");
744 let a1 = mark_ident(a,92);
745 assert!(mtwt_token_eq(&IDENT(a,true),&IDENT(a1,false)));