Address review comments.

diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs
index 8a345e67c57b32188d9ed0787998bb7a85ddbbde..06f9634d70613e2a47fe4b7d5a246087aff9a865 100644 (file)
--- a/src/libproc_macro/lib.rs
+++ b/src/libproc_macro/lib.rs
@@ -46,14 +46,14 @@
 extern crate syntax;
 extern crate syntax_pos;
 
-use std::{fmt, iter, ops};
+use std::{ascii, fmt, iter};
 use std::str::FromStr;
 
 use syntax::ast;
 use syntax::errors::DiagnosticBuilder;
 use syntax::parse::{self, token, parse_stream_from_source_str};
 use syntax::print::pprust;
-use syntax::symbol;
+use syntax::symbol::Symbol;
 use syntax::tokenstream;
 use syntax_pos::DUMMY_SP;
 use syntax_pos::SyntaxContext;
@@ -68,12 +68,12 @@
 /// The API of this type is intentionally bare-bones, but it'll be expanded over
 /// time!
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct TokenStream(tokenstream::TokenStream);
 
 /// Error returned from `TokenStream::from_str`.
-#[derive(Debug)]
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
+#[derive(Debug)]
 pub struct LexError {
     _inner: (),
 }
@@ -110,16 +110,20 @@ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 #[macro_export]
 macro_rules! quote { () => {} }
 
+#[unstable(feature = "proc_macro_internals", issue = "27812")]
+#[doc(hidden)]
+mod quote;
+
 #[unstable(feature = "proc_macro", issue = "38356")]
 impl From<TokenTree> for TokenStream {
     fn from(tree: TokenTree) -> TokenStream {
-        TokenStream(tree.to_raw())
+        TokenStream(tree.to_internal())
     }
 }
 
 #[unstable(feature = "proc_macro", issue = "38356")]
-impl From<TokenKind> for TokenStream {
-    fn from(kind: TokenKind) -> TokenStream {
+impl From<TokenNode> for TokenStream {
+    fn from(kind: TokenNode) -> TokenStream {
         TokenTree::from(kind).into()
     }
 }
@@ -127,7 +131,7 @@ fn from(kind: TokenKind) -> TokenStream {
 #[unstable(feature = "proc_macro", issue = "38356")]
 impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream {
     fn from_iter<I: IntoIterator<Item = T>>(streams: I) -> Self {
-        let mut builder = tokenstream::TokenStream::builder();
+        let mut builder = tokenstream::TokenStreamBuilder::new();
         for stream in streams {
             builder.push(stream.into().0);
         }
@@ -138,10 +142,10 @@ fn from_iter<I: IntoIterator<Item = T>>(streams: I) -> Self {
 #[unstable(feature = "proc_macro", issue = "38356")]
 impl IntoIterator for TokenStream {
     type Item = TokenTree;
-    type IntoIter = TokenIter;
+    type IntoIter = TokenTreeIter;
 
-    fn into_iter(self) -> TokenIter {
-        TokenIter { cursor: self.0.trees(), next: None }
+    fn into_iter(self) -> TokenTreeIter {
+        TokenTreeIter { cursor: self.0.trees(), next: None }
     }
 }
 
@@ -161,7 +165,7 @@ pub fn is_empty(&self) -> bool {
 
 /// A region of source code, along with macro expansion information.
 #[unstable(feature = "proc_macro", issue = "38356")]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub struct Span(syntax_pos::Span);
 
 #[unstable(feature = "proc_macro", issue = "38356")]
@@ -174,6 +178,13 @@ fn default() -> Span {
     }
 }
 
+/// Quote a `Span` into a `TokenStream`.
+/// This is needed to implement a custom quoter.
+#[unstable(feature = "proc_macro", issue = "38356")]
+pub fn quote_span(span: Span) -> TokenStream {
+    TokenStream(quote::Quote::quote(&span.0))
+}
+
 impl Span {
     /// The span of the invocation of the current procedural macro.
     #[unstable(feature = "proc_macro", issue = "38356")]
@@ -184,17 +195,17 @@ pub fn call_site() -> Span {
 
 /// A single token or a delimited sequence of token trees (e.g. `[1, (), ..]`).
 #[unstable(feature = "proc_macro", issue = "38356")]
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct TokenTree {
     /// The `TokenTree`'s span
     pub span: Span,
     /// Description of the `TokenTree`
-    pub kind: TokenKind,
+    pub kind: TokenNode,
 }
 
 #[unstable(feature = "proc_macro", issue = "38356")]
-impl From<TokenKind> for TokenTree {
-    fn from(kind: TokenKind) -> TokenTree {
+impl From<TokenNode> for TokenTree {
+    fn from(kind: TokenNode) -> TokenTree {
         TokenTree { span: Span::default(), kind: kind }
     }
 }
@@ -207,21 +218,21 @@ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 }
 
 /// Description of a `TokenTree`
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 #[unstable(feature = "proc_macro", issue = "38356")]
-pub enum TokenKind {
+pub enum TokenNode {
     /// A delimited tokenstream.
-    Sequence(Delimiter, TokenStream),
+    Group(Delimiter, TokenStream),
     /// A unicode identifier.
-    Word(Symbol),
+    Term(Term),
     /// A punctuation character (`+`, `,`, `$`, etc.).
-    Op(char, OpKind),
+    Op(char, Spacing),
     /// A literal character (`'a'`), string (`"hello"`), or number (`2.3`).
     Literal(Literal),
 }
 
 /// Describes how a sequence of token trees is delimited.
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 #[unstable(feature = "proc_macro", issue = "38356")]
 pub enum Delimiter {
     /// `( ... )`
@@ -235,30 +246,28 @@ pub enum Delimiter {
 }
 
 /// An interned string.
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 #[unstable(feature = "proc_macro", issue = "38356")]
-pub struct Symbol(symbol::Symbol);
+pub struct Term(Symbol);
 
-#[unstable(feature = "proc_macro", issue = "38356")]
-impl<'a> From<&'a str> for Symbol {
-    fn from(string: &'a str) -> Symbol {
-        Symbol(symbol::Symbol::intern(string))
+impl Term {
+    /// Intern a string into a `Term`.
+    #[unstable(feature = "proc_macro", issue = "38356")]
+    pub fn intern(string: &str) -> Term {
+        Term(Symbol::intern(string))
     }
-}
 
-#[unstable(feature = "proc_macro", issue = "38356")]
-impl ops::Deref for Symbol {
-    type Target = str;
-
-    fn deref(&self) -> &str {
-        unsafe { &*(self.0.as_str().deref() as *const str) }
+    /// Get a reference to the interned string.
+    #[unstable(feature = "proc_macro", issue = "38356")]
+    pub fn as_str(&self) -> &str {
+        unsafe { &*(&*self.0.as_str() as *const str) }
     }
 }
 
 /// Whether an `Op` is either followed immediately by another `Op` or followed by whitespace.
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 #[unstable(feature = "proc_macro", issue = "38356")]
-pub enum OpKind {
+pub enum Spacing {
     /// e.g. `+` is `Alone` in `+ =`.
     Alone,
     /// e.g. `+` is `Joint` in `+=`.
@@ -266,14 +275,14 @@ pub enum OpKind {
 }
 
 /// A literal character (`'a'`), string (`"hello"`), or number (`2.3`).
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 #[unstable(feature = "proc_macro", issue = "38356")]
 pub struct Literal(token::Token);
 
 #[unstable(feature = "proc_macro", issue = "38356")]
 impl fmt::Display for Literal {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        TokenTree { kind: TokenKind::Literal(self.clone()), span: Span(DUMMY_SP) }.fmt(f)
+        TokenTree { kind: TokenNode::Literal(self.clone()), span: Span(DUMMY_SP) }.fmt(f)
     }
 }
 
@@ -282,30 +291,51 @@ macro_rules! int_literals {
         /// Integer literal.
         #[unstable(feature = "proc_macro", issue = "38356")]
         pub fn $int_kind(n: $int_kind) -> Literal {
-            Literal::integer(n as i128, stringify!($int_kind))
+            Literal::typed_integer(n as i128, stringify!($int_kind))
         }
     )*}
 }
 
 impl Literal {
+    /// Integer literal
+    #[unstable(feature = "proc_macro", issue = "38356")]
+    pub fn integer(n: i128) -> Literal {
+        Literal(token::Literal(token::Lit::Integer(Symbol::intern(&n.to_string())), None))
+    }
+
     int_literals!(u8, i8, u16, i16, u32, i32, u64, i64);
-    fn integer(n: i128, kind: &'static str) -> Literal {
-        Literal(token::Literal(token::Lit::Integer(symbol::Symbol::intern(&n.to_string())),
-                               Some(symbol::Symbol::intern(kind))))
+    fn typed_integer(n: i128, kind: &'static str) -> Literal {
+        Literal(token::Literal(token::Lit::Integer(Symbol::intern(&n.to_string())),
+                               Some(Symbol::intern(kind))))
+    }
+
+    /// Floating point literal.
+    #[unstable(feature = "proc_macro", issue = "38356")]
+    pub fn float(n: f64) -> Literal {
+        if !n.is_finite() {
+            panic!("Invalid float literal {}", n);
+        }
+        Literal(token::Literal(token::Lit::Float(Symbol::intern(&n.to_string())), None))
     }
 
     /// Floating point literal.
     #[unstable(feature = "proc_macro", issue = "38356")]
     pub fn f32(n: f32) -> Literal {
-        Literal(token::Literal(token::Lit::Float(symbol::Symbol::intern(&n.to_string())),
-                               Some(symbol::Symbol::intern("f32"))))
+        if !n.is_finite() {
+            panic!("Invalid f32 literal {}", n);
+        }
+        Literal(token::Literal(token::Lit::Float(Symbol::intern(&n.to_string())),
+                               Some(Symbol::intern("f32"))))
     }
 
     /// Floating point literal.
     #[unstable(feature = "proc_macro", issue = "38356")]
-    pub fn f64(n: f32) -> Literal {
-        Literal(token::Literal(token::Lit::Float(symbol::Symbol::intern(&n.to_string())),
-                               Some(symbol::Symbol::intern("f64"))))
+    pub fn f64(n: f64) -> Literal {
+        if !n.is_finite() {
+            panic!("Invalid f64 literal {}", n);
+        }
+        Literal(token::Literal(token::Lit::Float(Symbol::intern(&n.to_string())),
+                               Some(Symbol::intern("f64"))))
     }
 
     /// String literal.
@@ -315,7 +345,7 @@ pub fn string(string: &str) -> Literal {
         for ch in string.chars() {
             escaped.extend(ch.escape_unicode());
         }
-        Literal(token::Literal(token::Lit::Str_(symbol::Symbol::intern(&escaped)), None))
+        Literal(token::Literal(token::Lit::Str_(Symbol::intern(&escaped)), None))
     }
 
     /// Character literal.
@@ -323,28 +353,36 @@ pub fn string(string: &str) -> Literal {
     pub fn character(ch: char) -> Literal {
         let mut escaped = String::new();
         escaped.extend(ch.escape_unicode());
-        Literal(token::Literal(token::Lit::Char(symbol::Symbol::intern(&escaped)), None))
+        Literal(token::Literal(token::Lit::Char(Symbol::intern(&escaped)), None))
+    }
+
+    /// Byte string literal.
+    #[unstable(feature = "proc_macro", issue = "38356")]
+    pub fn byte_string(bytes: &[u8]) -> Literal {
+        let string = bytes.iter().cloned().flat_map(ascii::escape_default)
+            .map(Into::<char>::into).collect::<String>();
+        Literal(token::Literal(token::Lit::ByteStr(Symbol::intern(&string)), None))
     }
 }
 
 /// An iterator over `TokenTree`s.
 #[unstable(feature = "proc_macro", issue = "38356")]
-pub struct TokenIter {
+pub struct TokenTreeIter {
     cursor: tokenstream::Cursor,
     next: Option<tokenstream::TokenStream>,
 }
 
 #[unstable(feature = "proc_macro", issue = "38356")]
-impl Iterator for TokenIter {
+impl Iterator for TokenTreeIter {
     type Item = TokenTree;
 
     fn next(&mut self) -> Option<TokenTree> {
         loop {
             let next =
                 unwrap_or!(self.next.take().or_else(|| self.cursor.next_as_stream()), return None);
-            let tree = TokenTree::from_raw(next, &mut self.next);
+            let tree = TokenTree::from_internal(next, &mut self.next);
             if tree.span.0 == DUMMY_SP {
-                if let TokenKind::Sequence(Delimiter::None, stream) = tree.kind {
+                if let TokenNode::Group(Delimiter::None, stream) = tree.kind {
                     self.cursor.insert(stream.0);
                     continue
                 }
@@ -355,7 +393,7 @@ fn next(&mut self) -> Option<TokenTree> {
 }
 
 impl Delimiter {
-    fn from_raw(delim: token::DelimToken) -> Delimiter {
+    fn from_internal(delim: token::DelimToken) -> Delimiter {
         match delim {
             token::Paren => Delimiter::Parenthesis,
             token::Brace => Delimiter::Brace,
@@ -364,7 +402,7 @@ fn from_raw(delim: token::DelimToken) -> Delimiter {
         }
     }
 
-    fn to_raw(self) -> token::DelimToken {
+    fn to_internal(self) -> token::DelimToken {
         match self {
             Delimiter::Parenthesis => token::Paren,
             Delimiter::Brace => token::Brace,
@@ -375,7 +413,7 @@ fn to_raw(self) -> token::DelimToken {
 }
 
 impl TokenTree {
-    fn from_raw(stream: tokenstream::TokenStream, next: &mut Option<tokenstream::TokenStream>)
+    fn from_internal(stream: tokenstream::TokenStream, next: &mut Option<tokenstream::TokenStream>)
                 -> TokenTree {
         use syntax::parse::token::*;
 
@@ -383,17 +421,17 @@ fn from_raw(stream: tokenstream::TokenStream, next: &mut Option<tokenstream::Tok
         let (mut span, token) = match tree {
             tokenstream::TokenTree::Token(span, token) => (span, token),
             tokenstream::TokenTree::Delimited(span, delimed) => {
-                let delimiter = Delimiter::from_raw(delimed.delim);
+                let delimiter = Delimiter::from_internal(delimed.delim);
                 return TokenTree {
                     span: Span(span),
-                    kind: TokenKind::Sequence(delimiter, TokenStream(delimed.tts.into())),
+                    kind: TokenNode::Group(delimiter, TokenStream(delimed.tts.into())),
                 };
             }
         };
 
-        let op_kind = if is_joint { OpKind::Joint } else { OpKind::Alone };
+        let op_kind = if is_joint { Spacing::Joint } else { Spacing::Alone };
         macro_rules! op {
-            ($op:expr) => { TokenKind::Op($op, op_kind) }
+            ($op:expr) => { TokenNode::Op($op, op_kind) }
         }
 
         macro_rules! joint {
@@ -402,12 +440,12 @@ macro_rules! joint {
 
         fn joint(first: char, rest: Token, is_joint: bool, span: &mut syntax_pos::Span,
                  next: &mut Option<tokenstream::TokenStream>)
-                 -> TokenKind {
+                 -> TokenNode {
             let (first_span, rest_span) = (*span, *span);
             *span = first_span;
             let tree = tokenstream::TokenTree::Token(rest_span, rest);
             *next = Some(if is_joint { tree.joint() } else { tree.into() });
-            TokenKind::Op(first, OpKind::Joint)
+            TokenNode::Op(first, Spacing::Joint)
         }
 
         let kind = match token {
@@ -458,11 +496,11 @@ fn joint(first: char, rest: Token, is_joint: bool, span: &mut syntax_pos::Span,
             Question => op!('?'),
             Underscore => op!('_'),
 
-            Ident(ident) | Lifetime(ident) => TokenKind::Word(Symbol(ident.name)),
-            Literal(..) | DocComment(..) => TokenKind::Literal(self::Literal(token)),
+            Ident(ident) | Lifetime(ident) => TokenNode::Term(Term(ident.name)),
+            Literal(..) | DocComment(..) => TokenNode::Literal(self::Literal(token)),
 
             Interpolated(ref nt) => __internal::with_sess(|(sess, _)| {
-                TokenKind::Sequence(Delimiter::None, TokenStream(nt.1.force(|| {
+                TokenNode::Group(Delimiter::None, TokenStream(nt.1.force(|| {
                     // FIXME(jseyfried): Avoid this pretty-print + reparse hack
                     let name = "<macro expansion>".to_owned();
                     let source = pprust::token_to_string(&token);
@@ -477,25 +515,25 @@ fn joint(first: char, rest: Token, is_joint: bool, span: &mut syntax_pos::Span,
         TokenTree { span: Span(span), kind: kind }
     }
 
-    fn to_raw(self) -> tokenstream::TokenStream {
+    fn to_internal(self) -> tokenstream::TokenStream {
         use syntax::parse::token::*;
         use syntax::tokenstream::{TokenTree, Delimited};
 
         let (op, kind) = match self.kind {
-            TokenKind::Op(op, kind) => (op, kind),
-            TokenKind::Sequence(delimiter, tokens) => {
+            TokenNode::Op(op, kind) => (op, kind),
+            TokenNode::Group(delimiter, tokens) => {
                 return TokenTree::Delimited(self.span.0, Delimited {
-                    delim: delimiter.to_raw(),
+                    delim: delimiter.to_internal(),
                     tts: tokens.0.into(),
                 }).into();
             },
-            TokenKind::Word(symbol) => {
+            TokenNode::Term(symbol) => {
                 let ident = ast::Ident { name: symbol.0, ctxt: self.span.0.ctxt };
                 let token =
                     if symbol.0.as_str().starts_with("'") { Lifetime(ident) } else { Ident(ident) };
                 return TokenTree::Token(self.span.0, token).into();
             }
-            TokenKind::Literal(token) => return TokenTree::Token(self.span.0, token.0).into(),
+            TokenNode::Literal(token) => return TokenTree::Token(self.span.0, token.0).into(),
         };
 
         let token = match op {
@@ -526,8 +564,8 @@ fn to_raw(self) -> tokenstream::TokenStream {
 
         let tree = TokenTree::Token(self.span.0, token);
         match kind {
-            OpKind::Alone => tree.into(),
-            OpKind::Joint => tree.joint(),
+            Spacing::Alone => tree.into(),
+            Spacing::Joint => tree.joint(),
         }
     }
 }
@@ -543,10 +581,8 @@ fn to_raw(self) -> tokenstream::TokenStream {
 /// all of the contents.
 #[unstable(feature = "proc_macro_internals", issue = "27812")]
 #[doc(hidden)]
-#[path = ""]
 pub mod __internal {
-    mod quote;
-    pub use self::quote::{Quoter, __rt};
+    pub use quote::{Quoter, __rt};
 
     use std::cell::Cell;
 

diff --git a/src/libproc_macro/quote.rs b/src/libproc_macro/quote.rs
index a3ea3925fcd4818f510ed979d129e918cdf6e83d..bee2c1e0eb6b608a21e273a383bea64221d4a3ba 100644 (file)
--- a/src/libproc_macro/quote.rs
+++ b/src/libproc_macro/quote.rs
@@ -9,13 +9,17 @@
 // except according to those terms.
 
 //! # Quasiquoter
-//! This file contains the implementation internals of the quasiquoter provided by `qquote!`.
+//! This file contains the implementation internals of the quasiquoter provided by `quote!`.
+
+//! This quasiquoter uses macros 2.0 hygiene to reliably use items from `__rt`,
+//! including re-exported API `libsyntax`, to build a `syntax::tokenstream::TokenStream`
+//! and wrap it into a `proc_macro::TokenStream`.
 
 use syntax::ast::Ident;
 use syntax::ext::base::{ExtCtxt, ProcMacro};
 use syntax::parse::token::{self, Token, Lit};
 use syntax::symbol::Symbol;
-use syntax::tokenstream::{Delimited, TokenTree, TokenStream};
+use syntax::tokenstream::{Delimited, TokenTree, TokenStream, TokenStreamBuilder};
 use syntax_pos::{DUMMY_SP, Span};
 use syntax_pos::hygiene::SyntaxContext;
 
@@ -25,7 +29,7 @@ pub mod __rt {
     pub use syntax::ast::Ident;
     pub use syntax::parse::token;
     pub use syntax::symbol::Symbol;
-    pub use syntax::tokenstream::{TokenStream, TokenTree, Delimited};
+    pub use syntax::tokenstream::{TokenStream, TokenStreamBuilder, TokenTree, Delimited};
     pub use super::{ctxt, span};
 
     pub fn unquote<T: Into<::TokenStream> + Clone>(tokens: &T) -> TokenStream {
@@ -41,7 +45,7 @@ pub fn span() -> Span {
     ::Span::default().0
 }
 
-trait Quote {
+pub trait Quote {
     fn quote(&self) -> TokenStream;
 }
 
@@ -98,8 +102,8 @@ fn quote(&self) -> TokenStream {
 
 impl Quote for TokenStream {
     fn quote(&self) -> TokenStream {
-        let mut builder = TokenStream::builder();
-        builder.push(quote!(rt::TokenStream::builder()));
+        let mut builder = TokenStreamBuilder::new();
+        builder.push(quote!(rt::TokenStreamBuilder::new()));
 
         let mut trees = self.trees();
         loop {

diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs
index 57a09ed15032f4aace3fe9429cbbb2bb42136cd1..e008a5cd9ea6179e8814a902ac465f2c1a31539d 100644 (file)
--- a/src/librustc_metadata/creader.rs
+++ b/src/librustc_metadata/creader.rs
@@ -14,7 +14,7 @@
 use locator::{self, CratePaths};
 use schema::{CrateRoot, Tracked};
 
-use rustc::hir::def_id::{CrateNum, DefIndex};
+use rustc::hir::def_id::{CrateNum, DefIndex, CRATE_DEF_INDEX};
 use rustc::hir::svh::Svh;
 use rustc::middle::cstore::DepKind;
 use rustc::session::Session;
@@ -35,7 +35,7 @@
 use std::rc::Rc;
 use std::{cmp, fs};
 
-use syntax::ast::{self, Ident};
+use syntax::ast;
 use syntax::abi::Abi;
 use syntax::attr;
 use syntax::ext::base::SyntaxExtension;
@@ -1238,7 +1238,7 @@ fn proc_macro_def_path_table(proc_macros: &[(ast::Name, Rc<SyntaxExtension>)]) -
         let key = DefKey {
             parent: Some(CRATE_DEF_INDEX),
             disambiguated_data: DisambiguatedDefPathData {
-                data: DefPathData::MacroDef(Ident::with_empty_ctxt(proc_macro.0)),
+                data: DefPathData::MacroDef(proc_macro.0),
                 disambiguator: 0,
             },
         };

diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index afc1e583d69bb7a0928f1488ee40f3858b55f03b..66775d8c43d638edc0ef04ce2a772e873f292b4c 100644 (file)
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -483,7 +483,7 @@ fn scan_optional_raw_name(&mut self) -> Option<ast::Name> {
         self.with_str_from(start, |string| {
             if string == "_" {
                 self.sess.span_diagnostic
-                    .struct_span_warn(mk_sp(start, self.pos),
+                    .struct_span_warn(self.mk_sp(start, self.pos),
                                       "underscore literal suffix is not allowed")
                     .warn("this was previously accepted by the compiler but is \
                           being phased out; it will become a hard error in \

diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 189a18f44203346ebdb9bba842d3c1953dd8f560..d4198261d3f692a604fa04905e1e1626964b8f85 100644 (file)
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -20,8 +20,8 @@
 use symbol::keywords;
 use tokenstream::{TokenStream, TokenTree};
 
-use std::cell::RefCell;
-use std::fmt;
+use std::cell::Cell;
+use std::{cmp, fmt};
 use std::rc::Rc;
 
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
@@ -169,7 +169,8 @@ pub enum Token {
     Underscore,
     Lifetime(ast::Ident),
 
-    /* For interpolation */
+    // The `LazyTokenStream` is a pure function of the `Nonterminal`,
+    // and so the `LazyTokenStream` can be ignored by Eq, Hash, etc.
     Interpolated(Rc<(Nonterminal, LazyTokenStream)>),
     // Can be expanded into several tokens.
     /// Doc comment
@@ -468,19 +469,40 @@ pub fn is_op(tok: &Token) -> bool {
     }
 }
 
-#[derive(Clone, Eq, PartialEq, Debug)]
-pub struct LazyTokenStream(RefCell<Option<TokenStream>>);
+pub struct LazyTokenStream(Cell<Option<TokenStream>>);
+
+impl Clone for LazyTokenStream {
+    fn clone(&self) -> Self {
+        let opt_stream = self.0.take();
+        self.0.set(opt_stream.clone());
+        LazyTokenStream(Cell::new(opt_stream))
+    }
+}
+
+impl cmp::Eq for LazyTokenStream {}
+impl PartialEq for LazyTokenStream {
+    fn eq(&self, _other: &LazyTokenStream) -> bool {
+        true
+    }
+}
+
+impl fmt::Debug for LazyTokenStream {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        fmt::Debug::fmt(&self.clone().0.into_inner(), f)
+    }
+}
 
 impl LazyTokenStream {
     pub fn new() -> Self {
-        LazyTokenStream(RefCell::new(None))
+        LazyTokenStream(Cell::new(None))
     }
 
     pub fn force<F: FnOnce() -> TokenStream>(&self, f: F) -> TokenStream {
-        let mut opt_stream = self.0.borrow_mut();
+        let mut opt_stream = self.0.take();
         if opt_stream.is_none() {
-            *opt_stream = Some(f());
-        };
+            opt_stream = Some(f());
+        }
+        self.0.set(opt_stream.clone());
         opt_stream.clone().unwrap()
     }
 }
@@ -498,7 +520,5 @@ fn decode<D: Decoder>(_: &mut D) -> Result<LazyTokenStream, D::Error> {
 }
 
 impl ::std::hash::Hash for LazyTokenStream {
-    fn hash<H: ::std::hash::Hasher>(&self, hasher: &mut H) {
-        self.0.borrow().hash(hasher);
-    }
+    fn hash<H: ::std::hash::Hasher>(&self, _hasher: &mut H) {}
 }

diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index a3c3fa3a52ee7311686cbba664ff636acb5e6a8e..8eee25405df6bca679924f97ca76acd4e41ad9a2 100644 (file)
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -201,10 +201,6 @@ pub fn is_empty(&self) -> bool {
         }
     }
 
-    pub fn builder() -> TokenStreamBuilder {
-        TokenStreamBuilder(Vec::new())
-    }
-
     pub fn concat(mut streams: Vec<TokenStream>) -> TokenStream {
         match streams.len() {
             0 => TokenStream::empty(),
@@ -235,6 +231,8 @@ pub fn eq_unspanned(&self, other: &TokenStream) -> bool {
         true
     }
 
+    /// Precondition: `self` consists of a single token tree.
+    /// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`.
     pub fn as_tree(self) -> (TokenTree, bool /* joint? */) {
         match self.kind {
             TokenStreamKind::Tree(tree) => (tree, false),
@@ -277,6 +275,10 @@ fn last_tree_if_joint(&self) -> Option<TokenTree> {
 pub struct TokenStreamBuilder(Vec<TokenStream>);
 
 impl TokenStreamBuilder {
+    pub fn new() -> TokenStreamBuilder {
+        TokenStreamBuilder(Vec::new())
+    }
+
     pub fn push<T: Into<TokenStream>>(&mut self, stream: T) {
         let stream = stream.into();
         let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint);

diff --git a/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs b/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs
index 9406eda5231d50dabd6a9d32be05f3b6d00073de..6d6a452b03b627282cb24298f473f8bac3f3e5b3 100644 (file)
--- a/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs
@@ -15,7 +15,7 @@
 
 extern crate proc_macro;
 
-use proc_macro::{TokenStream, TokenKind, quote};
+use proc_macro::{TokenStream, TokenNode, quote};
 
 #[proc_macro]
 pub fn cond(input: TokenStream) -> TokenStream {
@@ -23,7 +23,7 @@ pub fn cond(input: TokenStream) -> TokenStream {
     let mut input = input.into_iter().peekable();
     while let Some(tree) = input.next() {
         let cond = match tree.kind {
-            TokenKind::Sequence(_, cond) => cond,
+            TokenNode::Sequence(_, cond) => cond,
             _ => panic!("Invalid input"),
         };
         let mut cond_trees = cond.clone().into_iter();
@@ -33,7 +33,7 @@ pub fn cond(input: TokenStream) -> TokenStream {
             panic!("Invalid macro usage in cond: {}", cond);
         }
         let is_else = match test.kind {
-            TokenKind::Word(word) => *word == *"else",
+            TokenNode::Word(word) => word.as_str() == "else",
             _ => false,
         };
         conds.push(if is_else || input.peek().is_none() {

diff --git a/src/test/run-pass-fulldeps/proc-macro/auxiliary/count_compound_ops.rs b/src/test/run-pass-fulldeps/proc-macro/auxiliary/count_compound_ops.rs
index 2ff9dc284946038ee58b94226edbeac62c9d9a11..e7a0283962d81fa613311ee9afb3d4e83937d1e0 100644 (file)
--- a/src/test/run-pass-fulldeps/proc-macro/auxiliary/count_compound_ops.rs
+++ b/src/test/run-pass-fulldeps/proc-macro/auxiliary/count_compound_ops.rs
@@ -15,20 +15,20 @@
 
 extern crate proc_macro;
 
-use proc_macro::{TokenStream, TokenKind, OpKind, Literal, quote};
+use proc_macro::{TokenStream, TokenNode, OpKind, Literal, quote};
 
 #[proc_macro]
 pub fn count_compound_ops(input: TokenStream) -> TokenStream {
     assert_eq!(count_compound_ops_helper(quote!(++ (&&) 4@a)), 3);
-    TokenKind::Literal(Literal::u32(count_compound_ops_helper(input))).into()
+    TokenNode::Literal(Literal::u32(count_compound_ops_helper(input))).into()
 }
 
 fn count_compound_ops_helper(input: TokenStream) -> u32 {
     let mut count = 0;
     for token in input {
         match token.kind {
-            TokenKind::Op(c, OpKind::Alone) => count += 1,
-            TokenKind::Sequence(_, tokens) => count += count_compound_ops_helper(tokens),
+            TokenNode::Op(c, OpKind::Alone) => count += 1,
+            TokenNode::Sequence(_, tokens) => count += count_compound_ops_helper(tokens),
             _ => {}
         }
     }