From: Nicholas Nethercote Date: Tue, 11 Dec 2018 22:55:43 +0000 (+1100) Subject: Merge `TokenStreamKind` into `TokenStream`. X-Git-Url: https://git.lizzy.rs/?a=commitdiff_plain;h=07c12fa89e297bf5940f903dea8186a0a33f9d82;p=rust.git Merge `TokenStreamKind` into `TokenStream`. Because the distinction provides little value, and removing it cleans up the code quite a bit. --- diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index e433f8d8dfd..e31bde4082d 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -123,7 +123,7 @@ pub fn eq_token(&self, t: Token) -> bool { } pub fn joint(self) -> TokenStream { - TokenStream { kind: TokenStreamKind::JointTree(self) } + TokenStream::JointTree(self) } /// Returns the opening delimiter as a token tree. @@ -154,8 +154,11 @@ pub fn close_tt(span: Span, delim: DelimToken) -> TokenTree { /// instead of a representation of the abstract syntax tree. /// Today's `TokenTree`s can still contain AST via `Token::Interpolated` for back-compat. #[derive(Clone, Debug)] -pub struct TokenStream { - kind: TokenStreamKind, +pub enum TokenStream { + Empty, + Tree(TokenTree), + JointTree(TokenTree), + Stream(Lrc>), } // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. @@ -167,24 +170,22 @@ impl TokenStream { /// separating the two arguments with a comma for diagnostic suggestions. pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> { // Used to suggest if a user writes `foo!(a b);` - if let TokenStreamKind::Stream(ref stream) = self.kind { + if let TokenStream::Stream(ref stream) = self { let mut suggestion = None; let mut iter = stream.iter().enumerate().peekable(); while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { - let sp = match (&ts.kind, &next.kind) { - (TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma)), _) | - (_, TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma))) => { + let sp = match (&ts, &next) { + (TokenStream::Tree(TokenTree::Token(_, token::Token::Comma)), _) | + (_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma))) => { continue; } - (TokenStreamKind::Tree(TokenTree::Token(sp, _)), _) => *sp, - (TokenStreamKind::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(), + (TokenStream::Tree(TokenTree::Token(sp, _)), _) => *sp, + (TokenStream::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(), _ => continue, }; let sp = sp.shrink_to_hi(); - let comma = TokenStream { - kind: TokenStreamKind::Tree(TokenTree::Token(sp, token::Comma)), - }; + let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma)); suggestion = Some((pos, comma, sp)); } } @@ -201,17 +202,9 @@ pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> { } } -#[derive(Clone, Debug)] -enum TokenStreamKind { - Empty, - Tree(TokenTree), - JointTree(TokenTree), - Stream(Lrc>), -} - impl From for TokenStream { fn from(tt: TokenTree) -> TokenStream { - TokenStream { kind: TokenStreamKind::Tree(tt) } + TokenStream::Tree(tt) } } @@ -230,22 +223,22 @@ fn from_iter>(iter: I) -> Self { impl Extend for TokenStream { fn extend>(&mut self, iter: I) { let iter = iter.into_iter(); - let kind = mem::replace(&mut self.kind, TokenStreamKind::Empty); + let this = mem::replace(self, TokenStream::Empty); // Vector of token streams originally in self. - let tts: Vec = match kind { - TokenStreamKind::Empty => { + let tts: Vec = match this { + TokenStream::Empty => { let mut vec = Vec::new(); vec.reserve(iter.size_hint().0); vec } - TokenStreamKind::Tree(_) | TokenStreamKind::JointTree(_) => { + TokenStream::Tree(_) | TokenStream::JointTree(_) => { let mut vec = Vec::new(); vec.reserve(1 + iter.size_hint().0); - vec.push(TokenStream { kind }); + vec.push(this); vec } - TokenStreamKind::Stream(rc_vec) => match Lrc::try_unwrap(rc_vec) { + TokenStream::Stream(rc_vec) => match Lrc::try_unwrap(rc_vec) { Ok(mut vec) => { // Extend in place using the existing capacity if possible. // This is the fast path for libraries like `quote` that @@ -286,7 +279,7 @@ fn eq(&self, other: &TokenStream) -> bool { impl TokenStream { pub fn len(&self) -> usize { - if let TokenStreamKind::Stream(ref slice) = self.kind { + if let TokenStream::Stream(ref slice) = self { slice.len() } else { 0 @@ -294,12 +287,12 @@ pub fn len(&self) -> usize { } pub fn empty() -> TokenStream { - TokenStream { kind: TokenStreamKind::Empty } + TokenStream::Empty } pub fn is_empty(&self) -> bool { - match self.kind { - TokenStreamKind::Empty => true, + match self { + TokenStream::Empty => true, _ => false, } } @@ -313,7 +306,7 @@ pub fn concat(mut streams: Vec) -> TokenStream { } fn concat_rc_vec(streams: Lrc>) -> TokenStream { - TokenStream { kind: TokenStreamKind::Stream(streams) } + TokenStream::Stream(streams) } pub fn trees(&self) -> Cursor { @@ -377,9 +370,9 @@ fn semantic_tree(tree: &TokenTree) -> bool { /// Precondition: `self` consists of a single token tree. /// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`. pub fn as_tree(self) -> (TokenTree, bool /* joint? */) { - match self.kind { - TokenStreamKind::Tree(tree) => (tree, false), - TokenStreamKind::JointTree(tree) => (tree, true), + match self { + TokenStream::Tree(tree) => (tree, false), + TokenStream::JointTree(tree) => (tree, true), _ => unreachable!(), } } @@ -389,9 +382,9 @@ pub fn map_enumerated TokenTree>(self, mut f: F) - let mut result = Vec::new(); let mut i = 0; while let Some(stream) = trees.next_as_stream() { - result.push(match stream.kind { - TokenStreamKind::Tree(tree) => f(i, tree).into(), - TokenStreamKind::JointTree(tree) => f(i, tree).joint(), + result.push(match stream { + TokenStream::Tree(tree) => f(i, tree).into(), + TokenStream::JointTree(tree) => f(i, tree).joint(), _ => unreachable!() }); i += 1; @@ -403,9 +396,9 @@ pub fn map TokenTree>(self, mut f: F) -> TokenStream { let mut trees = self.into_trees(); let mut result = Vec::new(); while let Some(stream) = trees.next_as_stream() { - result.push(match stream.kind { - TokenStreamKind::Tree(tree) => f(tree).into(), - TokenStreamKind::JointTree(tree) => f(tree).joint(), + result.push(match stream { + TokenStream::Tree(tree) => f(tree).into(), + TokenStream::JointTree(tree) => f(tree).joint(), _ => unreachable!() }); } @@ -413,19 +406,19 @@ pub fn map TokenTree>(self, mut f: F) -> TokenStream { } fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> { - match self.kind { - TokenStreamKind::Empty => None, - TokenStreamKind::Tree(ref tree) => Some((tree.clone(), false)), - TokenStreamKind::JointTree(ref tree) => Some((tree.clone(), true)), - TokenStreamKind::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(), + match self { + TokenStream::Empty => None, + TokenStream::Tree(ref tree) => Some((tree.clone(), false)), + TokenStream::JointTree(ref tree) => Some((tree.clone(), true)), + TokenStream::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(), } } fn last_tree_if_joint(&self) -> Option { - match self.kind { - TokenStreamKind::Empty | TokenStreamKind::Tree(..) => None, - TokenStreamKind::JointTree(ref tree) => Some(tree.clone()), - TokenStreamKind::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(), + match self { + TokenStream::Empty | TokenStream::Tree(..) => None, + TokenStream::JointTree(ref tree) => Some(tree.clone()), + TokenStream::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(), } } } @@ -472,7 +465,7 @@ pub fn build(self) -> TokenStream { } fn push_all_but_last_tree(&mut self, stream: &TokenStream) { - if let TokenStreamKind::Stream(ref streams) = stream.kind { + if let TokenStream::Stream(ref streams) = stream { let len = streams.len(); match len { 1 => {} @@ -484,7 +477,7 @@ fn push_all_but_last_tree(&mut self, stream: &TokenStream) { } fn push_all_but_first_tree(&mut self, stream: &TokenStream) { - if let TokenStreamKind::Stream(ref streams) = stream.kind { + if let TokenStream::Stream(ref streams) = stream { let len = streams.len(); match len { 1 => {} @@ -524,10 +517,10 @@ fn next_as_stream(&mut self) -> Option { if self.index < self.stream.len() { self.index += 1; let next = self.stream[self.index - 1].clone(); - match next.kind { - TokenStreamKind::Tree(..) | TokenStreamKind::JointTree(..) => return Some(next), - TokenStreamKind::Stream(stream) => self.insert(stream), - TokenStreamKind::Empty => {} + match next { + TokenStream::Tree(..) | TokenStream::JointTree(..) => return Some(next), + TokenStream::Stream(stream) => self.insert(stream), + TokenStream::Empty => {} } } else if let Some((stream, index)) = self.stack.pop() { self.stream = stream; @@ -548,8 +541,8 @@ impl Iterator for Cursor { type Item = TokenTree; fn next(&mut self) -> Option { - self.next_as_stream().map(|stream| match stream.kind { - TokenStreamKind::Tree(tree) | TokenStreamKind::JointTree(tree) => tree, + self.next_as_stream().map(|stream| match stream { + TokenStream::Tree(tree) | TokenStream::JointTree(tree) => tree, _ => unreachable!() }) } @@ -557,11 +550,11 @@ fn next(&mut self) -> Option { impl Cursor { fn new(stream: TokenStream) -> Self { - Cursor(match stream.kind { - TokenStreamKind::Empty => CursorKind::Empty, - TokenStreamKind::Tree(tree) => CursorKind::Tree(tree, false), - TokenStreamKind::JointTree(tree) => CursorKind::JointTree(tree, false), - TokenStreamKind::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)), + Cursor(match stream { + TokenStream::Empty => CursorKind::Empty, + TokenStream::Tree(tree) => CursorKind::Tree(tree, false), + TokenStream::JointTree(tree) => CursorKind::JointTree(tree, false), + TokenStream::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)), }) } @@ -610,11 +603,11 @@ pub fn original_stream(&self) -> TokenStream { pub fn look_ahead(&self, n: usize) -> Option { fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result { for stream in streams { - n = match stream.kind { - TokenStreamKind::Tree(ref tree) | TokenStreamKind::JointTree(ref tree) + n = match stream { + TokenStream::Tree(ref tree) | TokenStream::JointTree(ref tree) if n == 0 => return Ok(tree.clone()), - TokenStreamKind::Tree(..) | TokenStreamKind::JointTree(..) => n - 1, - TokenStreamKind::Stream(ref stream) => match look_ahead(stream, n) { + TokenStream::Tree(..) | TokenStream::JointTree(..) => n - 1, + TokenStream::Stream(ref stream) => match look_ahead(stream, n) { Ok(tree) => return Ok(tree), Err(n) => n, }, @@ -660,11 +653,11 @@ pub fn stream(&self) -> TokenStream { impl From for ThinTokenStream { fn from(stream: TokenStream) -> ThinTokenStream { - ThinTokenStream(match stream.kind { - TokenStreamKind::Empty => None, - TokenStreamKind::Tree(tree) => Some(Lrc::new(vec![tree.into()])), - TokenStreamKind::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])), - TokenStreamKind::Stream(stream) => Some(stream), + ThinTokenStream(match stream { + TokenStream::Empty => None, + TokenStream::Tree(tree) => Some(Lrc::new(vec![tree.into()])), + TokenStream::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])), + TokenStream::Stream(stream) => Some(stream), }) } }