From ce0d9949b817267e88e8d366a8cee929abf1e4ba Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 9 Jan 2019 16:53:14 +1100 Subject: [PATCH] Remove `ThinTokenStream`. `TokenStream` is now almost identical to `ThinTokenStream`. This commit removes the latter, replacing it with the former. --- src/librustc/ich/impls_syntax.rs | 2 +- src/librustc_lint/builtin.rs | 2 +- src/libsyntax/ast.rs | 8 ++--- src/libsyntax/attr/mod.rs | 2 +- src/libsyntax/ext/quote.rs | 2 +- src/libsyntax/fold.rs | 2 +- src/libsyntax/parse/mod.rs | 6 ++-- src/libsyntax/parse/parser.rs | 12 +++---- src/libsyntax/print/pprust.rs | 2 +- src/libsyntax/tokenstream.rs | 55 ++------------------------------ src/libsyntax/visit.rs | 2 +- 11 files changed, 23 insertions(+), 72 deletions(-) diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index 70ec72d73bc..de567183a3c 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -258,7 +258,7 @@ fn hash_stable(&self, tokenstream::TokenTree::Delimited(span, delim, ref tts) => { span.hash_stable(hcx, hasher); std_hash::Hash::hash(&delim, hasher); - for sub_tt in tts.stream().trees() { + for sub_tt in tts.trees() { sub_tt.hash_stable(hcx, hasher); } } diff --git a/src/librustc_lint/builtin.rs b/src/librustc_lint/builtin.rs index 5678f30dabc..0fce166d828 100644 --- a/src/librustc_lint/builtin.rs +++ b/src/librustc_lint/builtin.rs @@ -1540,7 +1540,7 @@ fn check_tokens(&mut self, cx: &EarlyContext, tokens: TokenStream) { _ => {}, } TokenTree::Delimited(_, _, tts) => { - self.check_tokens(cx, tts.stream()) + self.check_tokens(cx, tts) }, } } diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index e3a8980a975..1e91f4adc36 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -15,7 +15,7 @@ use source_map::{dummy_spanned, respan, Spanned}; use symbol::{keywords, Symbol}; use syntax_pos::{Span, DUMMY_SP}; -use tokenstream::{ThinTokenStream, TokenStream}; +use tokenstream::TokenStream; use ThinVec; use rustc_data_structures::fx::FxHashSet; @@ -1216,7 +1216,7 @@ pub enum Movability { pub struct Mac_ { pub path: Path, pub delim: MacDelimiter, - pub tts: ThinTokenStream, + pub tts: TokenStream, } #[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Debug)] @@ -1228,13 +1228,13 @@ pub enum MacDelimiter { impl Mac_ { pub fn stream(&self) -> TokenStream { - self.tts.stream() + self.tts.clone() } } #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct MacroDef { - pub tokens: ThinTokenStream, + pub tokens: TokenStream, pub legacy: bool, } diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index d03563f8891..0f8ca5e7b99 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -565,7 +565,7 @@ fn from_tokens(tokens: &mut iter::Peekable) -> Option } Some(TokenTree::Delimited(_, delim, ref tts)) if delim == token::Paren => { tokens.next(); - tts.stream() + tts.clone() } _ => return Some(MetaItemKind::Word), }; diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index c3124144009..c01e7f538b9 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -748,7 +748,7 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec { let mut stmts = statements_mk_tt(cx, &TokenTree::open_tt(span.open, delim), false); - stmts.extend(statements_mk_tts(cx, tts.stream())); + stmts.extend(statements_mk_tts(cx, tts.clone())); stmts.extend(statements_mk_tt(cx, &TokenTree::close_tt(span.close, delim), false)); stmts } diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 8ac103856dc..a4c3b38f691 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -598,7 +598,7 @@ pub fn noop_fold_tt(tt: TokenTree, fld: &mut T) -> TokenTree { TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( DelimSpan::from_pair(fld.new_span(span.open), fld.new_span(span.close)), delim, - fld.fold_tts(tts.stream()).into(), + fld.fold_tts(tts).into(), ), } } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index ba5676a65d7..759de578847 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -811,7 +811,7 @@ fn string_to_tts_macro () { ) if name_macro_rules.name == "macro_rules" && name_zip.name == "zip" => { - let tts = ¯o_tts.stream().trees().collect::>(); + let tts = ¯o_tts.trees().collect::>(); match (tts.len(), tts.get(0), tts.get(1), tts.get(2)) { ( 3, @@ -820,7 +820,7 @@ fn string_to_tts_macro () { Some(&TokenTree::Delimited(_, second_delim, ref second_tts)), ) if macro_delim == token::Paren => { - let tts = &first_tts.stream().trees().collect::>(); + let tts = &first_tts.trees().collect::>(); match (tts.len(), tts.get(0), tts.get(1)) { ( 2, @@ -830,7 +830,7 @@ fn string_to_tts_macro () { if first_delim == token::Paren && ident.name == "a" => {}, _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), } - let tts = &second_tts.stream().trees().collect::>(); + let tts = &second_tts.trees().collect::>(); match (tts.len(), tts.get(0), tts.get(1)) { ( 2, diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 5c8ed94731a..6df95d539af 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -46,7 +46,7 @@ use ptr::P; use parse::PResult; use ThinVec; -use tokenstream::{self, DelimSpan, ThinTokenStream, TokenTree, TokenStream}; +use tokenstream::{self, DelimSpan, TokenTree, TokenStream}; use symbol::{Symbol, keywords}; use std::borrow::Cow; @@ -285,12 +285,12 @@ enum LastToken { } impl TokenCursorFrame { - fn new(sp: DelimSpan, delim: DelimToken, tts: &ThinTokenStream) -> Self { + fn new(sp: DelimSpan, delim: DelimToken, tts: &TokenStream) -> Self { TokenCursorFrame { delim: delim, span: sp, open_delim: delim == token::NoDelim, - tree_cursor: tts.stream().into_trees(), + tree_cursor: tts.clone().into_trees(), close_delim: delim == token::NoDelim, last_token: LastToken::Was(None), } @@ -2325,7 +2325,7 @@ pub fn mk_mac_expr(&mut self, span: Span, m: Mac_, attrs: ThinVec) -> }) } - fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, ThinTokenStream)> { + fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, TokenStream)> { let delim = match self.token { token::OpenDelim(delim) => delim, _ => { @@ -2345,7 +2345,7 @@ fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, ThinToke token::Brace => MacDelimiter::Brace, token::NoDelim => self.bug("unexpected no delimiter"), }; - Ok((delim, tts.stream().into())) + Ok((delim, tts.into())) } /// At the bottom (top?) of the precedence hierarchy, @@ -4633,7 +4633,7 @@ fn eat_macro_def(&mut self, attrs: &[Attribute], vis: &Visibility, lo: Span) let ident = self.parse_ident()?; let tokens = if self.check(&token::OpenDelim(token::Brace)) { match self.parse_token_tree() { - TokenTree::Delimited(_, _, tts) => tts.stream(), + TokenTree::Delimited(_, _, tts) => tts, _ => unreachable!(), } } else if self.check(&token::OpenDelim(token::Paren)) { diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 2ad3d3a6d64..c53594032a0 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -807,7 +807,7 @@ fn print_tt(&mut self, tt: tokenstream::TokenTree) -> io::Result<()> { TokenTree::Delimited(_, delim, tts) => { self.writer().word(token_to_string(&token::OpenDelim(delim)))?; self.writer().space()?; - self.print_tts(tts.stream())?; + self.print_tts(tts)?; self.writer().space()?; self.writer().word(token_to_string(&token::CloseDelim(delim))) }, diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 95ff7728897..d5c362490ca 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -41,7 +41,7 @@ pub enum TokenTree { /// A single token Token(Span, token::Token), /// A delimited sequence of token trees - Delimited(DelimSpan, DelimToken, ThinTokenStream), + Delimited(DelimSpan, DelimToken, TokenStream), } impl TokenTree { @@ -62,8 +62,7 @@ pub fn eq_unspanned(&self, other: &TokenTree) -> bool { (&TokenTree::Token(_, ref tk), &TokenTree::Token(_, ref tk2)) => tk == tk2, (&TokenTree::Delimited(_, delim, ref tts), &TokenTree::Delimited(_, delim2, ref tts2)) => { - delim == delim2 && - tts.stream().eq_unspanned(&tts2.stream()) + delim == delim2 && tts.eq_unspanned(&tts2) } (_, _) => false, } @@ -81,8 +80,7 @@ pub fn probably_equal_for_proc_macro(&self, other: &TokenTree) -> bool { } (&TokenTree::Delimited(_, delim, ref tts), &TokenTree::Delimited(_, delim2, ref tts2)) => { - delim == delim2 && - tts.stream().probably_equal_for_proc_macro(&tts2.stream()) + delim == delim2 && tts.probably_equal_for_proc_macro(&tts2) } (_, _) => false, } @@ -492,41 +490,6 @@ pub fn look_ahead(&self, n: usize) -> Option { } } -/// The `TokenStream` type is large enough to represent a single `TokenTree` without allocation. -/// `ThinTokenStream` is smaller, but needs to allocate to represent a single `TokenTree`. -/// We must use `ThinTokenStream` in `TokenTree::Delimited` to avoid infinite size due to recursion. -#[derive(Debug, Clone)] -pub struct ThinTokenStream(Option>>); - -impl ThinTokenStream { - pub fn stream(&self) -> TokenStream { - self.clone().into() - } -} - -impl From for ThinTokenStream { - fn from(stream: TokenStream) -> ThinTokenStream { - ThinTokenStream(match stream { - TokenStream::Empty => None, - TokenStream::Stream(stream) => Some(stream), - }) - } -} - -impl From for TokenStream { - fn from(stream: ThinTokenStream) -> TokenStream { - stream.0.map(TokenStream::Stream).unwrap_or_else(TokenStream::empty) - } -} - -impl Eq for ThinTokenStream {} - -impl PartialEq for ThinTokenStream { - fn eq(&self, other: &ThinTokenStream) -> bool { - TokenStream::from(self.clone()) == TokenStream::from(other.clone()) - } -} - impl fmt::Display for TokenStream { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.write_str(&pprust::tokens_to_string(self.clone())) @@ -545,18 +508,6 @@ fn decode(decoder: &mut D) -> Result { } } -impl Encodable for ThinTokenStream { - fn encode(&self, encoder: &mut E) -> Result<(), E::Error> { - TokenStream::from(self.clone()).encode(encoder) - } -} - -impl Decodable for ThinTokenStream { - fn decode(decoder: &mut D) -> Result { - TokenStream::decode(decoder).map(Into::into) - } -} - #[derive(Debug, Copy, Clone, PartialEq, RustcEncodable, RustcDecodable)] pub struct DelimSpan { pub open: Span, diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs index 156546bbba9..8cbd47ca70f 100644 --- a/src/libsyntax/visit.rs +++ b/src/libsyntax/visit.rs @@ -832,7 +832,7 @@ pub fn walk_attribute<'a, V: Visitor<'a>>(visitor: &mut V, attr: &'a Attribute) pub fn walk_tt<'a, V: Visitor<'a>>(visitor: &mut V, tt: TokenTree) { match tt { TokenTree::Token(_, tok) => visitor.visit_token(tok), - TokenTree::Delimited(_, _, tts) => visitor.visit_tts(tts.stream()), + TokenTree::Delimited(_, _, tts) => visitor.visit_tts(tts), } } -- 2.44.0