From a5b3f33cb90bf991342afa552bcd993e36f80fa7 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Thu, 9 May 2019 19:04:04 +0300 Subject: [PATCH] Eliminate `comments::Literal` --- src/librustc/hir/print.rs | 45 ++------ src/librustc_driver/pretty.rs | 6 +- src/librustdoc/clean/cfg.rs | 16 +-- src/libsyntax/attr/mod.rs | 7 +- src/libsyntax/parse/lexer/comments.rs | 29 +---- src/libsyntax/parse/parser.rs | 10 +- src/libsyntax/print/pprust.rs | 158 ++++++-------------------- 7 files changed, 61 insertions(+), 210 deletions(-) diff --git a/src/librustc/hir/print.rs b/src/librustc/hir/print.rs index b7260abb521..6d48ad94a66 100644 --- a/src/librustc/hir/print.rs +++ b/src/librustc/hir/print.rs @@ -19,7 +19,6 @@ use std::borrow::Cow; use std::cell::Cell; use std::io::{self, Write, Read}; -use std::iter::Peekable; use std::vec; pub enum AnnNode<'a> { @@ -77,7 +76,6 @@ pub struct State<'a> { pub s: pp::Printer<'a>, cm: Option<&'a SourceMap>, comments: Option>, - literals: Peekable>, cur_cmnt: usize, boxes: Vec, ann: &'a (dyn PpAnn + 'a), @@ -99,14 +97,6 @@ fn comments(&mut self) -> &mut Option> { fn cur_cmnt(&mut self) -> &mut usize { &mut self.cur_cmnt } - - fn cur_lit(&mut self) -> Option<&comments::Literal> { - self.literals.peek() - } - - fn bump_lit(&mut self) -> Option { - self.literals.next() - } } #[allow(non_upper_case_globals)] @@ -117,18 +107,16 @@ fn bump_lit(&mut self) -> Option { /// Requires you to pass an input filename and reader so that -/// it can scan the input text for comments and literals to -/// copy forward. +/// it can scan the input text for comments to copy forward. pub fn print_crate<'a>(cm: &'a SourceMap, sess: &ParseSess, krate: &hir::Crate, filename: FileName, input: &mut dyn Read, out: Box, - ann: &'a dyn PpAnn, - is_expanded: bool) + ann: &'a dyn PpAnn) -> io::Result<()> { - let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded); + let mut s = State::new_from_input(cm, sess, filename, input, out, ann); // When printing the AST, we sometimes need to inject `#[no_std]` here. // Since you can't compile the HIR, it's not necessary. @@ -144,36 +132,21 @@ pub fn new_from_input(cm: &'a SourceMap, filename: FileName, input: &mut dyn Read, out: Box, - ann: &'a dyn PpAnn, - is_expanded: bool) + ann: &'a dyn PpAnn) -> State<'a> { - let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input); - - State::new(cm, - out, - ann, - Some(cmnts), - // If the code is post expansion, don't use the table of - // literals, since it doesn't correspond with the literals - // in the AST anymore. - if is_expanded { - None - } else { - Some(lits) - }) + let comments = comments::gather_comments(sess, filename, input); + State::new(cm, out, ann, Some(comments)) } pub fn new(cm: &'a SourceMap, out: Box, ann: &'a dyn PpAnn, - comments: Option>, - literals: Option>) + comments: Option>) -> State<'a> { State { s: pp::mk_printer(out, default_columns), cm: Some(cm), comments, - literals: literals.unwrap_or_default().into_iter().peekable(), cur_cmnt: 0, boxes: Vec::new(), ann, @@ -190,7 +163,6 @@ pub fn to_string(ann: &dyn PpAnn, f: F) -> String s: pp::mk_printer(Box::new(&mut wr), default_columns), cm: None, comments: None, - literals: vec![].into_iter().peekable(), cur_cmnt: 0, boxes: Vec::new(), ann, @@ -1279,9 +1251,6 @@ fn print_expr_addr_of(&mut self, fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> { self.maybe_print_comment(lit.span.lo())?; - if let Some(ltrl) = self.next_lit(lit.span.lo()) { - return self.writer().word(ltrl.lit.clone()); - } match lit.node { hir::LitKind::Str(st, style) => self.print_string(&st.as_str(), style), hir::LitKind::Err(st) => { diff --git a/src/librustc_driver/pretty.rs b/src/librustc_driver/pretty.rs index fc8bf0baa99..c74ed5ec30c 100644 --- a/src/librustc_driver/pretty.rs +++ b/src/librustc_driver/pretty.rs @@ -805,8 +805,7 @@ pub fn print_after_hir_lowering<'tcx>( src_name, &mut rdr, box out, - annotation.pp_ann(), - true) + annotation.pp_ann()) }) } @@ -829,8 +828,7 @@ pub fn print_after_hir_lowering<'tcx>( src_name, &mut rdr, box out, - annotation.pp_ann(), - true); + annotation.pp_ann()); for node_id in uii.all_matching_node_ids(hir_map) { let node = hir_map.get(node_id); pp_state.print_node(node)?; diff --git a/src/librustdoc/clean/cfg.rs b/src/librustdoc/clean/cfg.rs index 69445451503..257f02af4cd 100644 --- a/src/librustdoc/clean/cfg.rs +++ b/src/librustdoc/clean/cfg.rs @@ -414,10 +414,9 @@ fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { mod test { use super::Cfg; - use syntax::symbol::Symbol; - use syntax::ast::*; - use syntax::source_map::dummy_spanned; use syntax_pos::DUMMY_SP; + use syntax::ast::*; + use syntax::symbol::Symbol; use syntax::with_globals; fn word_cfg(s: &str) -> Cfg { @@ -592,12 +591,11 @@ fn test_parse_ok() { let mi = dummy_meta_item_word("all"); assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all"))); + let node = LitKind::Str(Symbol::intern("done"), StrStyle::Cooked); + let (token, suffix) = node.lit_token(); let mi = MetaItem { path: Path::from_ident(Ident::from_str("all")), - node: MetaItemKind::NameValue(dummy_spanned(LitKind::Str( - Symbol::intern("done"), - StrStyle::Cooked, - ))), + node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }), span: DUMMY_SP, }; assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done"))); @@ -627,9 +625,11 @@ fn test_parse_ok() { #[test] fn test_parse_err() { with_globals(|| { + let node = LitKind::Bool(false); + let (token, suffix) = node.lit_token(); let mi = MetaItem { path: Path::from_ident(Ident::from_str("foo")), - node: MetaItemKind::NameValue(dummy_spanned(LitKind::Bool(false))), + node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }), span: DUMMY_SP, }; assert!(Cfg::parse(&mi).is_err()); diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 2b874581083..e23c1826651 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -565,8 +565,9 @@ fn from_tokens(tokens: &mut iter::Peekable) -> Option Some(TokenTree::Token(_, token::Eq)) => { tokens.next(); return if let Some(TokenTree::Token(span, token)) = tokens.next() { - LitKind::from_token(token) - .map(|(node, token, suffix)| MetaItemKind::NameValue(Lit { node, token, suffix, span })) + LitKind::from_token(token).map(|(node, token, suffix)| { + MetaItemKind::NameValue(Lit { node, token, suffix, span }) + }) } else { None }; @@ -635,7 +636,7 @@ fn token(&self) -> Token { } } - pub(crate) fn lit_token(&self) -> (token::Lit, Option) { + pub fn lit_token(&self) -> (token::Lit, Option) { use std::ascii; match *self { diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index 74fff3324ea..97d3fc002e9 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -3,8 +3,7 @@ use crate::ast; use crate::source_map::SourceMap; use crate::parse::lexer::{is_block_doc_comment, is_pattern_whitespace}; -use crate::parse::lexer::{self, ParseSess, StringReader, TokenAndSpan}; -use crate::print::pprust; +use crate::parse::lexer::{self, ParseSess, StringReader}; use syntax_pos::{BytePos, CharPos, Pos, FileName}; use log::debug; @@ -339,16 +338,9 @@ fn consume_comment(rdr: &mut StringReader<'_>, debug!("<<< consume comment"); } -#[derive(Clone)] -pub struct Literal { - pub lit: String, - pub pos: BytePos, -} - // it appears this function is called only from pprust... that's // probably not a good thing. -pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut dyn Read) - -> (Vec, Vec) +pub fn gather_comments(sess: &ParseSess, path: FileName, srdr: &mut dyn Read) -> Vec { let mut src = String::new(); srdr.read_to_string(&mut src).unwrap(); @@ -357,7 +349,6 @@ pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut let mut rdr = lexer::StringReader::new_raw(sess, source_file, None); let mut comments: Vec = Vec::new(); - let mut literals: Vec = Vec::new(); let mut code_to_the_left = false; // Only code let mut anything_to_the_left = false; // Code or comments @@ -382,26 +373,12 @@ pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut } } - let bstart = rdr.pos; rdr.next_token(); - // discard, and look ahead; we're working with internal state - let TokenAndSpan { tok, sp } = rdr.peek(); - if tok.is_lit() { - rdr.with_str_from(bstart, |s| { - debug!("tok lit: {}", s); - literals.push(Literal { - lit: s.to_string(), - pos: sp.lo(), - }); - }) - } else { - debug!("tok: {}", pprust::token_to_string(&tok)); - } code_to_the_left = true; anything_to_the_left = true; } - (comments, literals) + comments } #[cfg(test)] diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index b988cb1447d..8c0c2f4b6e2 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -2121,11 +2121,11 @@ fn parse_lit_token(&mut self) -> PResult<'a, (LitKind, token::Lit, Option (ast::LitKind::Float(val, ast::FloatTy::F32), token::Float(val), suffix), - "f64" => (ast::LitKind::Float(val, ast::FloatTy::F64), token::Float(val), suffix), - _ => (ast::LitKind::FloatUnsuffixed(val), token::Float(val), suffix), - }); + return Ok((match float_suffix { + "f32" => ast::LitKind::Float(val, ast::FloatTy::F32), + "f64" => ast::LitKind::Float(val, ast::FloatTy::F64), + _ => ast::LitKind::FloatUnsuffixed(val), + }, token::Float(val), suffix)); } else { unreachable!(); }; diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 0e93b857708..fa16a2b200f 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -20,10 +20,8 @@ use syntax_pos::{self, BytePos}; use syntax_pos::{DUMMY_SP, FileName}; -use std::ascii; use std::borrow::Cow; use std::io::{self, Write, Read}; -use std::iter::Peekable; use std::vec; pub enum AnnNode<'a> { @@ -49,8 +47,7 @@ impl PpAnn for NoAnn {} pub struct State<'a> { pub s: pp::Printer<'a>, cm: Option<&'a SourceMap>, - comments: Option >, - literals: Peekable>, + comments: Option>, cur_cmnt: usize, boxes: Vec, ann: &'a (dyn PpAnn+'a), @@ -62,7 +59,6 @@ fn rust_printer<'a>(writer: Box, ann: &'a dyn PpAnn) -> State<'a> s: pp::mk_printer(writer, DEFAULT_COLUMNS), cm: None, comments: None, - literals: vec![].into_iter().peekable(), cur_cmnt: 0, boxes: Vec::new(), ann, @@ -75,8 +71,7 @@ fn rust_printer<'a>(writer: Box, ann: &'a dyn PpAnn) -> State<'a> pub const DEFAULT_COLUMNS: usize = 78; /// Requires you to pass an input filename and reader so that -/// it can scan the input text for comments and literals to -/// copy forward. +/// it can scan the input text for comments to copy forward. pub fn print_crate<'a>(cm: &'a SourceMap, sess: &ParseSess, krate: &ast::Crate, @@ -118,36 +113,23 @@ pub fn new_from_input(cm: &'a SourceMap, out: Box, ann: &'a dyn PpAnn, is_expanded: bool) -> State<'a> { - let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input); - - State::new( - cm, - out, - ann, - Some(cmnts), - // If the code is post expansion, don't use the table of - // literals, since it doesn't correspond with the literals - // in the AST anymore. - if is_expanded { None } else { Some(lits) }, - is_expanded - ) + let comments = comments::gather_comments(sess, filename, input); + State::new(cm, out, ann, Some(comments), is_expanded) } pub fn new(cm: &'a SourceMap, out: Box, ann: &'a dyn PpAnn, comments: Option>, - literals: Option>, is_expanded: bool) -> State<'a> { State { s: pp::mk_printer(out, DEFAULT_COLUMNS), cm: Some(cm), comments, - literals: literals.unwrap_or_default().into_iter().peekable(), cur_cmnt: 0, boxes: Vec::new(), ann, - is_expanded: is_expanded + is_expanded, } } } @@ -180,6 +162,31 @@ fn binop_to_string(op: BinOpToken) -> &'static str { } } +fn literal_to_string(lit: token::Lit, suffix: Option) -> String { + let mut out = match lit { + token::Byte(b) => format!("b'{}'", b), + token::Char(c) => format!("'{}'", c), + token::Err(c) => format!("'{}'", c), + token::Bool(c) | + token::Float(c) | + token::Integer(c) => c.to_string(), + token::Str_(s) => format!("\"{}\"", s), + token::StrRaw(s, n) => format!("r{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=s), + token::ByteStr(v) => format!("b\"{}\"", v), + token::ByteStrRaw(s, n) => format!("br{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=s), + }; + + if let Some(suffix) = suffix { + out.push_str(&suffix.as_str()) + } + + out +} + pub fn token_to_string(tok: &Token) -> String { match *tok { token::Eq => "=".to_string(), @@ -223,30 +230,7 @@ pub fn token_to_string(tok: &Token) -> String { token::SingleQuote => "'".to_string(), /* Literals */ - token::Literal(lit, suf) => { - let mut out = match lit { - token::Bool(_) => panic!("literal token contains `Lit::Bool`"), - token::Byte(b) => format!("b'{}'", b), - token::Char(c) => format!("'{}'", c), - token::Err(c) => format!("'{}'", c), - token::Float(c) | - token::Integer(c) => c.to_string(), - token::Str_(s) => format!("\"{}\"", s), - token::StrRaw(s, n) => format!("r{delim}\"{string}\"{delim}", - delim="#".repeat(n as usize), - string=s), - token::ByteStr(v) => format!("b\"{}\"", v), - token::ByteStrRaw(s, n) => format!("br{delim}\"{string}\"{delim}", - delim="#".repeat(n as usize), - string=s), - }; - - if let Some(s) = suf { - out.push_str(&s.as_str()) - } - - out - } + token::Literal(lit, suf) => literal_to_string(lit, suf), /* Name components */ token::Ident(s, false) => s.to_string(), @@ -439,8 +423,6 @@ pub trait PrintState<'a> { fn boxes(&mut self) -> &mut Vec; fn comments(&mut self) -> &mut Option>; fn cur_cmnt(&mut self) -> &mut usize; - fn cur_lit(&mut self) -> Option<&comments::Literal>; - fn bump_lit(&mut self) -> Option; fn word_space>>(&mut self, w: S) -> io::Result<()> { self.writer().word(w)?; @@ -505,21 +487,6 @@ fn commasep(&mut self, b: Breaks, elts: &[T], mut op: F) -> io::Result<()> self.end() } - fn next_lit(&mut self, pos: BytePos) -> Option { - while let Some(ltrl) = self.cur_lit().cloned() { - if ltrl.pos > pos { break; } - - // we don't need the value here since we're forced to clone cur_lit - // due to lack of NLL. - self.bump_lit(); - if ltrl.pos == pos { - return Some(ltrl); - } - } - - None - } - fn maybe_print_comment(&mut self, pos: BytePos) -> io::Result<()> { while let Some(ref cmnt) = self.next_comment() { if cmnt.pos < pos { @@ -607,60 +574,7 @@ fn next_comment(&mut self) -> Option { fn print_literal(&mut self, lit: &ast::Lit) -> io::Result<()> { self.maybe_print_comment(lit.span.lo())?; - if let Some(ltrl) = self.next_lit(lit.span.lo()) { - return self.writer().word(ltrl.lit.clone()); - } - match lit.node { - ast::LitKind::Str(st, style) => self.print_string(&st.as_str(), style), - ast::LitKind::Err(st) => { - let st = st.as_str().escape_debug().to_string(); - let mut res = String::with_capacity(st.len() + 2); - res.push('\''); - res.push_str(&st); - res.push('\''); - self.writer().word(res) - } - ast::LitKind::Byte(byte) => { - let mut res = String::from("b'"); - res.extend(ascii::escape_default(byte).map(|c| c as char)); - res.push('\''); - self.writer().word(res) - } - ast::LitKind::Char(ch) => { - let mut res = String::from("'"); - res.extend(ch.escape_default()); - res.push('\''); - self.writer().word(res) - } - ast::LitKind::Int(i, t) => { - match t { - ast::LitIntType::Signed(st) => { - self.writer().word(st.val_to_string(i as i128)) - } - ast::LitIntType::Unsigned(ut) => { - self.writer().word(ut.val_to_string(i)) - } - ast::LitIntType::Unsuffixed => { - self.writer().word(i.to_string()) - } - } - } - ast::LitKind::Float(ref f, t) => { - self.writer().word(format!("{}{}", &f, t.ty_to_string())) - } - ast::LitKind::FloatUnsuffixed(ref f) => self.writer().word(f.as_str().to_string()), - ast::LitKind::Bool(val) => { - if val { self.writer().word("true") } else { self.writer().word("false") } - } - ast::LitKind::ByteStr(ref v) => { - let mut escaped: String = String::new(); - for &ch in v.iter() { - escaped.extend(ascii::escape_default(ch) - .map(|c| c as char)); - } - self.writer().word(format!("b\"{}\"", escaped)) - } - } + self.writer().word(literal_to_string(lit.token, lit.suffix)) } fn print_string(&mut self, st: &str, @@ -881,14 +795,6 @@ fn comments(&mut self) -> &mut Option> { fn cur_cmnt(&mut self) -> &mut usize { &mut self.cur_cmnt } - - fn cur_lit(&mut self) -> Option<&comments::Literal> { - self.literals.peek() - } - - fn bump_lit(&mut self) -> Option { - self.literals.next() - } } impl<'a> State<'a> { -- 2.44.0