1 // Copyright 2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 #![feature(plugin, rustc_private, str_char, collections)]
19 use std::collections::HashMap;
22 use std::io::{BufRead, Read};
26 use syntax::parse::lexer;
27 use rustc::session::{self, config};
30 use syntax::ast::Name;
32 use syntax::codemap::Pos;
33 use syntax::parse::token;
34 use syntax::parse::lexer::TokenAndSpan;
36 fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
37 fn id() -> token::Token {
38 token::Ident(ast::Ident { name: Name(0), ctxt: 0, }, token::Plain)
41 let mut res = HashMap::new();
43 res.insert("-1".to_string(), token::Eof);
45 for line in file.split('\n') {
46 let eq = match line.trim().rfind('=') {
51 let val = &line[..eq];
52 let num = &line[eq + 1..];
55 "SHR" => token::BinOp(token::Shr),
56 "DOLLAR" => token::Dollar,
58 "STAR" => token::BinOp(token::Star),
59 "FLOAT_SUFFIX" => id(),
61 "SHL" => token::BinOp(token::Shl),
62 "LBRACE" => token::OpenDelim(token::Brace),
63 "RARROW" => token::RArrow,
64 "LIT_STR" => token::Literal(token::Str_(Name(0)), None),
65 "DOTDOT" => token::DotDot,
66 "MOD_SEP" => token::ModSep,
67 "DOTDOTDOT" => token::DotDotDot,
69 "AND" => token::BinOp(token::And),
70 "LPAREN" => token::OpenDelim(token::Paren),
71 "ANDAND" => token::AndAnd,
73 "LBRACKET" => token::OpenDelim(token::Bracket),
74 "LIT_STR_RAW" => token::Literal(token::StrRaw(Name(0), 0), None),
75 "RPAREN" => token::CloseDelim(token::Paren),
76 "SLASH" => token::BinOp(token::Slash),
77 "COMMA" => token::Comma,
78 "LIFETIME" => token::Lifetime(ast::Ident { name: Name(0), ctxt: 0 }),
79 "CARET" => token::BinOp(token::Caret),
80 "TILDE" => token::Tilde,
82 "PLUS" => token::BinOp(token::Plus),
83 "LIT_CHAR" => token::Literal(token::Char(Name(0)), None),
84 "LIT_BYTE" => token::Literal(token::Byte(Name(0)), None),
86 "RBRACKET" => token::CloseDelim(token::Bracket),
87 "COMMENT" => token::Comment,
88 "DOC_COMMENT" => token::DocComment(Name(0)),
90 "EQEQ" => token::EqEq,
93 "PERCENT" => token::BinOp(token::Percent),
94 "RBRACE" => token::CloseDelim(token::Brace),
95 "BINOP" => token::BinOp(token::Plus),
96 "POUND" => token::Pound,
97 "OROR" => token::OrOr,
98 "LIT_INTEGER" => token::Literal(token::Integer(Name(0)), None),
99 "BINOPEQ" => token::BinOpEq(token::Plus),
100 "LIT_FLOAT" => token::Literal(token::Float(Name(0)), None),
101 "WHITESPACE" => token::Whitespace,
102 "UNDERSCORE" => token::Underscore,
103 "MINUS" => token::BinOp(token::Minus),
104 "SEMI" => token::Semi,
105 "COLON" => token::Colon,
106 "FAT_ARROW" => token::FatArrow,
107 "OR" => token::BinOp(token::Or),
110 "LIT_BINARY" => token::Literal(token::Binary(Name(0)), None),
111 "LIT_BINARY_RAW" => token::Literal(token::BinaryRaw(Name(0), 0), None),
112 "QUESTION" => token::Question,
113 "SHEBANG" => token::Shebang(Name(0)),
117 res.insert(num.to_string(), tok);
120 debug!("Token map: {:?}", res);
124 fn str_to_binop(s: &str) -> token::BinOpToken {
130 "%" => token::Percent,
136 _ => panic!("Bad binop str `{}`", s),
140 /// Assuming a string/binary literal, strip out the leading/trailing
141 /// hashes and surrounding quotes/raw/binary prefix.
142 fn fix(mut lit: &str) -> ast::Name {
143 if lit.char_at(0) == 'r' {
144 if lit.char_at(1) == 'b' {
149 } else if lit.char_at(0) == 'b' {
153 let leading_hashes = count(lit);
155 // +1/-1 to adjust for single quotes
156 parse::token::intern(&lit[leading_hashes + 1..lit.len() - leading_hashes - 1])
159 /// Assuming a char/byte literal, strip the 'b' prefix and the single quotes.
160 fn fixchar(mut lit: &str) -> ast::Name {
161 if lit.char_at(0) == 'b' {
165 parse::token::intern(&lit[1..lit.len() - 1])
168 fn count(lit: &str) -> usize {
169 lit.chars().take_while(|c| *c == '#').count()
172 fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>, surrogate_pairs_pos: &[usize],
176 // \[@(?P<seq>\d+),(?P<start>\d+):(?P<end>\d+)='(?P<content>.+?)',<(?P<toknum>-?\d+)>,\d+:\d+]
177 let start = s.find("[@").unwrap();
178 let comma = start + s[start..].find(",").unwrap();
179 let colon = comma + s[comma..].find(":").unwrap();
180 let content_start = colon + s[colon..].find("='").unwrap();
181 // Use rfind instead of find, because we don't want to stop at the content
182 let content_end = content_start + s[content_start..].rfind("',<").unwrap();
183 let toknum_end = content_end + s[content_end..].find(">,").unwrap();
185 let start = &s[comma + 1 .. colon];
186 let end = &s[colon + 1 .. content_start];
187 let content = &s[content_start + 2 .. content_end];
188 let toknum = &s[content_end + 3 .. toknum_end];
190 let not_found = format!("didn't find token {:?} in the map", toknum);
191 let proto_tok = tokens.get(toknum).expect(¬_found[..]);
193 let nm = parse::token::intern(content);
195 debug!("What we got: content (`{}`), proto: {:?}", content, proto_tok);
197 let real_tok = match *proto_tok {
198 token::BinOp(..) => token::BinOp(str_to_binop(content)),
199 token::BinOpEq(..) => token::BinOpEq(str_to_binop(&content[..content.len() - 1])),
200 token::Literal(token::Str_(..), n) => token::Literal(token::Str_(fix(content)), n),
201 token::Literal(token::StrRaw(..), n) => token::Literal(token::StrRaw(fix(content),
203 token::Literal(token::Char(..), n) => token::Literal(token::Char(fixchar(content)), n),
204 token::Literal(token::Byte(..), n) => token::Literal(token::Byte(fixchar(content)), n),
205 token::DocComment(..) => token::DocComment(nm),
206 token::Literal(token::Integer(..), n) => token::Literal(token::Integer(nm), n),
207 token::Literal(token::Float(..), n) => token::Literal(token::Float(nm), n),
208 token::Literal(token::Binary(..), n) => token::Literal(token::Binary(nm), n),
209 token::Literal(token::BinaryRaw(..), n) => token::Literal(token::BinaryRaw(fix(content),
211 token::Ident(..) => token::Ident(ast::Ident { name: nm, ctxt: 0 },
213 token::Lifetime(..) => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }),
217 let start_offset = if real_tok == token::Eof {
223 let offset = if has_bom { 1 } else { 0 };
225 let mut lo = start.parse::<u32>().unwrap() - start_offset - offset;
226 let mut hi = end.parse::<u32>().unwrap() + 1 - offset;
228 // Adjust the span: For each surrogate pair already encountered, subtract one position.
229 lo -= surrogate_pairs_pos.binary_search(&(lo as usize)).unwrap_or_else(|x| x) as u32;
230 hi -= surrogate_pairs_pos.binary_search(&(hi as usize)).unwrap_or_else(|x| x) as u32;
232 let sp = codemap::Span {
233 lo: codemap::BytePos(lo),
234 hi: codemap::BytePos(hi),
235 expn_id: codemap::NO_EXPANSION
244 fn tok_cmp(a: &token::Token, b: &token::Token) -> bool {
246 &token::Ident(id, _) => match b {
247 &token::Ident(id2, _) => id == id2,
254 fn span_cmp(antlr_sp: codemap::Span, rust_sp: codemap::Span, cm: &codemap::CodeMap) -> bool {
255 antlr_sp.expn_id == rust_sp.expn_id &&
256 antlr_sp.lo.to_usize() == cm.bytepos_to_file_charpos(rust_sp.lo).to_usize() &&
257 antlr_sp.hi.to_usize() == cm.bytepos_to_file_charpos(rust_sp.hi).to_usize()
261 fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
262 use syntax::parse::lexer::Reader;
266 let mut args = env::args().skip(1);
267 let filename = args.next().unwrap();
268 if filename.find("parse-fail").is_some() {
273 let mut code = String::new();
274 File::open(&Path::new(&filename)).unwrap().read_to_string(&mut code).unwrap();
276 let surrogate_pairs_pos: Vec<usize> = code.chars().enumerate()
277 .filter(|&(_, c)| c as usize > 0xFFFF)
283 let has_bom = code.starts_with("\u{feff}");
285 debug!("Pairs: {:?}", surrogate_pairs_pos);
287 let options = config::basic_options();
288 let session = session::build_session(options, None,
289 syntax::diagnostics::registry::Registry::new(&[]));
290 let filemap = session.parse_sess.codemap().new_filemap(String::from_str("<n/a>"), code);
291 let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);
292 let cm = session.codemap();
295 let mut token_file = File::open(&Path::new(&args.next().unwrap())).unwrap();
296 let mut token_list = String::new();
297 token_file.read_to_string(&mut token_list).unwrap();
298 let token_map = parse_token_list(&token_list[..]);
300 let stdin = std::io::stdin();
301 let lock = stdin.lock();
302 let lines = lock.lines();
303 let antlr_tokens = lines.map(|l| parse_antlr_token(l.unwrap().trim(),
305 &surrogate_pairs_pos[..],
308 for antlr_tok in antlr_tokens {
309 let rustc_tok = next(&mut lexer);
310 if rustc_tok.tok == token::Eof && antlr_tok.tok == token::Eof {
314 assert!(span_cmp(antlr_tok.sp, rustc_tok.sp, cm), "{:?} and {:?} have different spans",
318 macro_rules! matches {
320 match rustc_tok.tok {
321 $($x => match antlr_tok.tok {
323 if !tok_cmp(&rustc_tok.tok, &antlr_tok.tok) {
324 // FIXME #15677: needs more robust escaping in
326 warn!("Different names for {:?} and {:?}", rustc_tok, antlr_tok);
329 _ => panic!("{:?} is not {:?}", antlr_tok, rustc_tok)
331 ref c => assert!(c == &antlr_tok.tok, "{:?} is not {:?}", antlr_tok, rustc_tok)
337 token::Literal(token::Byte(..), _),
338 token::Literal(token::Char(..), _),
339 token::Literal(token::Integer(..), _),
340 token::Literal(token::Float(..), _),
341 token::Literal(token::Str_(..), _),
342 token::Literal(token::StrRaw(..), _),
343 token::Literal(token::Binary(..), _),
344 token::Literal(token::BinaryRaw(..), _),
347 token::Interpolated(..),
348 token::DocComment(..),