1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 // ignore-lexer-test FIXME #15679
13 //! This is an Earley-like parser, without support for in-grammar nonterminals,
14 //! only by calling out to the main rust parser for named nonterminals (which it
15 //! commits to fully when it hits one in a grammar). This means that there are no
16 //! completer or predictor rules, and therefore no need to store one column per
17 //! token: instead, there's a set of current Earley items and a set of next
18 //! ones. Instead of NTs, we have a special case for Kleene star. The big-O, in
19 //! pathological cases, is worse than traditional Earley parsing, but it's an
20 //! easier fit for Macro-by-Example-style rules, and I think the overhead is
21 //! lower. (In order to prevent the pathological case, we'd need to lazily
22 //! construct the resulting `NamedMatch`es at the very end. It'd be a pain,
23 //! and require more memory to keep around old items, but it would also save
26 //! Quick intro to how the parser works:
28 //! A 'position' is a dot in the middle of a matcher, usually represented as a
29 //! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`.
31 //! The parser walks through the input a character at a time, maintaining a list
32 //! of items consistent with the current position in the input string: `cur_eis`.
34 //! As it processes them, it fills up `eof_eis` with items that would be valid if
35 //! the macro invocation is now over, `bb_eis` with items that are waiting on
36 //! a Rust nonterminal like `$e:expr`, and `next_eis` with items that are waiting
37 //! on the a particular token. Most of the logic concerns moving the · through the
38 //! repetitions indicated by Kleene stars. It only advances or calls out to the
39 //! real Rust parser when no `cur_eis` items remain
41 //! Example: Start parsing `a a a a b` against [· a $( a )* a b].
43 //! Remaining input: `a a a a b`
44 //! next_eis: [· a $( a )* a b]
46 //! - - - Advance over an `a`. - - -
48 //! Remaining input: `a a a b`
49 //! cur: [a · $( a )* a b]
50 //! Descend/Skip (first item).
51 //! next: [a $( · a )* a b] [a $( a )* · a b].
53 //! - - - Advance over an `a`. - - -
55 //! Remaining input: `a a b`
56 //! cur: [a $( a · )* a b] next: [a $( a )* a · b]
57 //! Finish/Repeat (first item)
58 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
60 //! - - - Advance over an `a`. - - - (this looks exactly like the last step)
62 //! Remaining input: `a b`
63 //! cur: [a $( a · )* a b] next: [a $( a )* a · b]
64 //! Finish/Repeat (first item)
65 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
67 //! - - - Advance over an `a`. - - - (this looks exactly like the last step)
69 //! Remaining input: `b`
70 //! cur: [a $( a · )* a b] next: [a $( a )* a · b]
71 //! Finish/Repeat (first item)
72 //! next: [a $( a )* · a b] [a $( · a )* a b]
74 //! - - - Advance over a `b`. - - -
76 //! Remaining input: ``
77 //! eof: [a $( a )* a b ·]
79 pub use self::NamedMatch::*;
80 pub use self::ParseResult::*;
81 use self::TokenTreeOrTokenTreeVec::*;
84 use ast::{TokenTree, Ident};
85 use ast::{TtDelimited, TtSequence, TtToken};
86 use codemap::{BytePos, mk_sp};
88 use parse::lexer::*; //resolve bug?
90 use parse::attr::ParserAttr;
91 use parse::parser::{LifetimeAndTypesWithoutColons, Parser};
92 use parse::token::{Eof, DocComment, MatchNt, SubstNt};
93 use parse::token::{Token, Nonterminal};
100 use std::collections::HashMap;
101 use std::collections::hash_map::Entry::{Vacant, Occupied};
103 // To avoid costly uniqueness checks, we require that `MatchSeq` always has
107 enum TokenTreeOrTokenTreeVec {
109 TtSeq(Rc<Vec<ast::TokenTree>>),
112 impl TokenTreeOrTokenTreeVec {
113 fn len(&self) -> uint {
115 &TtSeq(ref v) => v.len(),
116 &Tt(ref tt) => tt.len(),
120 fn get_tt(&self, index: uint) -> TokenTree {
122 &TtSeq(ref v) => v[index].clone(),
123 &Tt(ref tt) => tt.get_tt(index),
128 /// an unzipping of `TokenTree`s
130 struct MatcherTtFrame {
131 elts: TokenTreeOrTokenTreeVec,
136 pub struct MatcherPos {
137 stack: Vec<MatcherTtFrame>,
138 top_elts: TokenTreeOrTokenTreeVec,
141 up: Option<Box<MatcherPos>>,
142 matches: Vec<Vec<Rc<NamedMatch>>>,
149 pub fn count_names(ms: &[TokenTree]) -> uint {
150 ms.iter().fold(0, |count, elt| {
152 &TtSequence(_, ref seq) => {
155 &TtDelimited(_, ref delim) => {
156 count_names(delim.tts[])
158 &TtToken(_, MatchNt(..)) => {
166 pub fn initial_matcher_pos(ms: Rc<Vec<TokenTree>>, sep: Option<Token>, lo: BytePos)
168 let match_idx_hi = count_names(ms[]);
169 let matches: Vec<_> = range(0, match_idx_hi).map(|_| Vec::new()).collect();
179 match_hi: match_idx_hi,
184 /// NamedMatch is a pattern-match result for a single token::MATCH_NONTERMINAL:
185 /// so it is associated with a single ident in a parse, and all
186 /// `MatchedNonterminal`s in the NamedMatch have the same nonterminal type
187 /// (expr, item, etc). Each leaf in a single NamedMatch corresponds to a
188 /// single token::MATCH_NONTERMINAL in the TokenTree that produced it.
190 /// The in-memory structure of a particular NamedMatch represents the match
191 /// that occurred when a particular subset of a matcher was applied to a
192 /// particular token tree.
194 /// The width of each MatchedSeq in the NamedMatch, and the identity of the
195 /// `MatchedNonterminal`s, will depend on the token tree it was applied to:
196 /// each MatchedSeq corresponds to a single TTSeq in the originating
197 /// token tree. The depth of the NamedMatch structure will therefore depend
198 /// only on the nesting depth of `ast::TTSeq`s in the originating
199 /// token tree it was derived from.
201 pub enum NamedMatch {
202 MatchedSeq(Vec<Rc<NamedMatch>>, codemap::Span),
203 MatchedNonterminal(Nonterminal)
206 pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc<NamedMatch>])
207 -> HashMap<Ident, Rc<NamedMatch>> {
208 fn n_rec(p_s: &ParseSess, m: &TokenTree, res: &[Rc<NamedMatch>],
209 ret_val: &mut HashMap<Ident, Rc<NamedMatch>>, idx: &mut uint) {
211 &TtSequence(_, ref seq) => {
212 for next_m in seq.tts.iter() {
213 n_rec(p_s, next_m, res, ret_val, idx)
216 &TtDelimited(_, ref delim) => {
217 for next_m in delim.tts.iter() {
218 n_rec(p_s, next_m, res, ret_val, idx)
221 &TtToken(sp, MatchNt(bind_name, _, _, _)) => {
222 match ret_val.entry(bind_name) {
224 spot.set(res[*idx].clone());
228 let string = token::get_ident(bind_name);
231 format!("duplicated bind name: {}",
236 &TtToken(_, SubstNt(..)) => panic!("Cannot fill in a NT"),
237 &TtToken(_, _) => (),
240 let mut ret_val = HashMap::new();
242 for m in ms.iter() { n_rec(p_s, m, res, &mut ret_val, &mut idx) }
246 pub enum ParseResult {
247 Success(HashMap<Ident, Rc<NamedMatch>>),
248 Failure(codemap::Span, String),
249 Error(codemap::Span, String)
252 pub fn parse_or_else(sess: &ParseSess,
253 cfg: ast::CrateConfig,
256 -> HashMap<Ident, Rc<NamedMatch>> {
257 match parse(sess, cfg, rdr, ms[]) {
259 Failure(sp, str) => {
260 sess.span_diagnostic.span_fatal(sp, str[])
263 sess.span_diagnostic.span_fatal(sp, str[])
268 /// Perform a token equality check, ignoring syntax context (that is, an
269 /// unhygienic comparison)
270 pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool {
272 (&token::Ident(id1,_),&token::Ident(id2,_))
273 | (&token::Lifetime(id1),&token::Lifetime(id2)) =>
274 id1.name == id2.name,
279 pub fn parse(sess: &ParseSess,
280 cfg: ast::CrateConfig,
284 let mut cur_eis = Vec::new();
285 cur_eis.push(initial_matcher_pos(Rc::new(ms.iter()
286 .map(|x| (*x).clone())
292 let mut bb_eis = Vec::new(); // black-box parsed by parser.rs
293 let mut next_eis = Vec::new(); // or proceed normally
294 let mut eof_eis = Vec::new();
296 let TokenAndSpan { tok, sp } = rdr.peek();
298 /* we append new items to this while we go */
300 let mut ei = match cur_eis.pop() {
301 None => break, /* for each Earley Item */
305 // When unzipped trees end, remove them
306 while ei.idx >= ei.top_elts.len() {
307 match ei.stack.pop() {
308 Some(MatcherTtFrame { elts, idx }) => {
317 let len = ei.top_elts.len();
319 /* at end of sequence */
321 // can't move out of `match`es, so:
323 // hack: a matcher sequence is repeating iff it has a
324 // parent (the top level is just a container)
327 // disregard separator, try to go up
328 // (remove this condition to make trailing seps ok)
330 // pop from the matcher position
332 let mut new_pos = ei.up.clone().unwrap();
334 // update matches (the MBE "parse tree") by appending
335 // each tree as a subtree.
337 // I bet this is a perf problem: we're preemptively
338 // doing a lot of array work that will get thrown away
341 // Only touch the binders we have actually bound
342 for idx in range(ei.match_lo, ei.match_hi) {
343 let sub = (ei.matches[idx]).clone();
345 .push(Rc::new(MatchedSeq(sub, mk_sp(ei.sp_lo,
349 new_pos.match_cur = ei.match_hi;
351 cur_eis.push(new_pos);
354 // can we go around again?
356 // the *_t vars are workarounds for the lack of unary move
358 Some(ref t) if idx == len => { // we need a separator
359 // i'm conflicted about whether this should be hygienic....
360 // though in this case, if the separators are never legal
361 // idents, it shouldn't matter.
362 if token_name_eq(&tok, t) { //pass the separator
363 let mut ei_t = ei.clone();
364 // ei_t.match_cur = ei_t.match_lo;
369 _ => { // we don't need a separator
371 ei_t.match_cur = ei_t.match_lo;
380 match ei.top_elts.get_tt(idx) {
381 /* need to descend into sequence */
382 TtSequence(sp, seq) => {
383 if seq.op == ast::ZeroOrMore {
384 let mut new_ei = ei.clone();
385 new_ei.match_cur += seq.num_captures;
387 //we specifically matched zero repeats.
388 for idx in range(ei.match_cur, ei.match_cur + seq.num_captures) {
389 new_ei.matches[idx].push(Rc::new(MatchedSeq(Vec::new(), sp)));
392 cur_eis.push(new_ei);
395 let matches: Vec<_> = range(0, ei.matches.len())
396 .map(|_| Vec::new()).collect();
398 cur_eis.push(box MatcherPos {
400 sep: seq.separator.clone(),
403 match_lo: ei_t.match_cur,
404 match_cur: ei_t.match_cur,
405 match_hi: ei_t.match_cur + seq.num_captures,
408 top_elts: Tt(TtSequence(sp, seq)),
411 TtToken(_, MatchNt(..)) => {
412 // Built-in nonterminals never start with these tokens,
413 // so we can eliminate them from consideration.
415 token::CloseDelim(_) => {},
416 _ => bb_eis.push(ei),
419 TtToken(sp, SubstNt(..)) => {
420 return Error(sp, "Cannot transcribe in macro LHS".to_string())
422 seq @ TtDelimited(..) | seq @ TtToken(_, DocComment(..)) => {
423 let lower_elts = mem::replace(&mut ei.top_elts, Tt(seq));
425 ei.stack.push(MatcherTtFrame {
432 TtToken(_, ref t) => {
433 let mut ei_t = ei.clone();
434 if token_name_eq(t,&tok) {
443 /* error messages here could be improved with links to orig. rules */
444 if token_name_eq(&tok, &token::Eof) {
445 if eof_eis.len() == 1u {
446 let mut v = Vec::new();
447 for dv in eof_eis[0].matches.iter_mut() {
448 v.push(dv.pop().unwrap());
450 return Success(nameize(sess, ms, v[]));
451 } else if eof_eis.len() > 1u {
452 return Error(sp, "ambiguity: multiple successful parses".to_string());
454 return Failure(sp, "unexpected end of macro invocation".to_string());
457 if (bb_eis.len() > 0u && next_eis.len() > 0u)
458 || bb_eis.len() > 1u {
459 let nts = bb_eis.iter().map(|ei| {
460 match ei.top_elts.get_tt(ei.idx) {
461 TtToken(_, MatchNt(bind, name, _, _)) => {
462 (format!("{} ('{}')",
463 token::get_ident(name),
464 token::get_ident(bind))).to_string()
467 } }).collect::<Vec<String>>().connect(" or ");
468 return Error(sp, format!(
469 "local ambiguity: multiple parsing options: \
470 built-in NTs {} or {} other options.",
471 nts, next_eis.len()).to_string());
472 } else if bb_eis.len() == 0u && next_eis.len() == 0u {
473 return Failure(sp, format!("no rules expected the token `{}`",
474 pprust::token_to_string(&tok)).to_string());
475 } else if next_eis.len() > 0u {
476 /* Now process the next token */
477 while next_eis.len() > 0u {
478 cur_eis.push(next_eis.pop().unwrap());
481 } else /* bb_eis.len() == 1 */ {
482 let mut rust_parser = Parser::new(sess, cfg.clone(), box rdr.clone());
484 let mut ei = bb_eis.pop().unwrap();
485 match ei.top_elts.get_tt(ei.idx) {
486 TtToken(_, MatchNt(_, name, _, _)) => {
487 let name_string = token::get_ident(name);
488 let match_cur = ei.match_cur;
489 ei.matches[match_cur].push(Rc::new(MatchedNonterminal(
490 parse_nt(&mut rust_parser, name_string.get()))));
498 for _ in range(0, rust_parser.tokens_consumed) {
499 let _ = rdr.next_token();
504 assert!(cur_eis.len() > 0u);
508 pub fn parse_nt(p: &mut Parser, name: &str) -> Nonterminal {
510 "item" => match p.parse_item(Vec::new()) {
511 Some(i) => token::NtItem(i),
512 None => p.fatal("expected an item keyword")
514 "block" => token::NtBlock(p.parse_block()),
515 "stmt" => token::NtStmt(p.parse_stmt(Vec::new())),
516 "pat" => token::NtPat(p.parse_pat()),
517 "expr" => token::NtExpr(p.parse_expr()),
518 "ty" => token::NtTy(p.parse_ty()),
519 // this could be handled like a token, since it is one
520 "ident" => match p.token {
521 token::Ident(sn,b) => { p.bump(); token::NtIdent(box sn,b) }
523 let token_str = pprust::token_to_string(&p.token);
524 p.fatal((format!("expected ident, found {}",
529 token::NtPath(box p.parse_path(LifetimeAndTypesWithoutColons))
531 "meta" => token::NtMeta(p.parse_meta_item()),
533 p.quote_depth += 1u; //but in theory, non-quoted tts might be useful
534 let res = token::NtTT(P(p.parse_token_tree()));
539 p.fatal(format!("unsupported builtin nonterminal parser: {}", name)[])