1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 // ignore-lexer-test FIXME #15679
13 //! This is an Earley-like parser, without support for in-grammar nonterminals,
14 //! only by calling out to the main rust parser for named nonterminals (which it
15 //! commits to fully when it hits one in a grammar). This means that there are no
16 //! completer or predictor rules, and therefore no need to store one column per
17 //! token: instead, there's a set of current Earley items and a set of next
18 //! ones. Instead of NTs, we have a special case for Kleene star. The big-O, in
19 //! pathological cases, is worse than traditional Earley parsing, but it's an
20 //! easier fit for Macro-by-Example-style rules, and I think the overhead is
21 //! lower. (In order to prevent the pathological case, we'd need to lazily
22 //! construct the resulting `NamedMatch`es at the very end. It'd be a pain,
23 //! and require more memory to keep around old items, but it would also save
26 //! Quick intro to how the parser works:
28 //! A 'position' is a dot in the middle of a matcher, usually represented as a
29 //! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`.
31 //! The parser walks through the input a character at a time, maintaining a list
32 //! of items consistent with the current position in the input string: `cur_eis`.
34 //! As it processes them, it fills up `eof_eis` with items that would be valid if
35 //! the macro invocation is now over, `bb_eis` with items that are waiting on
36 //! a Rust nonterminal like `$e:expr`, and `next_eis` with items that are waiting
37 //! on the a particular token. Most of the logic concerns moving the · through the
38 //! repetitions indicated by Kleene stars. It only advances or calls out to the
39 //! real Rust parser when no `cur_eis` items remain
41 //! Example: Start parsing `a a a a b` against [· a $( a )* a b].
43 //! Remaining input: `a a a a b`
44 //! next_eis: [· a $( a )* a b]
46 //! - - - Advance over an `a`. - - -
48 //! Remaining input: `a a a b`
49 //! cur: [a · $( a )* a b]
50 //! Descend/Skip (first item).
51 //! next: [a $( · a )* a b] [a $( a )* · a b].
53 //! - - - Advance over an `a`. - - -
55 //! Remaining input: `a a b`
56 //! cur: [a $( a · )* a b] next: [a $( a )* a · b]
57 //! Finish/Repeat (first item)
58 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
60 //! - - - Advance over an `a`. - - - (this looks exactly like the last step)
62 //! Remaining input: `a b`
63 //! cur: [a $( a · )* a b] next: [a $( a )* a · b]
64 //! Finish/Repeat (first item)
65 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
67 //! - - - Advance over an `a`. - - - (this looks exactly like the last step)
69 //! Remaining input: `b`
70 //! cur: [a $( a · )* a b] next: [a $( a )* a · b]
71 //! Finish/Repeat (first item)
72 //! next: [a $( a )* · a b] [a $( · a )* a b]
74 //! - - - Advance over a `b`. - - -
76 //! Remaining input: ``
77 //! eof: [a $( a )* a b ·]
81 use ast::{Matcher, MatchTok, MatchSeq, MatchNonterminal, Ident};
82 use codemap::{BytePos, mk_sp};
84 use parse::lexer::*; //resolve bug?
86 use parse::attr::ParserAttr;
87 use parse::parser::{LifetimeAndTypesWithoutColons, Parser};
88 use parse::token::{Token, EOF, Nonterminal};
93 use std::collections::HashMap;
95 /* to avoid costly uniqueness checks, we require that `MatchSeq` always has a
100 pub struct MatcherPos {
101 elts: Vec<ast::Matcher> , // maybe should be <'>? Need to understand regions.
104 up: Option<Box<MatcherPos>>,
105 matches: Vec<Vec<Rc<NamedMatch>>>,
106 match_lo: uint, match_hi: uint,
110 pub fn count_names(ms: &[Matcher]) -> uint {
111 ms.iter().fold(0, |ct, m| {
114 MatchSeq(ref more_ms, _, _, _, _) => {
115 count_names(more_ms.as_slice())
117 MatchNonterminal(_, _, _) => 1u
121 pub fn initial_matcher_pos(ms: Vec<Matcher> , sep: Option<Token>, lo: BytePos)
123 let mut match_idx_hi = 0u;
124 for elt in ms.iter() {
127 MatchSeq(_,_,_,_,hi) => {
128 match_idx_hi = hi; // it is monotonic...
130 MatchNonterminal(_,_,pos) => {
131 match_idx_hi = pos+1u; // ...so latest is highest
135 let matches = Vec::from_fn(count_names(ms.as_slice()), |_i| Vec::new());
143 match_hi: match_idx_hi,
148 /// NamedMatch is a pattern-match result for a single ast::MatchNonterminal:
149 /// so it is associated with a single ident in a parse, and all
150 /// MatchedNonterminal's in the NamedMatch have the same nonterminal type
151 /// (expr, item, etc). All the leaves in a single NamedMatch correspond to a
152 /// single matcher_nonterminal in the ast::Matcher that produced it.
154 /// It should probably be renamed, it has more or less exact correspondence to
155 /// ast::match nodes, and the in-memory structure of a particular NamedMatch
156 /// represents the match that occurred when a particular subset of an
157 /// ast::match -- those ast::Matcher nodes leading to a single
158 /// MatchNonterminal -- was applied to a particular token tree.
160 /// The width of each MatchedSeq in the NamedMatch, and the identity of the
161 /// MatchedNonterminal's, will depend on the token tree it was applied to: each
162 /// MatchedSeq corresponds to a single MatchSeq in the originating
163 /// ast::Matcher. The depth of the NamedMatch structure will therefore depend
164 /// only on the nesting depth of ast::MatchSeq's in the originating
165 /// ast::Matcher it was derived from.
167 pub enum NamedMatch {
168 MatchedSeq(Vec<Rc<NamedMatch>>, codemap::Span),
169 MatchedNonterminal(Nonterminal)
172 pub fn nameize(p_s: &ParseSess, ms: &[Matcher], res: &[Rc<NamedMatch>])
173 -> HashMap<Ident, Rc<NamedMatch>> {
174 fn n_rec(p_s: &ParseSess, m: &Matcher, res: &[Rc<NamedMatch>],
175 ret_val: &mut HashMap<Ident, Rc<NamedMatch>>) {
177 codemap::Spanned {node: MatchTok(_), .. } => (),
178 codemap::Spanned {node: MatchSeq(ref more_ms, _, _, _, _), .. } => {
179 for next_m in more_ms.iter() {
180 n_rec(p_s, next_m, res, ret_val)
184 node: MatchNonterminal(bind_name, _, idx),
187 if ret_val.contains_key(&bind_name) {
188 let string = token::get_ident(bind_name);
191 format!("duplicated bind name: {}",
192 string.get()).as_slice())
194 ret_val.insert(bind_name, res[idx].clone());
198 let mut ret_val = HashMap::new();
199 for m in ms.iter() { n_rec(p_s, m, res, &mut ret_val) }
203 pub enum ParseResult {
204 Success(HashMap<Ident, Rc<NamedMatch>>),
205 Failure(codemap::Span, String),
206 Error(codemap::Span, String)
209 pub fn parse_or_else(sess: &ParseSess,
210 cfg: ast::CrateConfig,
213 -> HashMap<Ident, Rc<NamedMatch>> {
214 match parse(sess, cfg, rdr, ms.as_slice()) {
216 Failure(sp, str) => {
217 sess.span_diagnostic.span_fatal(sp, str.as_slice())
220 sess.span_diagnostic.span_fatal(sp, str.as_slice())
225 /// Perform a token equality check, ignoring syntax context (that is, an
226 /// unhygienic comparison)
227 pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool {
229 (&token::IDENT(id1,_),&token::IDENT(id2,_))
230 | (&token::LIFETIME(id1),&token::LIFETIME(id2)) =>
231 id1.name == id2.name,
236 pub fn parse(sess: &ParseSess,
237 cfg: ast::CrateConfig,
241 let mut cur_eis = Vec::new();
242 cur_eis.push(initial_matcher_pos(ms.iter()
243 .map(|x| (*x).clone())
249 let mut bb_eis = Vec::new(); // black-box parsed by parser.rs
250 let mut next_eis = Vec::new(); // or proceed normally
251 let mut eof_eis = Vec::new();
253 let TokenAndSpan {tok: tok, sp: sp} = rdr.peek();
255 /* we append new items to this while we go */
257 let ei = match cur_eis.pop() {
258 None => break, /* for each Earley Item */
263 let len = ei.elts.len();
265 /* at end of sequence */
267 // can't move out of `match`es, so:
269 // hack: a matcher sequence is repeating iff it has a
270 // parent (the top level is just a container)
273 // disregard separator, try to go up
274 // (remove this condition to make trailing seps ok)
276 // pop from the matcher position
278 let mut new_pos = ei.up.clone().unwrap();
280 // update matches (the MBE "parse tree") by appending
281 // each tree as a subtree.
283 // I bet this is a perf problem: we're preemptively
284 // doing a lot of array work that will get thrown away
287 // Only touch the binders we have actually bound
288 for idx in range(ei.match_lo, ei.match_hi) {
289 let sub = (*ei.matches.get(idx)).clone();
292 .push(Rc::new(MatchedSeq(sub, mk_sp(ei.sp_lo,
297 cur_eis.push(new_pos);
300 // can we go around again?
302 // the *_t vars are workarounds for the lack of unary move
304 Some(ref t) if idx == len => { // we need a separator
305 // i'm conflicted about whether this should be hygienic....
306 // though in this case, if the separators are never legal
307 // idents, it shouldn't matter.
308 if token_name_eq(&tok, t) { //pass the separator
309 let mut ei_t = ei.clone();
314 _ => { // we don't need a separator
324 match ei.elts.get(idx).node.clone() {
325 /* need to descend into sequence */
326 MatchSeq(ref matchers, ref sep, zero_ok,
327 match_idx_lo, match_idx_hi) => {
329 let mut new_ei = ei.clone();
331 //we specifically matched zero repeats.
332 for idx in range(match_idx_lo, match_idx_hi) {
335 .push(Rc::new(MatchedSeq(Vec::new(), sp)));
338 cur_eis.push(new_ei);
341 let matches = Vec::from_elem(ei.matches.len(), Vec::new());
343 cur_eis.push(box MatcherPos {
344 elts: (*matchers).clone(),
349 match_lo: match_idx_lo, match_hi: match_idx_hi,
353 MatchNonterminal(_,_,_) => { bb_eis.push(ei) }
355 let mut ei_t = ei.clone();
356 if token_name_eq(t,&tok) {
365 /* error messages here could be improved with links to orig. rules */
366 if token_name_eq(&tok, &EOF) {
367 if eof_eis.len() == 1u {
368 let mut v = Vec::new();
369 for dv in eof_eis.get_mut(0).matches.mut_iter() {
370 v.push(dv.pop().unwrap());
372 return Success(nameize(sess, ms, v.as_slice()));
373 } else if eof_eis.len() > 1u {
374 return Error(sp, "ambiguity: multiple successful parses".to_string());
376 return Failure(sp, "unexpected end of macro invocation".to_string());
379 if (bb_eis.len() > 0u && next_eis.len() > 0u)
380 || bb_eis.len() > 1u {
381 let nts = bb_eis.iter().map(|ei| {
382 match ei.elts.get(ei.idx).node {
383 MatchNonterminal(bind, name, _) => {
384 (format!("{} ('{}')",
385 token::get_ident(name),
386 token::get_ident(bind))).to_string()
389 } }).collect::<Vec<String>>().connect(" or ");
390 return Error(sp, format!(
391 "local ambiguity: multiple parsing options: \
392 built-in NTs {} or {} other options.",
393 nts, next_eis.len()).to_string());
394 } else if bb_eis.len() == 0u && next_eis.len() == 0u {
395 return Failure(sp, format!("no rules expected the token `{}`",
396 token::to_string(&tok)).to_string());
397 } else if next_eis.len() > 0u {
398 /* Now process the next token */
399 while next_eis.len() > 0u {
400 cur_eis.push(next_eis.pop().unwrap());
403 } else /* bb_eis.len() == 1 */ {
404 let mut rust_parser = Parser::new(sess, cfg.clone(), box rdr.clone());
406 let mut ei = bb_eis.pop().unwrap();
407 match ei.elts.get(ei.idx).node {
408 MatchNonterminal(_, name, idx) => {
409 let name_string = token::get_ident(name);
410 ei.matches.get_mut(idx).push(Rc::new(MatchedNonterminal(
411 parse_nt(&mut rust_parser, name_string.get()))));
418 for _ in range(0, rust_parser.tokens_consumed) {
419 let _ = rdr.next_token();
424 assert!(cur_eis.len() > 0u);
428 pub fn parse_nt(p: &mut Parser, name: &str) -> Nonterminal {
430 "item" => match p.parse_item(Vec::new()) {
431 Some(i) => token::NtItem(i),
432 None => p.fatal("expected an item keyword")
434 "block" => token::NtBlock(p.parse_block()),
435 "stmt" => token::NtStmt(p.parse_stmt(Vec::new())),
436 "pat" => token::NtPat(p.parse_pat()),
437 "expr" => token::NtExpr(p.parse_expr()),
438 "ty" => token::NtTy(p.parse_ty(false /* no need to disambiguate*/)),
439 // this could be handled like a token, since it is one
440 "ident" => match p.token {
441 token::IDENT(sn,b) => { p.bump(); token::NtIdent(box sn,b) }
443 let token_str = token::to_string(&p.token);
444 p.fatal((format!("expected ident, found {}",
445 token_str.as_slice())).as_slice())
449 token::NtPath(box p.parse_path(LifetimeAndTypesWithoutColons).path)
451 "meta" => token::NtMeta(p.parse_meta_item()),
453 p.quote_depth += 1u; //but in theory, non-quoted tts might be useful
454 let res = token::NtTT(P(p.parse_token_tree()));
458 "matchers" => token::NtMatchers(p.parse_matchers()),
460 p.fatal(format!("unsupported builtin nonterminal parser: {}",