1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! This is an Earley-like parser, without support for in-grammar nonterminals,
12 //! only by calling out to the main rust parser for named nonterminals (which it
13 //! commits to fully when it hits one in a grammar). This means that there are no
14 //! completer or predictor rules, and therefore no need to store one column per
15 //! token: instead, there's a set of current Earley items and a set of next
16 //! ones. Instead of NTs, we have a special case for Kleene star. The big-O, in
17 //! pathological cases, is worse than traditional Earley parsing, but it's an
18 //! easier fit for Macro-by-Example-style rules, and I think the overhead is
19 //! lower. (In order to prevent the pathological case, we'd need to lazily
20 //! construct the resulting `NamedMatch`es at the very end. It'd be a pain,
21 //! and require more memory to keep around old items, but it would also save
24 //! Quick intro to how the parser works:
26 //! A 'position' is a dot in the middle of a matcher, usually represented as a
27 //! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`.
29 //! The parser walks through the input a character at a time, maintaining a list
30 //! of items consistent with the current position in the input string: `cur_eis`.
32 //! As it processes them, it fills up `eof_eis` with items that would be valid if
33 //! the macro invocation is now over, `bb_eis` with items that are waiting on
34 //! a Rust nonterminal like `$e:expr`, and `next_eis` with items that are waiting
35 //! on a particular token. Most of the logic concerns moving the · through the
36 //! repetitions indicated by Kleene stars. It only advances or calls out to the
37 //! real Rust parser when no `cur_eis` items remain
42 //! Start parsing a a a a b against [· a $( a )* a b].
44 //! Remaining input: a a a a b
45 //! next_eis: [· a $( a )* a b]
47 //! - - - Advance over an a. - - -
49 //! Remaining input: a a a b
50 //! cur: [a · $( a )* a b]
51 //! Descend/Skip (first item).
52 //! next: [a $( · a )* a b] [a $( a )* · a b].
54 //! - - - Advance over an a. - - -
56 //! Remaining input: a a b
57 //! cur: [a $( a · )* a b] next: [a $( a )* a · b]
58 //! Finish/Repeat (first item)
59 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
61 //! - - - Advance over an a. - - - (this looks exactly like the last step)
63 //! Remaining input: a b
64 //! cur: [a $( a · )* a b] next: [a $( a )* a · b]
65 //! Finish/Repeat (first item)
66 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
68 //! - - - Advance over an a. - - - (this looks exactly like the last step)
70 //! Remaining input: b
71 //! cur: [a $( a · )* a b] next: [a $( a )* a · b]
72 //! Finish/Repeat (first item)
73 //! next: [a $( a )* · a b] [a $( · a )* a b]
75 //! - - - Advance over a b. - - -
77 //! Remaining input: ''
78 //! eof: [a $( a )* a b ·]
81 pub use self::NamedMatch::*;
82 pub use self::ParseResult::*;
83 use self::TokenTreeOrTokenTreeVec::*;
86 use syntax_pos::{self, BytePos, Span};
88 use errors::FatalError;
89 use ext::tt::quoted::{self, TokenTree};
90 use parse::{Directory, ParseSess};
91 use parse::parser::{PathStyle, Parser};
92 use parse::token::{self, DocComment, Token, Nonterminal};
95 use tokenstream::TokenStream;
96 use util::small_vector::SmallVector;
100 use std::collections::HashMap;
101 use std::collections::hash_map::Entry::{Vacant, Occupied};
103 // To avoid costly uniqueness checks, we require that `MatchSeq` always has
107 enum TokenTreeOrTokenTreeVec {
109 TtSeq(Vec<TokenTree>),
112 impl TokenTreeOrTokenTreeVec {
113 fn len(&self) -> usize {
115 TtSeq(ref v) => v.len(),
116 Tt(ref tt) => tt.len(),
120 fn get_tt(&self, index: usize) -> TokenTree {
122 TtSeq(ref v) => v[index].clone(),
123 Tt(ref tt) => tt.get_tt(index),
128 /// an unzipping of `TokenTree`s
130 struct MatcherTtFrame {
131 elts: TokenTreeOrTokenTreeVec,
137 stack: Vec<MatcherTtFrame>,
138 top_elts: TokenTreeOrTokenTreeVec,
141 up: Option<Box<MatcherPos>>,
142 matches: Vec<Rc<Vec<NamedMatch>>>,
150 fn push_match(&mut self, idx: usize, m: NamedMatch) {
151 let matches = Rc::make_mut(&mut self.matches[idx]);
156 pub type NamedParseResult = ParseResult<HashMap<Ident, Rc<NamedMatch>>>;
158 pub fn count_names(ms: &[TokenTree]) -> usize {
159 ms.iter().fold(0, |count, elt| {
161 TokenTree::Sequence(_, ref seq) => {
164 TokenTree::Delimited(_, ref delim) => {
165 count_names(&delim.tts)
167 TokenTree::MetaVarDecl(..) => {
170 TokenTree::Token(..) => 0,
175 fn initial_matcher_pos(ms: Vec<TokenTree>, lo: BytePos) -> Box<MatcherPos> {
176 let match_idx_hi = count_names(&ms[..]);
177 let matches = create_matches(match_idx_hi);
178 Box::new(MatcherPos {
187 match_hi: match_idx_hi,
192 /// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`:
193 /// so it is associated with a single ident in a parse, and all
194 /// `MatchedNonterminal`s in the `NamedMatch` have the same nonterminal type
195 /// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a
196 /// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it.
198 /// The in-memory structure of a particular `NamedMatch` represents the match
199 /// that occurred when a particular subset of a matcher was applied to a
200 /// particular token tree.
202 /// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of
203 /// the `MatchedNonterminal`s, will depend on the token tree it was applied
204 /// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating
205 /// token tree. The depth of the `NamedMatch` structure will therefore depend
206 /// only on the nesting depth of `ast::TTSeq`s in the originating
207 /// token tree it was derived from.
209 #[derive(Debug, Clone)]
210 pub enum NamedMatch {
211 MatchedSeq(Rc<Vec<NamedMatch>>, syntax_pos::Span),
212 MatchedNonterminal(Rc<Nonterminal>)
215 fn nameize<I: Iterator<Item=NamedMatch>>(sess: &ParseSess, ms: &[TokenTree], mut res: I)
216 -> NamedParseResult {
217 fn n_rec<I: Iterator<Item=NamedMatch>>(sess: &ParseSess, m: &TokenTree, mut res: &mut I,
218 ret_val: &mut HashMap<Ident, Rc<NamedMatch>>)
219 -> Result<(), (syntax_pos::Span, String)> {
221 TokenTree::Sequence(_, ref seq) => {
222 for next_m in &seq.tts {
223 n_rec(sess, next_m, res.by_ref(), ret_val)?
226 TokenTree::Delimited(_, ref delim) => {
227 for next_m in &delim.tts {
228 n_rec(sess, next_m, res.by_ref(), ret_val)?;
231 TokenTree::MetaVarDecl(span, _, id) if id.name == keywords::Invalid.name() => {
232 if sess.missing_fragment_specifiers.borrow_mut().remove(&span) {
233 return Err((span, "missing fragment specifier".to_string()));
236 TokenTree::MetaVarDecl(sp, bind_name, _) => {
237 match ret_val.entry(bind_name) {
239 // FIXME(simulacrum): Don't construct Rc here
240 spot.insert(Rc::new(res.next().unwrap()));
243 return Err((sp, format!("duplicated bind name: {}", bind_name)))
247 TokenTree::Token(..) => (),
253 let mut ret_val = HashMap::new();
255 match n_rec(sess, m, res.by_ref(), &mut ret_val) {
257 Err((sp, msg)) => return Error(sp, msg),
264 pub enum ParseResult<T> {
266 /// Arm failed to match. If the second parameter is `token::Eof`, it
267 /// indicates an unexpected end of macro invocation. Otherwise, it
268 /// indicates that no rules expected the given token.
269 Failure(syntax_pos::Span, Token),
270 /// Fatal error (malformed macro?). Abort compilation.
271 Error(syntax_pos::Span, String)
274 pub fn parse_failure_msg(tok: Token) -> String {
276 token::Eof => "unexpected end of macro invocation".to_string(),
277 _ => format!("no rules expected the token `{}`", pprust::token_to_string(&tok)),
281 /// Perform a token equality check, ignoring syntax context (that is, an unhygienic comparison)
282 fn token_name_eq(t1 : &Token, t2 : &Token) -> bool {
283 if let (Some(id1), Some(id2)) = (t1.ident(), t2.ident()) {
285 } else if let (&token::Lifetime(id1), &token::Lifetime(id2)) = (t1, t2) {
292 fn create_matches(len: usize) -> Vec<Rc<Vec<NamedMatch>>> {
293 (0..len).into_iter().map(|_| Rc::new(Vec::new())).collect()
296 fn inner_parse_loop(sess: &ParseSess,
297 cur_eis: &mut SmallVector<Box<MatcherPos>>,
298 next_eis: &mut Vec<Box<MatcherPos>>,
299 eof_eis: &mut SmallVector<Box<MatcherPos>>,
300 bb_eis: &mut SmallVector<Box<MatcherPos>>,
302 span: syntax_pos::Span)
304 while let Some(mut ei) = cur_eis.pop() {
305 // When unzipped trees end, remove them
306 while ei.idx >= ei.top_elts.len() {
307 match ei.stack.pop() {
308 Some(MatcherTtFrame { elts, idx }) => {
317 let len = ei.top_elts.len();
319 // at end of sequence
321 // We are repeating iff there is a parent
323 // Disregarding the separator, add the "up" case to the tokens that should be
325 // (remove this condition to make trailing seps ok)
327 let mut new_pos = ei.up.clone().unwrap();
329 // update matches (the MBE "parse tree") by appending
330 // each tree as a subtree.
332 // Only touch the binders we have actually bound
333 for idx in ei.match_lo..ei.match_hi {
334 let sub = ei.matches[idx].clone();
335 new_pos.push_match(idx, MatchedSeq(sub, Span { lo: ei.sp_lo, ..span }));
338 new_pos.match_cur = ei.match_hi;
340 cur_eis.push(new_pos);
343 // Check if we need a separator
344 if idx == len && ei.sep.is_some() {
345 // We have a separator, and it is the current token.
346 if ei.sep.as_ref().map(|sep| token_name_eq(token, sep)).unwrap_or(false) {
350 } else { // we don't need a separator
351 ei.match_cur = ei.match_lo;
356 // We aren't repeating, so we must be potentially at the end of the input.
360 match ei.top_elts.get_tt(idx) {
361 /* need to descend into sequence */
362 TokenTree::Sequence(sp, seq) => {
363 if seq.op == quoted::KleeneOp::ZeroOrMore {
364 // Examine the case where there are 0 matches of this sequence
365 let mut new_ei = ei.clone();
366 new_ei.match_cur += seq.num_captures;
368 for idx in ei.match_cur..ei.match_cur + seq.num_captures {
369 new_ei.push_match(idx, MatchedSeq(Rc::new(vec![]), sp));
371 cur_eis.push(new_ei);
374 // Examine the case where there is at least one match of this sequence
375 let matches = create_matches(ei.matches.len());
376 cur_eis.push(Box::new(MatcherPos {
378 sep: seq.separator.clone(),
381 match_lo: ei.match_cur,
382 match_cur: ei.match_cur,
383 match_hi: ei.match_cur + seq.num_captures,
386 top_elts: Tt(TokenTree::Sequence(sp, seq)),
389 TokenTree::MetaVarDecl(span, _, id) if id.name == keywords::Invalid.name() => {
390 if sess.missing_fragment_specifiers.borrow_mut().remove(&span) {
391 return Error(span, "missing fragment specifier".to_string());
394 TokenTree::MetaVarDecl(..) => {
395 // Built-in nonterminals never start with these tokens,
396 // so we can eliminate them from consideration.
398 token::CloseDelim(_) => {},
399 _ => bb_eis.push(ei),
402 seq @ TokenTree::Delimited(..) | seq @ TokenTree::Token(_, DocComment(..)) => {
403 let lower_elts = mem::replace(&mut ei.top_elts, Tt(seq));
405 ei.stack.push(MatcherTtFrame {
412 TokenTree::Token(_, ref t) => {
413 if token_name_eq(t, token) {
425 pub fn parse(sess: &ParseSess,
428 directory: Option<Directory>,
429 recurse_into_modules: bool)
430 -> NamedParseResult {
431 let mut parser = Parser::new(sess, tts, directory, recurse_into_modules, true);
432 let mut cur_eis = SmallVector::one(initial_matcher_pos(ms.to_owned(), parser.span.lo));
433 let mut next_eis = Vec::new(); // or proceed normally
436 let mut bb_eis = SmallVector::new(); // black-box parsed by parser.rs
437 let mut eof_eis = SmallVector::new();
438 assert!(next_eis.is_empty());
440 match inner_parse_loop(sess, &mut cur_eis, &mut next_eis, &mut eof_eis, &mut bb_eis,
441 &parser.token, parser.span) {
443 Failure(sp, tok) => return Failure(sp, tok),
444 Error(sp, msg) => return Error(sp, msg),
447 // inner parse loop handled all cur_eis, so it's empty
448 assert!(cur_eis.is_empty());
450 /* error messages here could be improved with links to orig. rules */
451 if token_name_eq(&parser.token, &token::Eof) {
452 if eof_eis.len() == 1 {
453 let matches = eof_eis[0].matches.iter_mut().map(|mut dv| {
454 Rc::make_mut(dv).pop().unwrap()
456 return nameize(sess, ms, matches);
457 } else if eof_eis.len() > 1 {
458 return Error(parser.span, "ambiguity: multiple successful parses".to_string());
460 return Failure(parser.span, token::Eof);
462 } else if (!bb_eis.is_empty() && !next_eis.is_empty()) || bb_eis.len() > 1 {
463 let nts = bb_eis.iter().map(|ei| match ei.top_elts.get_tt(ei.idx) {
464 TokenTree::MetaVarDecl(_, bind, name) => {
465 format!("{} ('{}')", name, bind)
468 }).collect::<Vec<String>>().join(" or ");
470 return Error(parser.span, format!(
471 "local ambiguity: multiple parsing options: {}",
472 match next_eis.len() {
473 0 => format!("built-in NTs {}.", nts),
474 1 => format!("built-in NTs {} or 1 other option.", nts),
475 n => format!("built-in NTs {} or {} other options.", nts, n),
478 } else if bb_eis.is_empty() && next_eis.is_empty() {
479 return Failure(parser.span, parser.token);
480 } else if !next_eis.is_empty() {
481 /* Now process the next token */
482 cur_eis.extend(next_eis.drain(..));
484 } else /* bb_eis.len() == 1 */ {
485 let mut ei = bb_eis.pop().unwrap();
486 if let TokenTree::MetaVarDecl(span, _, ident) = ei.top_elts.get_tt(ei.idx) {
487 let match_cur = ei.match_cur;
488 ei.push_match(match_cur,
489 MatchedNonterminal(Rc::new(parse_nt(&mut parser, span, &ident.name.as_str()))));
498 assert!(!cur_eis.is_empty());
502 fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
504 return token::NtTT(p.parse_token_tree());
506 // check at the beginning and the parser checks after each bump
507 p.process_potential_macro_variable();
509 "item" => match panictry!(p.parse_item()) {
510 Some(i) => token::NtItem(i),
512 p.fatal("expected an item keyword").emit();
516 "block" => token::NtBlock(panictry!(p.parse_block())),
517 "stmt" => match panictry!(p.parse_stmt()) {
518 Some(s) => token::NtStmt(s),
520 p.fatal("expected a statement").emit();
524 "pat" => token::NtPat(panictry!(p.parse_pat())),
525 "expr" => token::NtExpr(panictry!(p.parse_expr())),
526 "ty" => token::NtTy(panictry!(p.parse_ty())),
527 // this could be handled like a token, since it is one
528 "ident" => match p.token {
529 token::Ident(sn) => {
531 token::NtIdent(Spanned::<Ident>{node: sn, span: p.prev_span})
534 let token_str = pprust::token_to_string(&p.token);
535 p.fatal(&format!("expected ident, found {}",
536 &token_str[..])).emit();
541 token::NtPath(panictry!(p.parse_path(PathStyle::Type)))
543 "meta" => token::NtMeta(panictry!(p.parse_meta_item())),
544 "vis" => token::NtVis(panictry!(p.parse_visibility(true))),
545 // this is not supposed to happen, since it has been checked
546 // when compiling the macro.
547 _ => p.span_bug(sp, "invalid fragment specifier")