1 // Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! This is an NFA-based parser, which calls out to the main rust parser for named nonterminals
12 //! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads
13 //! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in
14 //! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier
15 //! fit for Macro-by-Example-style rules.
17 //! (In order to prevent the pathological case, we'd need to lazily construct the resulting
18 //! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old
19 //! items, but it would also save overhead)
21 //! We don't say this parser uses the Earley algorithm, because it's unnecessarily innacurate.
22 //! The macro parser restricts itself to the features of finite state automata. Earley parsers
23 //! can be described as an extension of NFAs with completion rules, prediction rules, and recursion.
25 //! Quick intro to how the parser works:
27 //! A 'position' is a dot in the middle of a matcher, usually represented as a
28 //! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`.
30 //! The parser walks through the input a character at a time, maintaining a list
31 //! of threads consistent with the current position in the input string: `cur_items`.
33 //! As it processes them, it fills up `eof_items` with threads that would be valid if
34 //! the macro invocation is now over, `bb_items` with threads that are waiting on
35 //! a Rust nonterminal like `$e:expr`, and `next_items` with threads that are waiting
36 //! on a particular token. Most of the logic concerns moving the · through the
37 //! repetitions indicated by Kleene stars. The rules for moving the · without
38 //! consuming any input are called epsilon transitions. It only advances or calls
39 //! out to the real Rust parser when no `cur_items` threads remain.
44 //! Start parsing a a a a b against [· a $( a )* a b].
46 //! Remaining input: a a a a b
47 //! next: [· a $( a )* a b]
49 //! - - - Advance over an a. - - -
51 //! Remaining input: a a a b
52 //! cur: [a · $( a )* a b]
53 //! Descend/Skip (first item).
54 //! next: [a $( · a )* a b] [a $( a )* · a b].
56 //! - - - Advance over an a. - - -
58 //! Remaining input: a a b
59 //! cur: [a $( a · )* a b] [a $( a )* a · b]
60 //! Follow epsilon transition: Finish/Repeat (first item)
61 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
63 //! - - - Advance over an a. - - - (this looks exactly like the last step)
65 //! Remaining input: a b
66 //! cur: [a $( a · )* a b] [a $( a )* a · b]
67 //! Follow epsilon transition: Finish/Repeat (first item)
68 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
70 //! - - - Advance over an a. - - - (this looks exactly like the last step)
72 //! Remaining input: b
73 //! cur: [a $( a · )* a b] [a $( a )* a · b]
74 //! Follow epsilon transition: Finish/Repeat (first item)
75 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
77 //! - - - Advance over a b. - - -
79 //! Remaining input: ''
80 //! eof: [a $( a )* a b ·]
83 pub use self::NamedMatch::*;
84 pub use self::ParseResult::*;
85 use self::TokenTreeOrTokenTreeVec::*;
88 use syntax_pos::{self, BytePos, Span};
89 use errors::FatalError;
90 use ext::tt::quoted::{self, TokenTree};
91 use parse::{Directory, ParseSess};
92 use parse::parser::{Parser, PathStyle};
93 use parse::token::{self, DocComment, Nonterminal, Token};
96 use tokenstream::TokenStream;
97 use util::small_vector::SmallVector;
101 use std::collections::HashMap;
102 use std::collections::hash_map::Entry::{Occupied, Vacant};
104 // To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body.
106 /// Either a sequence of token trees or a single one. This is used as the representation of the
107 /// sequence of tokens that make up a matcher.
109 enum TokenTreeOrTokenTreeVec {
111 TtSeq(Vec<TokenTree>),
114 impl TokenTreeOrTokenTreeVec {
115 /// Returns the number of constituent top-level token trees of `self` (top-level in that it
116 /// will not recursively descend into subtrees).
117 fn len(&self) -> usize {
119 TtSeq(ref v) => v.len(),
120 Tt(ref tt) => tt.len(),
124 /// The the `index`-th token tree of `self`.
125 fn get_tt(&self, index: usize) -> TokenTree {
127 TtSeq(ref v) => v[index].clone(),
128 Tt(ref tt) => tt.get_tt(index),
133 /// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`.
135 /// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have
138 struct MatcherTtFrame {
139 /// The "parent" matcher that we are descending into.
140 elts: TokenTreeOrTokenTreeVec,
141 /// The position of the "dot" in `elts` at the time we descended.
145 /// Represents a single "position" (aka "matcher position", aka "item"), as described in the module
149 /// The token or sequence of tokens that make up the matcher
150 top_elts: TokenTreeOrTokenTreeVec,
151 /// The position of the "dot" in this matcher
153 /// The beginning position in the source that the beginning of this matcher corresponds to. In
154 /// other words, the token in the source at `sp_lo` is matched against the first token of the
158 /// For each named metavar in the matcher, we keep track of token trees matched against the
159 /// metavar by the black box parser. In particular, there may be more than one match per
160 /// metavar if we are in a repetition (each repetition matches each of the variables).
161 /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the
162 /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of
163 /// the current position of the `self` matcher position in the shared `matches` list.
165 /// Also, note that while we are descending into a sequence, matchers are given their own
166 /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add
167 /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep`
168 /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one
169 /// wants the shared `matches`, one should use `up.matches`.
170 matches: Vec<Rc<Vec<NamedMatch>>>,
171 /// The position in `matches` corresponding to the first metavar in this matcher's sequence of
172 /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds
173 /// to `matches[match_lo]`.
175 /// The position in `matches` corresponding to the metavar we are currently trying to match
176 /// against the source token stream. `match_lo <= match_cur <= match_hi`.
178 /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar
182 // Specifically used if we are matching a repetition. If we aren't both should be `None`.
183 /// The KleeneOp of this sequence if we are in a repetition.
184 seq_op: Option<quoted::KleeneOp>,
185 /// The separator if we are in a repetition
187 /// The "parent" matcher position if we are in a repetition. That is, the matcher position just
188 /// before we enter the sequence.
189 up: Option<Box<MatcherPos>>,
191 // Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from
192 // a delimited token tree (e.g. something wrapped in `(` `)`) or to get the contents of a doc
194 /// When matching against matchers with nested delimited submatchers (e.g. `pat ( pat ( .. )
195 /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does
196 /// that where the bottom of the stack is the outermost matcher.
197 // Also, throughout the comments, this "descent" is often referred to as "unzipping"...
198 stack: Vec<MatcherTtFrame>,
202 /// Add `m` as a named match for the `idx`-th metavar.
203 fn push_match(&mut self, idx: usize, m: NamedMatch) {
204 let matches = Rc::make_mut(&mut self.matches[idx]);
209 /// Represents the possible results of an attempted parse.
210 pub enum ParseResult<T> {
211 /// Parsed successfully.
213 /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected
214 /// end of macro invocation. Otherwise, it indicates that no rules expected the given token.
215 Failure(syntax_pos::Span, Token),
216 /// Fatal error (malformed macro?). Abort compilation.
217 Error(syntax_pos::Span, String),
220 /// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es.
221 /// This represents the mapping of metavars to the token trees they bind to.
222 pub type NamedParseResult = ParseResult<HashMap<Ident, Rc<NamedMatch>>>;
224 /// Count how many metavars are named in the given matcher `ms`.
225 pub fn count_names(ms: &[TokenTree]) -> usize {
226 ms.iter().fold(0, |count, elt| {
228 TokenTree::Sequence(_, ref seq) => seq.num_captures,
229 TokenTree::Delimited(_, ref delim) => count_names(&delim.tts),
230 TokenTree::MetaVar(..) => 0,
231 TokenTree::MetaVarDecl(..) => 1,
232 TokenTree::Token(..) => 0,
237 /// Initialize `len` empty shared `Vec`s to be used to store matches of metavars.
238 fn create_matches(len: usize) -> Vec<Rc<Vec<NamedMatch>>> {
239 (0..len).into_iter().map(|_| Rc::new(Vec::new())).collect()
242 /// Generate the top-level matcher position in which the "dot" is before the first token of the
243 /// matcher `ms` and we are going to start matching at position `lo` in the source.
244 fn initial_matcher_pos(ms: Vec<TokenTree>, lo: BytePos) -> Box<MatcherPos> {
245 let match_idx_hi = count_names(&ms[..]);
246 let matches = create_matches(match_idx_hi);
247 Box::new(MatcherPos {
248 // Start with the top level matcher given to us
249 top_elts: TtSeq(ms), // "elts" is an abbr. for "elements"
250 // The "dot" is before the first token of the matcher
252 // We start matching with byte `lo` in the source code
255 // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`.
256 // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since
257 // we haven't actually matched anything yet.
261 match_hi: match_idx_hi,
263 // Haven't descended into any delimiters, so empty stack
266 // Haven't descended into any sequences, so both of these are `None`.
273 /// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`:
274 /// so it is associated with a single ident in a parse, and all
275 /// `MatchedNonterminal`s in the `NamedMatch` have the same nonterminal type
276 /// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a
277 /// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it.
279 /// The in-memory structure of a particular `NamedMatch` represents the match
280 /// that occurred when a particular subset of a matcher was applied to a
281 /// particular token tree.
283 /// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of
284 /// the `MatchedNonterminal`s, will depend on the token tree it was applied
285 /// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating
286 /// token tree. The depth of the `NamedMatch` structure will therefore depend
287 /// only on the nesting depth of `ast::TTSeq`s in the originating
288 /// token tree it was derived from.
289 #[derive(Debug, Clone)]
290 pub enum NamedMatch {
291 MatchedSeq(Rc<Vec<NamedMatch>>, syntax_pos::Span),
292 MatchedNonterminal(Rc<Nonterminal>),
295 /// Takes a sequence of token trees `ms` representing a matcher which successfully matched input
296 /// and an iterator of items that matched input and produces a `NamedParseResult`.
297 fn nameize<I: Iterator<Item = NamedMatch>>(
301 ) -> NamedParseResult {
302 // Recursively descend into each type of matcher (e.g. sequences, delimited, metavars) and make
303 // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one
304 // binding, then there is an error. If it does, then we insert the binding into the
305 // `NamedParseResult`.
306 fn n_rec<I: Iterator<Item = NamedMatch>>(
310 ret_val: &mut HashMap<Ident, Rc<NamedMatch>>,
311 ) -> Result<(), (syntax_pos::Span, String)> {
313 TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts {
314 n_rec(sess, next_m, res.by_ref(), ret_val)?
316 TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts {
317 n_rec(sess, next_m, res.by_ref(), ret_val)?;
319 TokenTree::MetaVarDecl(span, _, id) if id.name == keywords::Invalid.name() => {
320 if sess.missing_fragment_specifiers.borrow_mut().remove(&span) {
321 return Err((span, "missing fragment specifier".to_string()));
324 TokenTree::MetaVarDecl(sp, bind_name, _) => {
325 match ret_val.entry(bind_name) {
327 // FIXME(simulacrum): Don't construct Rc here
328 spot.insert(Rc::new(res.next().unwrap()));
331 return Err((sp, format!("duplicated bind name: {}", bind_name)))
335 TokenTree::MetaVar(..) | TokenTree::Token(..) => (),
341 let mut ret_val = HashMap::new();
343 match n_rec(sess, m, res.by_ref(), &mut ret_val) {
345 Err((sp, msg)) => return Error(sp, msg),
352 /// Generate an appropriate parsing failure message. For EOF, this is "unexpected end...". For
353 /// other tokens, this is "unexpected token...".
354 pub fn parse_failure_msg(tok: Token) -> String {
356 token::Eof => "unexpected end of macro invocation".to_string(),
358 "no rules expected the token `{}`",
359 pprust::token_to_string(&tok)
364 /// Perform a token equality check, ignoring syntax context (that is, an unhygienic comparison)
365 fn token_name_eq(t1: &Token, t2: &Token) -> bool {
366 if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) {
367 id1.name == id2.name && is_raw1 == is_raw2
368 } else if let (&token::Lifetime(id1), &token::Lifetime(id2)) = (t1, t2) {
375 /// Process the matcher positions of `cur_items` until it is empty. In the process, this will
376 /// produce more items in `next_items`, `eof_items`, and `bb_items`.
378 /// For more info about the how this happens, see the module-level doc comments and the inline
379 /// comments of this function.
383 /// - `sess`: the parsing session into which errors are emitted.
384 /// - `cur_items`: the set of current items to be processed. This should be empty by the end of a
385 /// successful execution of this function.
386 /// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in
387 /// the function `parse`.
388 /// - `eof_items`: the set of items that would be valid if this was the EOF.
389 /// - `bb_items`: the set of items that are waiting for the black-box parser.
390 /// - `token`: the current token of the parser.
391 /// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match
392 /// against the matcher positions in `cur_items`.
396 /// A `ParseResult`. Note that matches are kept track of through the items generated.
399 cur_items: &mut SmallVector<Box<MatcherPos>>,
400 next_items: &mut Vec<Box<MatcherPos>>,
401 eof_items: &mut SmallVector<Box<MatcherPos>>,
402 bb_items: &mut SmallVector<Box<MatcherPos>>,
404 span: syntax_pos::Span,
405 ) -> ParseResult<()> {
406 // Pop items from `cur_items` until it is empty.
407 while let Some(mut item) = cur_items.pop() {
408 // When unzipped trees end, remove them. This corresponds to backtracking out of a
409 // delimited submatcher into which we already descended. In backtracking out again, we need
410 // to advance the "dot" past the delimiters in the outer matcher.
411 while item.idx >= item.top_elts.len() {
412 match item.stack.pop() {
413 Some(MatcherTtFrame { elts, idx }) => {
414 item.top_elts = elts;
421 // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in
422 // the matcher (`len`).
424 let len = item.top_elts.len();
426 // If `idx >= len`, then we are at or past the end of the matcher of `item`.
428 // We are repeating iff there is a parent. If the matcher is inside of a repetition,
429 // then we could be at the end of a sequence or at the beginning of the next
431 if item.up.is_some() {
432 // At this point, regardless of whether there is a separator, we should add all
433 // matches from the complete repetition of the sequence to the shared, top-level
434 // `matches` list (actually, `up.matches`, which could itself not be the top-level,
435 // but anyway...). Moreover, we add another item to `cur_items` in which the "dot"
436 // is at the end of the `up` matcher. This ensures that the "dot" in the `up`
437 // matcher is also advanced sufficiently.
439 // NOTE: removing the condition `idx == len` allows trailing separators.
441 // Get the `up` matcher
442 let mut new_pos = item.up.clone().unwrap();
444 // Add matches from this repetition to the `matches` of `up`
445 for idx in item.match_lo..item.match_hi {
446 let sub = item.matches[idx].clone();
447 let span = span.with_lo(item.sp_lo);
448 new_pos.push_match(idx, MatchedSeq(sub, span));
451 // Move the "dot" past the repetition in `up`
452 new_pos.match_cur = item.match_hi;
454 cur_items.push(new_pos);
457 // Check if we need a separator.
458 if idx == len && item.sep.is_some() {
459 // We have a separator, and it is the current token. We can advance past the
463 .map(|sep| token_name_eq(token, sep))
467 next_items.push(item);
470 // We don't need a separator. Move the "dot" back to the beginning of the matcher
471 // and try to match again UNLESS we are only allowed to have _one_ repetition.
472 else if item.seq_op != Some(quoted::KleeneOp::ZeroOrOne) {
473 item.match_cur = item.match_lo;
475 cur_items.push(item);
478 // If we are not in a repetition, then being at the end of a matcher means that we have
479 // reached the potential end of the input.
481 eof_items.push(item);
484 // We are in the middle of a matcher.
486 // Look at what token in the matcher we are trying to match the current token (`token`)
487 // against. Depending on that, we may generate new items.
488 match item.top_elts.get_tt(idx) {
489 // Need to descend into a sequence
490 TokenTree::Sequence(sp, seq) => {
491 // Examine the case where there are 0 matches of this sequence
492 if seq.op == quoted::KleeneOp::ZeroOrMore
493 || seq.op == quoted::KleeneOp::ZeroOrOne
495 let mut new_item = item.clone();
496 new_item.match_cur += seq.num_captures;
498 for idx in item.match_cur..item.match_cur + seq.num_captures {
499 new_item.push_match(idx, MatchedSeq(Rc::new(vec![]), sp));
501 cur_items.push(new_item);
504 let matches = create_matches(item.matches.len());
505 cur_items.push(Box::new(MatcherPos {
507 sep: seq.separator.clone(),
508 seq_op: Some(seq.op),
511 match_lo: item.match_cur,
512 match_cur: item.match_cur,
513 match_hi: item.match_cur + seq.num_captures,
516 top_elts: Tt(TokenTree::Sequence(sp, seq)),
520 // We need to match a metavar (but the identifier is invalid)... this is an error
521 TokenTree::MetaVarDecl(span, _, id) if id.name == keywords::Invalid.name() => {
522 if sess.missing_fragment_specifiers.borrow_mut().remove(&span) {
523 return Error(span, "missing fragment specifier".to_string());
527 // We need to match a metavar with a valid ident... call out to the black-box
528 // parser by adding an item to `bb_items`.
529 TokenTree::MetaVarDecl(_, _, id) => {
530 // Built-in nonterminals never start with these tokens,
531 // so we can eliminate them from consideration.
532 if may_begin_with(&*id.name.as_str(), token) {
537 // We need to descend into a delimited submatcher or a doc comment. To do this, we
538 // push the current matcher onto a stack and push a new item containing the
539 // submatcher onto `cur_items`.
541 // At the beginning of the loop, if we reach the end of the delimited submatcher,
542 // we pop the stack to backtrack out of the descent.
543 seq @ TokenTree::Delimited(..) | seq @ TokenTree::Token(_, DocComment(..)) => {
544 let lower_elts = mem::replace(&mut item.top_elts, Tt(seq));
546 item.stack.push(MatcherTtFrame {
551 cur_items.push(item);
554 // We just matched a normal token. We can just advance the parser.
555 TokenTree::Token(_, ref t) if token_name_eq(t, token) => {
557 next_items.push(item);
560 // There was another token that was not `token`... This means we can't add any
561 // rules. NOTE that this is not necessarily an error unless _all_ items in
562 // `cur_items` end up doing this. There may still be some other matchers that do
563 // end up working out.
564 TokenTree::Token(..) | TokenTree::MetaVar(..) => {}
569 // Yay a successful parse (so far)!
573 /// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts`
574 /// against it and return the match.
578 /// - `sess`: The session into which errors are emitted
579 /// - `tts`: The tokenstream we are matching against the pattern `ms`
580 /// - `ms`: A sequence of token trees representing a pattern against which we are matching
581 /// - `directory`: Information about the file locations (needed for the black-box parser)
582 /// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box
588 directory: Option<Directory>,
589 recurse_into_modules: bool,
590 ) -> NamedParseResult {
591 // Create a parser that can be used for the "black box" parts.
592 let mut parser = Parser::new(sess, tts, directory, recurse_into_modules, true);
594 // A queue of possible matcher positions. We initialize it with the matcher position in which
595 // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then
596 // processes all of these possible matcher positions and produces posible next positions into
597 // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items`
598 // and we start over again.
599 let mut cur_items = SmallVector::one(initial_matcher_pos(ms.to_owned(), parser.span.lo()));
600 let mut next_items = Vec::new();
603 // Matcher positions black-box parsed by parser.rs (`parser`)
604 let mut bb_items = SmallVector::new();
606 // Matcher positions that would be valid if the macro invocation was over now
607 let mut eof_items = SmallVector::new();
608 assert!(next_items.is_empty());
610 // Process `cur_items` until either we have finished the input or we need to get some
611 // parsing from the black-box parser done. The result is that `next_items` will contain a
612 // bunch of possible next matcher positions in `next_items`.
613 match inner_parse_loop(
623 Failure(sp, tok) => return Failure(sp, tok),
624 Error(sp, msg) => return Error(sp, msg),
627 // inner parse loop handled all cur_items, so it's empty
628 assert!(cur_items.is_empty());
630 // We need to do some post processing after the `inner_parser_loop`.
632 // Error messages here could be improved with links to original rules.
634 // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise,
635 // either the parse is ambiguous (which should never happen) or their is a syntax error.
636 if token_name_eq(&parser.token, &token::Eof) {
637 if eof_items.len() == 1 {
638 let matches = eof_items[0]
641 .map(|dv| Rc::make_mut(dv).pop().unwrap());
642 return nameize(sess, ms, matches);
643 } else if eof_items.len() > 1 {
646 "ambiguity: multiple successful parses".to_string(),
649 return Failure(parser.span, token::Eof);
652 // Another possibility is that we need to call out to parse some rust nonterminal
653 // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong.
654 else if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 {
657 .map(|item| match item.top_elts.get_tt(item.idx) {
658 TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind),
661 .collect::<Vec<String>>()
667 "local ambiguity: multiple parsing options: {}",
668 match next_items.len() {
669 0 => format!("built-in NTs {}.", nts),
670 1 => format!("built-in NTs {} or 1 other option.", nts),
671 n => format!("built-in NTs {} or {} other options.", nts, n),
676 // If there are no posible next positions AND we aren't waiting for the black-box parser,
677 // then their is a syntax error.
678 else if bb_items.is_empty() && next_items.is_empty() {
679 return Failure(parser.span, parser.token);
681 // Dump all possible `next_items` into `cur_items` for the next iteration.
682 else if !next_items.is_empty() {
683 // Now process the next token
684 cur_items.extend(next_items.drain(..));
687 // Finally, we have the case where we need to call the black-box parser to get some
690 assert_eq!(bb_items.len(), 1);
692 let mut item = bb_items.pop().unwrap();
693 if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) {
694 let match_cur = item.match_cur;
697 MatchedNonterminal(Rc::new(parse_nt(&mut parser, span, &ident.name.as_str()))),
704 cur_items.push(item);
707 assert!(!cur_items.is_empty());
711 /// The token is an identifier, but not `_`.
712 /// We prohibit passing `_` to macros expecting `ident` for now.
713 fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
715 token::Ident(ident, is_raw) if ident.name != keywords::Underscore.name() =>
716 Some((ident, is_raw)),
721 /// Checks whether a non-terminal may begin with a particular token.
723 /// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that
724 /// token. Be conservative (return true) if not sure.
725 fn may_begin_with(name: &str, token: &Token) -> bool {
726 /// Checks whether the non-terminal may contain a single (non-keyword) identifier.
727 fn may_be_ident(nt: &token::Nonterminal) -> bool {
729 token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) => false,
735 "expr" => token.can_begin_expr(),
736 "ty" => token.can_begin_type(),
737 "ident" => get_macro_ident(token).is_some(),
738 "vis" => match *token {
739 // The follow-set of :vis + "priv" keyword + interpolated
740 Token::Comma | Token::Ident(..) | Token::Interpolated(_) => true,
741 _ => token.can_begin_type(),
743 "block" => match *token {
744 Token::OpenDelim(token::Brace) => true,
745 Token::Interpolated(ref nt) => match nt.0 {
752 | token::NtVis(_) => false, // none of these may start with '{'.
757 "path" | "meta" => match *token {
758 Token::ModSep | Token::Ident(..) => true,
759 Token::Interpolated(ref nt) => match nt.0 {
760 token::NtPath(_) | token::NtMeta(_) => true,
761 _ => may_be_ident(&nt.0),
765 "pat" => match *token {
766 Token::Ident(..) | // box, ref, mut, and other identifiers (can stricten)
767 Token::OpenDelim(token::Paren) | // tuple pattern
768 Token::OpenDelim(token::Bracket) | // slice pattern
769 Token::BinOp(token::And) | // reference
770 Token::BinOp(token::Minus) | // negative literal
771 Token::AndAnd | // double reference
772 Token::Literal(..) | // literal
773 Token::DotDot | // range pattern (future compat)
774 Token::DotDotDot | // range pattern (future compat)
775 Token::ModSep | // path
776 Token::Lt | // path (UFCS constant)
777 Token::BinOp(token::Shl) => true, // path (double UFCS)
778 Token::Interpolated(ref nt) => may_be_ident(&nt.0),
782 token::CloseDelim(_) => false,
788 /// A call to the "black-box" parser to parse some rust nonterminal.
792 /// - `p`: the "black-box" parser to use
793 /// - `sp`: the `Span` we want to parse
794 /// - `name`: the name of the metavar _matcher_ we want to match (e.g. `tt`, `ident`, `block`,
799 /// The parsed nonterminal.
800 fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
802 return token::NtTT(p.parse_token_tree());
804 // check at the beginning and the parser checks after each bump
805 p.process_potential_macro_variable();
807 "item" => match panictry!(p.parse_item()) {
808 Some(i) => token::NtItem(i),
810 p.fatal("expected an item keyword").emit();
814 "block" => token::NtBlock(panictry!(p.parse_block())),
815 "stmt" => match panictry!(p.parse_stmt()) {
816 Some(s) => token::NtStmt(s),
818 p.fatal("expected a statement").emit();
822 "pat" => token::NtPat(panictry!(p.parse_pat())),
823 "expr" => token::NtExpr(panictry!(p.parse_expr())),
824 "ty" => token::NtTy(panictry!(p.parse_ty())),
825 // this could be handled like a token, since it is one
826 "ident" => if let Some((ident, is_raw)) = get_macro_ident(&p.token) {
829 token::NtIdent(Ident::new(ident.name, span), is_raw)
831 let token_str = pprust::token_to_string(&p.token);
832 p.fatal(&format!("expected ident, found {}", &token_str)).emit();
835 "path" => token::NtPath(panictry!(p.parse_path_common(PathStyle::Type, false))),
836 "meta" => token::NtMeta(panictry!(p.parse_meta_item())),
837 "vis" => token::NtVis(panictry!(p.parse_visibility(true))),
838 "lifetime" => token::NtLifetime(p.expect_lifetime()),
839 // this is not supposed to happen, since it has been checked
840 // when compiling the macro.
841 _ => p.span_bug(sp, "invalid fragment specifier"),