1 //! This is an NFA-based parser, which calls out to the main rust parser for named non-terminals
2 //! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads
3 //! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in
4 //! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier
5 //! fit for Macro-by-Example-style rules.
7 //! (In order to prevent the pathological case, we'd need to lazily construct the resulting
8 //! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old
9 //! items, but it would also save overhead)
11 //! We don't say this parser uses the Earley algorithm, because it's unnecessarily inaccurate.
12 //! The macro parser restricts itself to the features of finite state automata. Earley parsers
13 //! can be described as an extension of NFAs with completion rules, prediction rules, and recursion.
15 //! Quick intro to how the parser works:
17 //! A 'position' is a dot in the middle of a matcher, usually represented as a
18 //! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`.
20 //! The parser walks through the input a character at a time, maintaining a list
21 //! of threads consistent with the current position in the input string: `cur_items`.
23 //! As it processes them, it fills up `eof_items` with threads that would be valid if
24 //! the macro invocation is now over, `bb_items` with threads that are waiting on
25 //! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting
26 //! on a particular token. Most of the logic concerns moving the · through the
27 //! repetitions indicated by Kleene stars. The rules for moving the · without
28 //! consuming any input are called epsilon transitions. It only advances or calls
29 //! out to the real Rust parser when no `cur_items` threads remain.
34 //! Start parsing a a a a b against [· a $( a )* a b].
36 //! Remaining input: a a a a b
37 //! next: [· a $( a )* a b]
39 //! - - - Advance over an a. - - -
41 //! Remaining input: a a a b
42 //! cur: [a · $( a )* a b]
43 //! Descend/Skip (first item).
44 //! next: [a $( · a )* a b] [a $( a )* · a b].
46 //! - - - Advance over an a. - - -
48 //! Remaining input: a a b
49 //! cur: [a $( a · )* a b] [a $( a )* a · b]
50 //! Follow epsilon transition: Finish/Repeat (first item)
51 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
53 //! - - - Advance over an a. - - - (this looks exactly like the last step)
55 //! Remaining input: a b
56 //! cur: [a $( a · )* a b] [a $( a )* a · b]
57 //! Follow epsilon transition: Finish/Repeat (first item)
58 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
60 //! - - - Advance over an a. - - - (this looks exactly like the last step)
62 //! Remaining input: b
63 //! cur: [a $( a · )* a b] [a $( a )* a · b]
64 //! Follow epsilon transition: Finish/Repeat (first item)
65 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
67 //! - - - Advance over a b. - - -
69 //! Remaining input: ''
70 //! eof: [a $( a )* a b ·]
73 crate use NamedMatch::*;
74 crate use ParseResult::*;
75 use TokenTreeOrTokenTreeSlice::*;
77 use crate::mbe::{self, TokenTree};
79 use rustc_parse::Directory;
80 use rustc_parse::parser::{Parser, PathStyle, FollowedByType};
81 use syntax::ast::{Ident, Name};
82 use syntax::print::pprust;
83 use syntax::sess::ParseSess;
84 use syntax::symbol::{kw, sym, Symbol};
85 use syntax::token::{self, DocComment, Nonterminal, Token};
86 use syntax::tokenstream::{DelimSpan, TokenStream};
88 use errors::{PResult, FatalError};
89 use smallvec::{smallvec, SmallVec};
92 use rustc_data_structures::fx::FxHashMap;
93 use rustc_data_structures::sync::Lrc;
94 use std::collections::hash_map::Entry::{Occupied, Vacant};
96 use std::ops::{Deref, DerefMut};
98 // To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body.
100 /// Either a sequence of token trees or a single one. This is used as the representation of the
101 /// sequence of tokens that make up a matcher.
103 enum TokenTreeOrTokenTreeSlice<'tt> {
105 TtSeq(&'tt [TokenTree]),
108 impl<'tt> TokenTreeOrTokenTreeSlice<'tt> {
109 /// Returns the number of constituent top-level token trees of `self` (top-level in that it
110 /// will not recursively descend into subtrees).
111 fn len(&self) -> usize {
113 TtSeq(ref v) => v.len(),
114 Tt(ref tt) => tt.len(),
118 /// The `index`-th token tree of `self`.
119 fn get_tt(&self, index: usize) -> TokenTree {
121 TtSeq(ref v) => v[index].clone(),
122 Tt(ref tt) => tt.get_tt(index),
127 /// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`.
129 /// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have
132 struct MatcherTtFrame<'tt> {
133 /// The "parent" matcher that we are descending into.
134 elts: TokenTreeOrTokenTreeSlice<'tt>,
135 /// The position of the "dot" in `elts` at the time we descended.
139 type NamedMatchVec = SmallVec<[NamedMatch; 4]>;
141 /// Represents a single "position" (aka "matcher position", aka "item"), as
142 /// described in the module documentation.
146 /// - `'root` represents the lifetime of the stack slot that holds the root
147 /// `MatcherPos`. As described in `MatcherPosHandle`, the root `MatcherPos`
148 /// structure is stored on the stack, but subsequent instances are put into
150 /// - `'tt` represents the lifetime of the token trees that this matcher
151 /// position refers to.
153 /// It is important to distinguish these two lifetimes because we have a
154 /// `SmallVec<TokenTreeOrTokenTreeSlice<'tt>>` below, and the destructor of
155 /// that is considered to possibly access the data from its elements (it lacks
156 /// a `#[may_dangle]` attribute). As a result, the compiler needs to know that
157 /// all the elements in that `SmallVec` strictly outlive the root stack slot
158 /// lifetime. By separating `'tt` from `'root`, we can show that.
160 struct MatcherPos<'root, 'tt> {
161 /// The token or sequence of tokens that make up the matcher
162 top_elts: TokenTreeOrTokenTreeSlice<'tt>,
164 /// The position of the "dot" in this matcher
167 /// The first span of source that the beginning of this matcher corresponds to. In other
168 /// words, the token in the source whose span is `sp_open` is matched against the first token of
172 /// For each named metavar in the matcher, we keep track of token trees matched against the
173 /// metavar by the black box parser. In particular, there may be more than one match per
174 /// metavar if we are in a repetition (each repetition matches each of the variables).
175 /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the
176 /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of
177 /// the current position of the `self` matcher position in the shared `matches` list.
179 /// Also, note that while we are descending into a sequence, matchers are given their own
180 /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add
181 /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep`
182 /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one
183 /// wants the shared `matches`, one should use `up.matches`.
184 matches: Box<[Lrc<NamedMatchVec>]>,
185 /// The position in `matches` corresponding to the first metavar in this matcher's sequence of
186 /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds
187 /// to `matches[match_lo]`.
189 /// The position in `matches` corresponding to the metavar we are currently trying to match
190 /// against the source token stream. `match_lo <= match_cur <= match_hi`.
192 /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar
196 // The following fields are used if we are matching a repetition. If we aren't, they should be
199 /// The KleeneOp of this sequence if we are in a repetition.
200 seq_op: Option<mbe::KleeneOp>,
202 /// The separator if we are in a repetition.
205 /// The "parent" matcher position if we are in a repetition. That is, the matcher position just
206 /// before we enter the sequence.
207 up: Option<MatcherPosHandle<'root, 'tt>>,
209 /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from
210 /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc
213 /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. )
214 /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does
215 /// that where the bottom of the stack is the outermost matcher.
216 /// Also, throughout the comments, this "descent" is often referred to as "unzipping"...
217 stack: SmallVec<[MatcherTtFrame<'tt>; 1]>,
220 impl<'root, 'tt> MatcherPos<'root, 'tt> {
221 /// Adds `m` as a named match for the `idx`-th metavar.
222 fn push_match(&mut self, idx: usize, m: NamedMatch) {
223 let matches = Lrc::make_mut(&mut self.matches[idx]);
228 // Lots of MatcherPos instances are created at runtime. Allocating them on the
229 // heap is slow. Furthermore, using SmallVec<MatcherPos> to allocate them all
230 // on the stack is also slow, because MatcherPos is quite a large type and
231 // instances get moved around a lot between vectors, which requires lots of
232 // slow memcpy calls.
234 // Therefore, the initial MatcherPos is always allocated on the stack,
235 // subsequent ones (of which there aren't that many) are allocated on the heap,
236 // and this type is used to encapsulate both cases.
237 enum MatcherPosHandle<'root, 'tt> {
238 Ref(&'root mut MatcherPos<'root, 'tt>),
239 Box(Box<MatcherPos<'root, 'tt>>),
242 impl<'root, 'tt> Clone for MatcherPosHandle<'root, 'tt> {
243 // This always produces a new Box.
244 fn clone(&self) -> Self {
245 MatcherPosHandle::Box(match *self {
246 MatcherPosHandle::Ref(ref r) => Box::new((**r).clone()),
247 MatcherPosHandle::Box(ref b) => b.clone(),
252 impl<'root, 'tt> Deref for MatcherPosHandle<'root, 'tt> {
253 type Target = MatcherPos<'root, 'tt>;
254 fn deref(&self) -> &Self::Target {
256 MatcherPosHandle::Ref(ref r) => r,
257 MatcherPosHandle::Box(ref b) => b,
262 impl<'root, 'tt> DerefMut for MatcherPosHandle<'root, 'tt> {
263 fn deref_mut(&mut self) -> &mut MatcherPos<'root, 'tt> {
265 MatcherPosHandle::Ref(ref mut r) => r,
266 MatcherPosHandle::Box(ref mut b) => b,
271 /// Represents the possible results of an attempted parse.
272 crate enum ParseResult<T> {
273 /// Parsed successfully.
275 /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected
276 /// end of macro invocation. Otherwise, it indicates that no rules expected the given token.
277 Failure(Token, &'static str),
278 /// Fatal error (malformed macro?). Abort compilation.
279 Error(syntax_pos::Span, String),
282 /// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es.
283 /// This represents the mapping of metavars to the token trees they bind to.
284 crate type NamedParseResult = ParseResult<FxHashMap<Ident, NamedMatch>>;
286 /// Count how many metavars are named in the given matcher `ms`.
287 pub(super) fn count_names(ms: &[TokenTree]) -> usize {
288 ms.iter().fold(0, |count, elt| {
290 TokenTree::Sequence(_, ref seq) => seq.num_captures,
291 TokenTree::Delimited(_, ref delim) => count_names(&delim.tts),
292 TokenTree::MetaVar(..) => 0,
293 TokenTree::MetaVarDecl(..) => 1,
294 TokenTree::Token(..) => 0,
299 /// `len` `Vec`s (initially shared and empty) that will store matches of metavars.
300 fn create_matches(len: usize) -> Box<[Lrc<NamedMatchVec>]> {
304 let empty_matches = Lrc::new(SmallVec::new());
305 vec![empty_matches; len]
309 /// Generates the top-level matcher position in which the "dot" is before the first token of the
310 /// matcher `ms` and we are going to start matching at the span `open` in the source.
311 fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherPos<'root, 'tt> {
312 let match_idx_hi = count_names(ms);
313 let matches = create_matches(match_idx_hi);
315 // Start with the top level matcher given to us
316 top_elts: TtSeq(ms), // "elts" is an abbr. for "elements"
317 // The "dot" is before the first token of the matcher
319 // We start matching at the span `open` in the source code
322 // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`.
323 // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since
324 // we haven't actually matched anything yet.
328 match_hi: match_idx_hi,
330 // Haven't descended into any delimiters, so empty stack
333 // Haven't descended into any sequences, so both of these are `None`.
340 /// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`:
341 /// so it is associated with a single ident in a parse, and all
342 /// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type
343 /// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a
344 /// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it.
346 /// The in-memory structure of a particular `NamedMatch` represents the match
347 /// that occurred when a particular subset of a matcher was applied to a
348 /// particular token tree.
350 /// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of
351 /// the `MatchedNonterminal`s, will depend on the token tree it was applied
352 /// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating
353 /// token tree. The depth of the `NamedMatch` structure will therefore depend
354 /// only on the nesting depth of `ast::TTSeq`s in the originating
355 /// token tree it was derived from.
356 #[derive(Debug, Clone)]
357 crate enum NamedMatch {
358 MatchedSeq(Lrc<NamedMatchVec>, DelimSpan),
359 MatchedNonterminal(Lrc<Nonterminal>),
362 /// Takes a sequence of token trees `ms` representing a matcher which successfully matched input
363 /// and an iterator of items that matched input and produces a `NamedParseResult`.
364 fn nameize<I: Iterator<Item = NamedMatch>>(
368 ) -> NamedParseResult {
369 // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make
370 // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one
371 // binding, then there is an error. If it does, then we insert the binding into the
372 // `NamedParseResult`.
373 fn n_rec<I: Iterator<Item = NamedMatch>>(
377 ret_val: &mut FxHashMap<Ident, NamedMatch>,
378 ) -> Result<(), (syntax_pos::Span, String)> {
380 TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts {
381 n_rec(sess, next_m, res.by_ref(), ret_val)?
383 TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts {
384 n_rec(sess, next_m, res.by_ref(), ret_val)?;
386 TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => {
387 if sess.missing_fragment_specifiers.borrow_mut().remove(&span) {
388 return Err((span, "missing fragment specifier".to_string()));
391 TokenTree::MetaVarDecl(sp, bind_name, _) => {
392 match ret_val.entry(bind_name) {
394 spot.insert(res.next().unwrap());
397 return Err((sp, format!("duplicated bind name: {}", bind_name)))
401 TokenTree::MetaVar(..) | TokenTree::Token(..) => (),
407 let mut ret_val = FxHashMap::default();
409 match n_rec(sess, m, res.by_ref(), &mut ret_val) {
411 Err((sp, msg)) => return Error(sp, msg),
418 /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
419 fn token_name_eq(t1: &Token, t2: &Token) -> bool {
420 if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) {
421 ident1.name == ident2.name && is_raw1 == is_raw2
422 } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) {
423 ident1.name == ident2.name
429 /// Process the matcher positions of `cur_items` until it is empty. In the process, this will
430 /// produce more items in `next_items`, `eof_items`, and `bb_items`.
432 /// For more info about the how this happens, see the module-level doc comments and the inline
433 /// comments of this function.
437 /// - `sess`: the parsing session into which errors are emitted.
438 /// - `cur_items`: the set of current items to be processed. This should be empty by the end of a
439 /// successful execution of this function.
440 /// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in
441 /// the function `parse`.
442 /// - `eof_items`: the set of items that would be valid if this was the EOF.
443 /// - `bb_items`: the set of items that are waiting for the black-box parser.
444 /// - `token`: the current token of the parser.
445 /// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match
446 /// against the matcher positions in `cur_items`.
450 /// A `ParseResult`. Note that matches are kept track of through the items generated.
451 fn inner_parse_loop<'root, 'tt>(
453 cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
454 next_items: &mut Vec<MatcherPosHandle<'root, 'tt>>,
455 eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
456 bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
458 ) -> ParseResult<()> {
459 // Pop items from `cur_items` until it is empty.
460 while let Some(mut item) = cur_items.pop() {
461 // When unzipped trees end, remove them. This corresponds to backtracking out of a
462 // delimited submatcher into which we already descended. In backtracking out again, we need
463 // to advance the "dot" past the delimiters in the outer matcher.
464 while item.idx >= item.top_elts.len() {
465 match item.stack.pop() {
466 Some(MatcherTtFrame { elts, idx }) => {
467 item.top_elts = elts;
474 // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in
475 // the matcher (`len`).
477 let len = item.top_elts.len();
479 // If `idx >= len`, then we are at or past the end of the matcher of `item`.
481 // We are repeating iff there is a parent. If the matcher is inside of a repetition,
482 // then we could be at the end of a sequence or at the beginning of the next
484 if item.up.is_some() {
485 // At this point, regardless of whether there is a separator, we should add all
486 // matches from the complete repetition of the sequence to the shared, top-level
487 // `matches` list (actually, `up.matches`, which could itself not be the top-level,
488 // but anyway...). Moreover, we add another item to `cur_items` in which the "dot"
489 // is at the end of the `up` matcher. This ensures that the "dot" in the `up`
490 // matcher is also advanced sufficiently.
492 // NOTE: removing the condition `idx == len` allows trailing separators.
494 // Get the `up` matcher
495 let mut new_pos = item.up.clone().unwrap();
497 // Add matches from this repetition to the `matches` of `up`
498 for idx in item.match_lo..item.match_hi {
499 let sub = item.matches[idx].clone();
500 let span = DelimSpan::from_pair(item.sp_open, token.span);
501 new_pos.push_match(idx, MatchedSeq(sub, span));
504 // Move the "dot" past the repetition in `up`
505 new_pos.match_cur = item.match_hi;
507 cur_items.push(new_pos);
510 // Check if we need a separator.
511 if idx == len && item.sep.is_some() {
512 // We have a separator, and it is the current token. We can advance past the
516 .map(|sep| token_name_eq(token, sep))
520 next_items.push(item);
523 // We don't need a separator. Move the "dot" back to the beginning of the matcher
524 // and try to match again UNLESS we are only allowed to have _one_ repetition.
525 else if item.seq_op != Some(mbe::KleeneOp::ZeroOrOne) {
526 item.match_cur = item.match_lo;
528 cur_items.push(item);
531 // If we are not in a repetition, then being at the end of a matcher means that we have
532 // reached the potential end of the input.
534 eof_items.push(item);
537 // We are in the middle of a matcher.
539 // Look at what token in the matcher we are trying to match the current token (`token`)
540 // against. Depending on that, we may generate new items.
541 match item.top_elts.get_tt(idx) {
542 // Need to descend into a sequence
543 TokenTree::Sequence(sp, seq) => {
544 // Examine the case where there are 0 matches of this sequence. We are
545 // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will
546 // result in a "no rules expected token" error by virtue of this matcher not
548 if seq.kleene.op == mbe::KleeneOp::ZeroOrMore
549 || seq.kleene.op == mbe::KleeneOp::ZeroOrOne
551 let mut new_item = item.clone();
552 new_item.match_cur += seq.num_captures;
554 for idx in item.match_cur..item.match_cur + seq.num_captures {
555 new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]), sp));
557 cur_items.push(new_item);
560 let matches = create_matches(item.matches.len());
561 cur_items.push(MatcherPosHandle::Box(Box::new(MatcherPos {
563 sep: seq.separator.clone(),
564 seq_op: Some(seq.kleene.op),
567 match_lo: item.match_cur,
568 match_cur: item.match_cur,
569 match_hi: item.match_cur + seq.num_captures,
572 top_elts: Tt(TokenTree::Sequence(sp, seq)),
576 // We need to match a metavar (but the identifier is invalid)... this is an error
577 TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => {
578 if sess.missing_fragment_specifiers.borrow_mut().remove(&span) {
579 return Error(span, "missing fragment specifier".to_string());
583 // We need to match a metavar with a valid ident... call out to the black-box
584 // parser by adding an item to `bb_items`.
585 TokenTree::MetaVarDecl(_, _, id) => {
586 // Built-in nonterminals never start with these tokens,
587 // so we can eliminate them from consideration.
588 if may_begin_with(token, id.name) {
593 // We need to descend into a delimited submatcher or a doc comment. To do this, we
594 // push the current matcher onto a stack and push a new item containing the
595 // submatcher onto `cur_items`.
597 // At the beginning of the loop, if we reach the end of the delimited submatcher,
598 // we pop the stack to backtrack out of the descent.
599 seq @ TokenTree::Delimited(..) |
600 seq @ TokenTree::Token(Token { kind: DocComment(..), .. }) => {
601 let lower_elts = mem::replace(&mut item.top_elts, Tt(seq));
603 item.stack.push(MatcherTtFrame {
608 cur_items.push(item);
611 // We just matched a normal token. We can just advance the parser.
612 TokenTree::Token(t) if token_name_eq(&t, token) => {
614 next_items.push(item);
617 // There was another token that was not `token`... This means we can't add any
618 // rules. NOTE that this is not necessarily an error unless _all_ items in
619 // `cur_items` end up doing this. There may still be some other matchers that do
620 // end up working out.
621 TokenTree::Token(..) | TokenTree::MetaVar(..) => {}
626 // Yay a successful parse (so far)!
630 /// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts`
631 /// against it and return the match.
635 /// - `sess`: The session into which errors are emitted
636 /// - `tts`: The tokenstream we are matching against the pattern `ms`
637 /// - `ms`: A sequence of token trees representing a pattern against which we are matching
638 /// - `directory`: Information about the file locations (needed for the black-box parser)
639 /// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box
645 directory: Option<Directory<'_>>,
646 recurse_into_modules: bool,
647 ) -> NamedParseResult {
648 // Create a parser that can be used for the "black box" parts.
649 let mut parser = Parser::new(
653 recurse_into_modules,
655 rustc_parse::MACRO_ARGUMENTS,
658 // A queue of possible matcher positions. We initialize it with the matcher position in which
659 // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then
660 // processes all of these possible matcher positions and produces possible next positions into
661 // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items`
662 // and we start over again.
664 // This MatcherPos instance is allocated on the stack. All others -- and
665 // there are frequently *no* others! -- are allocated on the heap.
666 let mut initial = initial_matcher_pos(ms, parser.token.span);
667 let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)];
668 let mut next_items = Vec::new();
671 // Matcher positions black-box parsed by parser.rs (`parser`)
672 let mut bb_items = SmallVec::new();
674 // Matcher positions that would be valid if the macro invocation was over now
675 let mut eof_items = SmallVec::new();
676 assert!(next_items.is_empty());
678 // Process `cur_items` until either we have finished the input or we need to get some
679 // parsing from the black-box parser done. The result is that `next_items` will contain a
680 // bunch of possible next matcher positions in `next_items`.
681 match inner_parse_loop(
690 Failure(token, msg) => return Failure(token, msg),
691 Error(sp, msg) => return Error(sp, msg),
694 // inner parse loop handled all cur_items, so it's empty
695 assert!(cur_items.is_empty());
697 // We need to do some post processing after the `inner_parser_loop`.
699 // Error messages here could be improved with links to original rules.
701 // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise,
702 // either the parse is ambiguous (which should never happen) or there is a syntax error.
703 if parser.token == token::Eof {
704 if eof_items.len() == 1 {
705 let matches = eof_items[0]
708 .map(|dv| Lrc::make_mut(dv).pop().unwrap());
709 return nameize(sess, ms, matches);
710 } else if eof_items.len() > 1 {
713 "ambiguity: multiple successful parses".to_string(),
717 Token::new(token::Eof, if parser.token.span.is_dummy() {
720 sess.source_map().next_point(parser.token.span)
722 "missing tokens in macro arguments",
726 // Performance hack: eof_items may share matchers via Rc with other things that we want
727 // to modify. Dropping eof_items now may drop these refcounts to 1, preventing an
728 // unnecessary implicit clone later in Rc::make_mut.
731 // Another possibility is that we need to call out to parse some rust nonterminal
732 // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong.
733 if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 {
736 .map(|item| match item.top_elts.get_tt(item.idx) {
737 TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind),
740 .collect::<Vec<String>>()
746 "local ambiguity: multiple parsing options: {}",
747 match next_items.len() {
748 0 => format!("built-in NTs {}.", nts),
749 1 => format!("built-in NTs {} or 1 other option.", nts),
750 n => format!("built-in NTs {} or {} other options.", nts, n),
755 // If there are no possible next positions AND we aren't waiting for the black-box parser,
756 // then there is a syntax error.
757 else if bb_items.is_empty() && next_items.is_empty() {
760 "no rules expected this token in macro call",
763 // Dump all possible `next_items` into `cur_items` for the next iteration.
764 else if !next_items.is_empty() {
765 // Now process the next token
766 cur_items.extend(next_items.drain(..));
769 // Finally, we have the case where we need to call the black-box parser to get some
772 assert_eq!(bb_items.len(), 1);
774 let mut item = bb_items.pop().unwrap();
775 if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) {
776 let match_cur = item.match_cur;
779 MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))),
786 cur_items.push(item);
789 assert!(!cur_items.is_empty());
793 /// The token is an identifier, but not `_`.
794 /// We prohibit passing `_` to macros expecting `ident` for now.
795 fn get_macro_name(token: &Token) -> Option<(Name, bool)> {
797 token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)),
802 /// Checks whether a non-terminal may begin with a particular token.
804 /// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that
805 /// token. Be conservative (return true) if not sure.
806 fn may_begin_with(token: &Token, name: Name) -> bool {
807 /// Checks whether the non-terminal may contain a single (non-keyword) identifier.
808 fn may_be_ident(nt: &token::Nonterminal) -> bool {
810 token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) => false,
816 sym::expr => token.can_begin_expr()
817 // This exception is here for backwards compatibility.
818 && !token.is_keyword(kw::Let),
819 sym::ty => token.can_begin_type(),
820 sym::ident => get_macro_name(token).is_some(),
821 sym::literal => token.can_begin_literal_or_bool(),
822 sym::vis => match token.kind {
823 // The follow-set of :vis + "priv" keyword + interpolated
824 token::Comma | token::Ident(..) | token::Interpolated(_) => true,
825 _ => token.can_begin_type(),
827 sym::block => match token.kind {
828 token::OpenDelim(token::Brace) => true,
829 token::Interpolated(ref nt) => match **nt {
836 | token::NtVis(_) => false, // none of these may start with '{'.
841 sym::path | sym::meta => match token.kind {
842 token::ModSep | token::Ident(..) => true,
843 token::Interpolated(ref nt) => match **nt {
844 token::NtPath(_) | token::NtMeta(_) => true,
845 _ => may_be_ident(&nt),
849 sym::pat => match token.kind {
850 token::Ident(..) | // box, ref, mut, and other identifiers (can stricten)
851 token::OpenDelim(token::Paren) | // tuple pattern
852 token::OpenDelim(token::Bracket) | // slice pattern
853 token::BinOp(token::And) | // reference
854 token::BinOp(token::Minus) | // negative literal
855 token::AndAnd | // double reference
856 token::Literal(..) | // literal
857 token::DotDot | // range pattern (future compat)
858 token::DotDotDot | // range pattern (future compat)
859 token::ModSep | // path
860 token::Lt | // path (UFCS constant)
861 token::BinOp(token::Shl) => true, // path (double UFCS)
862 token::Interpolated(ref nt) => may_be_ident(nt),
865 sym::lifetime => match token.kind {
866 token::Lifetime(_) => true,
867 token::Interpolated(ref nt) => match **nt {
868 token::NtLifetime(_) | token::NtTT(_) => true,
873 _ => match token.kind {
874 token::CloseDelim(_) => false,
880 /// A call to the "black-box" parser to parse some Rust non-terminal.
884 /// - `p`: the "black-box" parser to use
885 /// - `sp`: the `Span` we want to parse
886 /// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`,
891 /// The parsed non-terminal.
892 fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal {
893 // FIXME(Centril): Consider moving this to `parser.rs` to make
894 // the visibilities of the methods used below `pub(super)` at most.
897 return token::NtTT(p.parse_token_tree());
899 // check at the beginning and the parser checks after each bump
900 p.process_potential_macro_variable();
901 match parse_nt_inner(p, sp, name) {
910 fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> {
912 sym::item => match p.parse_item()? {
913 Some(i) => token::NtItem(i),
914 None => return Err(p.fatal("expected an item keyword")),
916 sym::block => token::NtBlock(p.parse_block()?),
917 sym::stmt => match p.parse_stmt()? {
918 Some(s) => token::NtStmt(s),
919 None => return Err(p.fatal("expected a statement")),
921 sym::pat => token::NtPat(p.parse_pat(None)?),
922 sym::expr => token::NtExpr(p.parse_expr()?),
923 sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?),
924 sym::ty => token::NtTy(p.parse_ty()?),
925 // this could be handled like a token, since it is one
926 sym::ident => if let Some((name, is_raw)) = get_macro_name(&p.token) {
927 let span = p.token.span;
929 token::NtIdent(Ident::new(name, span), is_raw)
931 let token_str = pprust::token_to_string(&p.token);
932 return Err(p.fatal(&format!("expected ident, found {}", &token_str)));
934 sym::path => token::NtPath(p.parse_path(PathStyle::Type)?),
935 sym::meta => token::NtMeta(p.parse_attr_item()?),
936 sym::vis => token::NtVis(p.parse_visibility(FollowedByType::Yes)?),
937 sym::lifetime => if p.check_lifetime() {
938 token::NtLifetime(p.expect_lifetime().ident)
940 let token_str = pprust::token_to_string(&p.token);
941 return Err(p.fatal(&format!("expected a lifetime, found `{}`", &token_str)));
943 // this is not supposed to happen, since it has been checked
944 // when compiling the macro.
945 _ => p.span_bug(sp, "invalid fragment specifier"),