last = match *token {
TtToken(sp, MatchNt(ref name, ref frag_spec, _, _)) => {
// ii. If T is a simple NT, look ahead to the next token T' in
- // M.
- let next_token = match tokens.peek() {
- // If T' closes a complex NT, replace T' with F
- Some(&&TtToken(_, CloseDelim(_))) => follow.clone(),
- Some(&&TtToken(_, ref tok)) => tok.clone(),
- Some(&&TtSequence(sp, _)) => {
- cx.span_err(sp,
- &format!("`${0}:{1}` is followed by a \
- sequence repetition, which is not \
- allowed for `{1}` fragments",
- name.as_str(), frag_spec.as_str())
+ // M. If T' is in the set FOLLOW(NT), continue. Else; reject.
+ if can_be_followed_by_any(frag_spec.as_str()) {
+ continue
+ } else {
+ let next_token = match tokens.peek() {
+ // If T' closes a complex NT, replace T' with F
+ Some(&&TtToken(_, CloseDelim(_))) => follow.clone(),
+ Some(&&TtToken(_, ref tok)) => tok.clone(),
+ Some(&&TtSequence(sp, _)) => {
+ // Be conservative around sequences: to be
+ // more specific, we would need to
+ // consider FIRST sets, but also the
+ // possibility that the sequence occurred
+ // zero times (in which case we need to
+ // look at the token that follows the
+ // sequence, which may itself a sequence,
+ // and so on).
+ cx.span_err(sp,
+ &format!("`${0}:{1}` is followed by a \
+ sequence repetition, which is not \
+ allowed for `{1}` fragments",
+ name.as_str(), frag_spec.as_str())
);
- Eof
- },
- // die next iteration
- Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
- // else, we're at the end of the macro or sequence
- None => follow.clone()
- };
-
- let tok = if let TtToken(_, ref tok) = *token { tok } else { unreachable!() };
- // If T' is in the set FOLLOW(NT), continue. Else, reject.
- match (&next_token, is_in_follow(cx, &next_token, frag_spec.as_str())) {
- (_, Err(msg)) => {
- cx.span_err(sp, &msg);
- continue
+ Eof
+ },
+ // die next iteration
+ Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
+ // else, we're at the end of the macro or sequence
+ None => follow.clone()
+ };
+
+ let tok = if let TtToken(_, ref tok) = *token { tok } else { unreachable!() };
+
+ // If T' is in the set FOLLOW(NT), continue. Else, reject.
+ match (&next_token, is_in_follow(cx, &next_token, frag_spec.as_str())) {
+ (_, Err(msg)) => {
+ cx.span_err(sp, &msg);
+ continue
+ }
+ (&Eof, _) => return Some((sp, tok.clone())),
+ (_, Ok(true)) => continue,
+ (next, Ok(false)) => {
+ cx.span_err(sp, &format!("`${0}:{1}` is followed by `{2}`, which \
+ is not allowed for `{1}` fragments",
+ name.as_str(), frag_spec.as_str(),
+ token_to_string(next)));
+ continue
+ },
}
- (&Eof, _) => return Some((sp, tok.clone())),
- (_, Ok(true)) => continue,
- (next, Ok(false)) => {
- cx.span_err(sp, &format!("`${0}:{1}` is followed by `{2}`, which \
- is not allowed for `{1}` fragments",
- name.as_str(), frag_spec.as_str(),
- token_to_string(next)));
- continue
- },
}
},
TtSequence(sp, ref seq) => {
last
}
+/// True if a fragment of type `frag` can be followed by any sort of
+/// token. We use this (among other things) as a useful approximation
+/// for when `frag` can be followed by a repetition like `$(...)*` or
+/// `$(...)+`. In general, these can be a bit tricky to reason about,
+/// so we adopt a conservative position that says that any fragment
+/// specifier which consumes at most one token tree can be followed by
+/// a fragment specifier (indeed, these fragments can be followed by
+/// ANYTHING without fear of future compatibility hazards).
+fn can_be_followed_by_any(frag: &str) -> bool {
+ match frag {
+ "item" | // always terminated by `}` or `;`
+ "block" | // exactly one token tree
+ "ident" | // exactly one token tree
+ "meta" | // exactly one token tree
+ "tt" => // exactly one token tree
+ true,
+
+ _ =>
+ false,
+ }
+}
+
+/// True if `frag` can legally be followed by the token `tok`. For
+/// fragments that can consume an unbounded numbe of tokens, `tok`
+/// must be within a well-defined follow set. This is intended to
+/// guarantee future compatibility: for example, without this rule, if
+/// we expanded `expr` to include a new binary operator, we might
+/// break macros that were relying on that binary operator as a
+/// separator.
fn is_in_follow(_: &ExtCtxt, tok: &Token, frag: &str) -> Result<bool, String> {
if let &CloseDelim(_) = tok {
+ // closing a token tree can never be matched by any fragment;
+ // iow, we always require that `(` and `)` match, etc.
Ok(true)
} else {
match frag {
--- /dev/null
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// Regression test for issue #25436: permit token-trees to be followed
+// by sequences, enabling more general parsing.
+
+use self::Join::*;
+
+#[derive(Debug)]
+enum Join<A,B> {
+ Keep(A,B),
+ Skip(A,B),
+}
+
+macro_rules! parse_list {
+ ( < $a:expr; > $($b:tt)* ) => { Keep(parse_item!($a),parse_list!($($b)*)) };
+ ( $a:tt $($b:tt)* ) => { Skip(parse_item!($a), parse_list!($($b)*)) };
+ ( ) => { () };
+}
+
+macro_rules! parse_item {
+ ( $x:expr ) => { $x }
+}
+
+fn main() {
+ let list = parse_list!(<1;> 2 <3;> 4);
+ assert_eq!("Keep(1, Skip(2, Keep(3, Skip(4, ()))))",
+ format!("{:?}", list));
+}
projects / rust.git / commitdiff