1 // Copyright 2015 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 use ast::{self, TokenTree};
12 use codemap::{Span, DUMMY_SP};
13 use ext::base::{DummyResult, ExtCtxt, MacResult, SyntaxExtension};
14 use ext::base::{NormalTT, TTMacroExpander};
15 use ext::tt::macro_parser::{Success, Error, Failure};
16 use ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal};
17 use ext::tt::macro_parser::parse;
18 use parse::lexer::new_tt_reader;
19 use parse::parser::Parser;
20 use parse::token::{self, special_idents, gensym_ident, NtTT, Token};
21 use parse::token::Token::*;
25 use util::small_vector::SmallVector;
27 use std::cell::RefCell;
31 struct ParserAnyMacro<'a> {
32 parser: RefCell<Parser<'a>>,
34 /// Span of the expansion site of the macro this parser is for
36 /// The ident of the macro we're parsing
37 macro_ident: ast::Ident
40 impl<'a> ParserAnyMacro<'a> {
41 /// Make sure we don't have any tokens left to parse, so we don't
42 /// silently drop anything. `allow_semi` is so that "optional"
43 /// semicolons at the end of normal expressions aren't complained
44 /// about e.g. the semicolon in `macro_rules! kapow { () => {
45 /// panic!(); } }` doesn't get picked up by .parse_expr(), but it's
46 /// allowed to be there.
47 fn ensure_complete_parse(&self, allow_semi: bool, context: &str) {
48 let mut parser = self.parser.borrow_mut();
49 if allow_semi && parser.token == token::Semi {
50 panictry!(parser.bump())
52 if parser.token != token::Eof {
53 let token_str = parser.this_token_to_string();
54 let msg = format!("macro expansion ignores token `{}` and any \
57 let span = parser.span;
58 parser.span_err(span, &msg[..]);
60 let msg = format!("caused by the macro expansion here; the usage \
61 of `{}!` is likely invalid in {} context",
62 self.macro_ident, context);
63 parser.span_note(self.site_span, &msg[..]);
68 impl<'a> MacResult for ParserAnyMacro<'a> {
69 fn make_expr(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Expr>> {
70 let ret = panictry!(self.parser.borrow_mut().parse_expr());
71 self.ensure_complete_parse(true, "expression");
74 fn make_pat(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Pat>> {
75 let ret = panictry!(self.parser.borrow_mut().parse_pat());
76 self.ensure_complete_parse(false, "pattern");
79 fn make_items(self: Box<ParserAnyMacro<'a>>) -> Option<SmallVector<P<ast::Item>>> {
80 let mut ret = SmallVector::zero();
81 while let Some(item) = panictry!(self.parser.borrow_mut().parse_item()) {
84 self.ensure_complete_parse(false, "item");
88 fn make_impl_items(self: Box<ParserAnyMacro<'a>>)
89 -> Option<SmallVector<P<ast::ImplItem>>> {
90 let mut ret = SmallVector::zero();
92 let mut parser = self.parser.borrow_mut();
95 _ => ret.push(panictry!(parser.parse_impl_item()))
98 self.ensure_complete_parse(false, "item");
102 fn make_stmts(self: Box<ParserAnyMacro<'a>>)
103 -> Option<SmallVector<P<ast::Stmt>>> {
104 let mut ret = SmallVector::zero();
106 let mut parser = self.parser.borrow_mut();
109 _ => match parser.parse_stmt() {
110 Ok(maybe_stmt) => match maybe_stmt {
111 Some(stmt) => ret.push(stmt),
118 self.ensure_complete_parse(false, "statement");
122 fn make_ty(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Ty>> {
123 let ret = panictry!(self.parser.borrow_mut().parse_ty());
124 self.ensure_complete_parse(false, "type");
129 struct MacroRulesMacroExpander {
131 imported_from: Option<ast::Ident>,
132 lhses: Vec<TokenTree>,
133 rhses: Vec<TokenTree>,
137 impl TTMacroExpander for MacroRulesMacroExpander {
138 fn expand<'cx>(&self,
139 cx: &'cx mut ExtCtxt,
142 -> Box<MacResult+'cx> {
144 return DummyResult::any(sp);
146 generic_extension(cx,
156 /// Given `lhses` and `rhses`, this is the new macro we create
157 fn generic_extension<'cx>(cx: &'cx ExtCtxt,
160 imported_from: Option<ast::Ident>,
164 -> Box<MacResult+'cx> {
165 if cx.trace_macros() {
166 println!("{}! {{ {} }}",
168 print::pprust::tts_to_string(arg));
171 // Which arm's failure should we report? (the one furthest along)
172 let mut best_fail_spot = DUMMY_SP;
173 let mut best_fail_msg = "internal error: ran no matchers".to_string();
175 for (i, lhs) in lhses.iter().enumerate() { // try each arm's matchers
176 let lhs_tt = match *lhs {
177 TokenTree::Delimited(_, ref delim) => &delim.tts[..],
178 _ => cx.span_bug(sp, "malformed macro lhs")
181 match TokenTree::parse(cx, lhs_tt, arg) {
182 Success(named_matches) => {
183 let rhs = match rhses[i] {
185 TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(),
186 _ => cx.span_bug(sp, "malformed macro rhs"),
188 // rhs has holes ( `$id` and `$(...)` that need filled)
189 let trncbr = new_tt_reader(&cx.parse_sess().span_diagnostic,
193 let mut p = Parser::new(cx.parse_sess(), cx.cfg(), Box::new(trncbr));
194 panictry!(p.check_unknown_macro_variable());
195 // Let the context choose how to interpret the result.
196 // Weird, but useful for X-macros.
197 return Box::new(ParserAnyMacro {
198 parser: RefCell::new(p),
200 // Pass along the original expansion site and the name of the macro
201 // so we can print a useful error message if the parse of the expanded
202 // macro leaves unparsed tokens.
207 Failure(sp, ref msg) => if sp.lo >= best_fail_spot.lo {
209 best_fail_msg = (*msg).clone();
211 Error(err_sp, ref msg) => {
212 cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..])
217 cx.span_fatal(best_fail_spot.substitute_dummy(sp), &best_fail_msg[..]);
220 // Note that macro-by-example's input is also matched against a token tree:
221 // $( $lhs:tt => $rhs:tt );+
223 // Holy self-referential!
225 /// Converts a `macro_rules!` invocation into a syntax extension.
226 pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
227 def: &ast::MacroDef) -> SyntaxExtension {
229 let lhs_nm = gensym_ident("lhs");
230 let rhs_nm = gensym_ident("rhs");
232 // The pattern that macro_rules matches.
233 // The grammar for macro_rules! is:
234 // $( $lhs:tt => $rhs:tt );+
235 // ...quasiquoting this would be nice.
236 // These spans won't matter, anyways
237 let match_lhs_tok = MatchNt(lhs_nm, special_idents::tt, token::Plain, token::Plain);
238 let match_rhs_tok = MatchNt(rhs_nm, special_idents::tt, token::Plain, token::Plain);
239 let argument_gram = vec!(
240 TokenTree::Sequence(DUMMY_SP,
241 Rc::new(ast::SequenceRepetition {
243 TokenTree::Token(DUMMY_SP, match_lhs_tok),
244 TokenTree::Token(DUMMY_SP, token::FatArrow),
245 TokenTree::Token(DUMMY_SP, match_rhs_tok)],
246 separator: Some(token::Semi),
250 //to phase into semicolon-termination instead of
251 //semicolon-separation
252 TokenTree::Sequence(DUMMY_SP,
253 Rc::new(ast::SequenceRepetition {
254 tts: vec![TokenTree::Token(DUMMY_SP, token::Semi)],
261 // Parse the macro_rules! invocation (`none` is for no interpolations):
262 let arg_reader = new_tt_reader(&cx.parse_sess().span_diagnostic,
267 let argument_map = match parse(cx.parse_sess(),
272 Failure(sp, str) | Error(sp, str) => {
273 panic!(cx.parse_sess().span_diagnostic
274 .span_fatal(sp.substitute_dummy(def.span), &str[..]));
278 let mut valid = true;
280 // Extract the arguments:
281 let lhses = match **argument_map.get(&lhs_nm.name).unwrap() {
282 MatchedSeq(ref s, _) => {
283 s.iter().map(|m| match **m {
284 MatchedNonterminal(NtTT(ref tt)) => (**tt).clone(),
285 _ => cx.span_bug(def.span, "wrong-structured lhs")
288 _ => cx.span_bug(def.span, "wrong-structured lhs")
292 check_lhs_nt_follows(cx, lhs, def.span);
295 let rhses = match **argument_map.get(&rhs_nm.name).unwrap() {
296 MatchedSeq(ref s, _) => {
297 s.iter().map(|m| match **m {
298 MatchedNonterminal(NtTT(ref tt)) => (**tt).clone(),
299 _ => cx.span_bug(def.span, "wrong-structured rhs")
302 _ => cx.span_bug(def.span, "wrong-structured rhs")
306 valid &= check_rhs(cx, rhs);
309 let exp: Box<_> = Box::new(MacroRulesMacroExpander {
311 imported_from: def.imported_from,
317 NormalTT(exp, Some(def.span), def.allow_internal_unstable)
320 fn check_lhs_nt_follows(cx: &mut ExtCtxt, lhs: &TokenTree, sp: Span) {
321 // lhs is going to be like TokenTree::Delimited(...), where the
322 // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens.
324 &TokenTree::Delimited(_, ref tts) => {
325 check_matcher(cx, tts.tts.iter(), &Eof);
327 tt @ &TokenTree::Sequence(..) => {
328 check_matcher(cx, Some(tt).into_iter(), &Eof);
330 _ => cx.span_err(sp, "invalid macro matcher; matchers must be contained \
331 in balanced delimiters or a repetition indicator")
333 // we don't abort on errors on rejection, the driver will do that for us
334 // after parsing/expansion. we can report every error in every macro this way.
337 fn check_rhs(cx: &mut ExtCtxt, rhs: &TokenTree) -> bool {
339 TokenTree::Delimited(..) => return true,
340 _ => cx.span_err(rhs.get_span(), "macro rhs must be delimited")
345 // returns the last token that was checked, for TokenTree::Sequence. this gets used later on.
346 fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
347 -> Option<(Span, Token)> where I: Iterator<Item=&'a TokenTree> {
348 use print::pprust::token_to_string;
352 // 2. For each token T in M:
353 let mut tokens = matcher.peekable();
354 while let Some(token) = tokens.next() {
355 last = match *token {
356 TokenTree::Token(sp, MatchNt(ref name, ref frag_spec, _, _)) => {
357 // ii. If T is a simple NT, look ahead to the next token T' in
358 // M. If T' is in the set FOLLOW(NT), continue. Else; reject.
359 if can_be_followed_by_any(&frag_spec.name.as_str()) {
362 let next_token = match tokens.peek() {
363 // If T' closes a complex NT, replace T' with F
364 Some(&&TokenTree::Token(_, CloseDelim(_))) => follow.clone(),
365 Some(&&TokenTree::Token(_, ref tok)) => tok.clone(),
366 Some(&&TokenTree::Sequence(sp, _)) => {
367 // Be conservative around sequences: to be
368 // more specific, we would need to
369 // consider FIRST sets, but also the
370 // possibility that the sequence occurred
371 // zero times (in which case we need to
372 // look at the token that follows the
373 // sequence, which may itself be a sequence,
376 &format!("`${0}:{1}` is followed by a \
377 sequence repetition, which is not \
378 allowed for `{1}` fragments",
383 // die next iteration
384 Some(&&TokenTree::Delimited(_, ref delim)) => delim.close_token(),
385 // else, we're at the end of the macro or sequence
386 None => follow.clone()
389 let tok = if let TokenTree::Token(_, ref tok) = *token {
395 // If T' is in the set FOLLOW(NT), continue. Else, reject.
396 match (&next_token, is_in_follow(cx, &next_token, &frag_spec.name.as_str())) {
398 cx.span_err(sp, &msg);
401 (&Eof, _) => return Some((sp, tok.clone())),
402 (_, Ok(true)) => continue,
403 (next, Ok(false)) => {
404 cx.span_err(sp, &format!("`${0}:{1}` is followed by `{2}`, which \
405 is not allowed for `{1}` fragments",
407 token_to_string(next)));
413 TokenTree::Sequence(sp, ref seq) => {
414 // iii. Else, T is a complex NT.
415 match seq.separator {
416 // If T has the form $(...)U+ or $(...)U* for some token U,
417 // run the algorithm on the contents with F set to U. If it
418 // accepts, continue, else, reject.
420 let last = check_matcher(cx, seq.tts.iter(), u);
422 // Since the delimiter isn't required after the last
423 // repetition, make sure that the *next* token is
424 // sane. This doesn't actually compute the FIRST of
425 // the rest of the matcher yet, it only considers
426 // single tokens and simple NTs. This is imprecise,
427 // but conservatively correct.
428 Some((span, tok)) => {
429 let fol = match tokens.peek() {
430 Some(&&TokenTree::Token(_, ref tok)) => tok.clone(),
431 Some(&&TokenTree::Delimited(_, ref delim)) =>
434 cx.span_err(sp, "sequence repetition followed by \
435 another sequence repetition, which is not allowed");
440 check_matcher(cx, once(&TokenTree::Token(span, tok.clone())),
446 // If T has the form $(...)+ or $(...)*, run the algorithm
447 // on the contents with F set to the token following the
448 // sequence. If it accepts, continue, else, reject.
450 let fol = match tokens.peek() {
451 Some(&&TokenTree::Token(_, ref tok)) => tok.clone(),
452 Some(&&TokenTree::Delimited(_, ref delim)) => delim.close_token(),
454 cx.span_err(sp, "sequence repetition followed by another \
455 sequence repetition, which is not allowed");
460 check_matcher(cx, seq.tts.iter(), &fol)
464 TokenTree::Token(..) => {
465 // i. If T is not an NT, continue.
468 TokenTree::Delimited(_, ref tts) => {
469 // if we don't pass in that close delimiter, we'll incorrectly consider the matcher
470 // `{ $foo:ty }` as having a follow that isn't `RBrace`
471 check_matcher(cx, tts.tts.iter(), &tts.close_token())
478 /// True if a fragment of type `frag` can be followed by any sort of
479 /// token. We use this (among other things) as a useful approximation
480 /// for when `frag` can be followed by a repetition like `$(...)*` or
481 /// `$(...)+`. In general, these can be a bit tricky to reason about,
482 /// so we adopt a conservative position that says that any fragment
483 /// specifier which consumes at most one token tree can be followed by
484 /// a fragment specifier (indeed, these fragments can be followed by
485 /// ANYTHING without fear of future compatibility hazards).
486 fn can_be_followed_by_any(frag: &str) -> bool {
488 "item" | // always terminated by `}` or `;`
489 "block" | // exactly one token tree
490 "ident" | // exactly one token tree
491 "meta" | // exactly one token tree
492 "tt" => // exactly one token tree
500 /// True if `frag` can legally be followed by the token `tok`. For
501 /// fragments that can consume an unbounded numbe of tokens, `tok`
502 /// must be within a well-defined follow set. This is intended to
503 /// guarantee future compatibility: for example, without this rule, if
504 /// we expanded `expr` to include a new binary operator, we might
505 /// break macros that were relying on that binary operator as a
507 fn is_in_follow(_: &ExtCtxt, tok: &Token, frag: &str) -> Result<bool, String> {
508 if let &CloseDelim(_) = tok {
509 // closing a token tree can never be matched by any fragment;
510 // iow, we always require that `(` and `)` match, etc.
515 // since items *must* be followed by either a `;` or a `}`, we can
516 // accept anything after them
520 // anything can follow block, the braces provide an easy boundary to
526 FatArrow | Comma | Semi => Ok(true),
532 FatArrow | Comma | Eq => Ok(true),
533 Ident(i, _) if i.name.as_str() == "if" || i.name.as_str() == "in" => Ok(true),
539 Comma | FatArrow | Colon | Eq | Gt | Semi => Ok(true),
540 Ident(i, _) if i.name.as_str() == "as" => Ok(true),
545 // being a single token, idents are harmless
549 // being either a single token or a delimited sequence, tt is
553 _ => Err(format!("invalid fragment specifier `{}`", frag))