src/libsyntax/ext/expand.rs

   1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 use ast::{self, Block, Ident, NodeId, PatKind, Path};
  12 use ast::{MacStmtStyle, StmtKind, ItemKind};
  13 use attr::{self, HasAttrs};
  14 use codemap::{ExpnInfo, NameAndSpan, MacroBang, MacroAttribute, dummy_spanned, respan};
  15 use config::{is_test_or_bench, StripUnconfigured};
  16 use errors::FatalError;
  17 use ext::base::*;
  18 use ext::derive::{add_derived_markers, collect_derives};
  19 use ext::hygiene::{Mark, SyntaxContext};
  20 use ext::placeholders::{placeholder, PlaceholderExpander};
  21 use feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err};
  22 use fold;
  23 use fold::*;
  24 use parse::{DirectoryOwnership, PResult};
  25 use parse::token::{self, Token};
  26 use parse::parser::Parser;
  27 use ptr::P;
  28 use symbol::Symbol;
  29 use symbol::keywords;
  30 use syntax_pos::{Span, DUMMY_SP, FileName};
  31 use syntax_pos::hygiene::ExpnFormat;
  32 use tokenstream::{TokenStream, TokenTree};
  33 use util::small_vector::SmallVector;
  34 use visit::Visitor;
  35
  36 use std::collections::HashMap;
  37 use std::fs::File;
  38 use std::io::Read;
  39 use std::mem;
  40 use std::rc::Rc;
  41 use std::path::PathBuf;
  42
  43 macro_rules! expansions {
  44     ($($kind:ident: $ty:ty [$($vec:ident, $ty_elt:ty)*], $kind_name:expr, .$make:ident,
  45             $(.$fold:ident)*  $(lift .$fold_elt:ident)*,
  46             $(.$visit:ident)*  $(lift .$visit_elt:ident)*;)*) => {
  47         #[derive(Copy, Clone, PartialEq, Eq)]
  48         pub enum ExpansionKind { OptExpr, $( $kind, )*  }
  49         pub enum Expansion { OptExpr(Option<P<ast::Expr>>), $( $kind($ty), )* }
  50
  51         impl ExpansionKind {
  52             pub fn name(self) -> &'static str {
  53                 match self {
  54                     ExpansionKind::OptExpr => "expression",
  55                     $( ExpansionKind::$kind => $kind_name, )*
  56                 }
  57             }
  58
  59             fn make_from<'a>(self, result: Box<MacResult + 'a>) -> Option<Expansion> {
  60                 match self {
  61                     ExpansionKind::OptExpr => result.make_expr().map(Some).map(Expansion::OptExpr),
  62                     $( ExpansionKind::$kind => result.$make().map(Expansion::$kind), )*
  63                 }
  64             }
  65         }
  66
  67         impl Expansion {
  68             pub fn make_opt_expr(self) -> Option<P<ast::Expr>> {
  69                 match self {
  70                     Expansion::OptExpr(expr) => expr,
  71                     _ => panic!("Expansion::make_* called on the wrong kind of expansion"),
  72                 }
  73             }
  74             $( pub fn $make(self) -> $ty {
  75                 match self {
  76                     Expansion::$kind(ast) => ast,
  77                     _ => panic!("Expansion::make_* called on the wrong kind of expansion"),
  78                 }
  79             } )*
  80
  81             pub fn fold_with<F: Folder>(self, folder: &mut F) -> Self {
  82                 use self::Expansion::*;
  83                 match self {
  84                     OptExpr(expr) => OptExpr(expr.and_then(|expr| folder.fold_opt_expr(expr))),
  85                     $($( $kind(ast) => $kind(folder.$fold(ast)), )*)*
  86                     $($( $kind(ast) => {
  87                         $kind(ast.into_iter().flat_map(|ast| folder.$fold_elt(ast)).collect())
  88                     }, )*)*
  89                 }
  90             }
  91
  92             pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
  93                 match *self {
  94                     Expansion::OptExpr(Some(ref expr)) => visitor.visit_expr(expr),
  95                     Expansion::OptExpr(None) => {}
  96                     $($( Expansion::$kind(ref ast) => visitor.$visit(ast), )*)*
  97                     $($( Expansion::$kind(ref ast) => for ast in &ast[..] {
  98                         visitor.$visit_elt(ast);
  99                     }, )*)*
 100                 }
 101             }
 102         }
 103
 104         impl<'a, 'b> Folder for MacroExpander<'a, 'b> {
 105             fn fold_opt_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
 106                 self.expand(Expansion::OptExpr(Some(expr))).make_opt_expr()
 107             }
 108             $($(fn $fold(&mut self, node: $ty) -> $ty {
 109                 self.expand(Expansion::$kind(node)).$make()
 110             })*)*
 111             $($(fn $fold_elt(&mut self, node: $ty_elt) -> $ty {
 112                 self.expand(Expansion::$kind(SmallVector::one(node))).$make()
 113             })*)*
 114         }
 115
 116         impl<'a> MacResult for ::ext::tt::macro_rules::ParserAnyMacro<'a> {
 117             $(fn $make(self: Box<::ext::tt::macro_rules::ParserAnyMacro<'a>>) -> Option<$ty> {
 118                 Some(self.make(ExpansionKind::$kind).$make())
 119             })*
 120         }
 121     }
 122 }
 123
 124 expansions! {
 125     Expr: P<ast::Expr> [], "expression", .make_expr, .fold_expr, .visit_expr;
 126     Pat: P<ast::Pat>   [], "pattern",    .make_pat,  .fold_pat,  .visit_pat;
 127     Ty: P<ast::Ty>     [], "type",       .make_ty,   .fold_ty,   .visit_ty;
 128     Stmts: SmallVector<ast::Stmt> [SmallVector, ast::Stmt],
 129         "statement",  .make_stmts,       lift .fold_stmt, lift .visit_stmt;
 130     Items: SmallVector<P<ast::Item>> [SmallVector, P<ast::Item>],
 131         "item",       .make_items,       lift .fold_item, lift .visit_item;
 132     TraitItems: SmallVector<ast::TraitItem> [SmallVector, ast::TraitItem],
 133         "trait item", .make_trait_items, lift .fold_trait_item, lift .visit_trait_item;
 134     ImplItems: SmallVector<ast::ImplItem> [SmallVector, ast::ImplItem],
 135         "impl item",  .make_impl_items,  lift .fold_impl_item,  lift .visit_impl_item;
 136     ForeignItems: SmallVector<ast::ForeignItem> [SmallVector, ast::ForeignItem],
 137         "foreign item", .make_foreign_items, lift .fold_foreign_item, lift .visit_foreign_item;
 138 }
 139
 140 impl ExpansionKind {
 141     fn dummy(self, span: Span) -> Option<Expansion> {
 142         self.make_from(DummyResult::any(span))
 143     }
 144
 145     fn expect_from_annotatables<I: IntoIterator<Item = Annotatable>>(self, items: I) -> Expansion {
 146         let mut items = items.into_iter();
 147         match self {
 148             ExpansionKind::Items =>
 149                 Expansion::Items(items.map(Annotatable::expect_item).collect()),
 150             ExpansionKind::ImplItems =>
 151                 Expansion::ImplItems(items.map(Annotatable::expect_impl_item).collect()),
 152             ExpansionKind::TraitItems =>
 153                 Expansion::TraitItems(items.map(Annotatable::expect_trait_item).collect()),
 154             ExpansionKind::ForeignItems =>
 155                 Expansion::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()),
 156             ExpansionKind::Stmts => Expansion::Stmts(items.map(Annotatable::expect_stmt).collect()),
 157             ExpansionKind::Expr => Expansion::Expr(
 158                 items.next().expect("expected exactly one expression").expect_expr()
 159             ),
 160             ExpansionKind::OptExpr =>
 161                 Expansion::OptExpr(items.next().map(Annotatable::expect_expr)),
 162             ExpansionKind::Pat | ExpansionKind::Ty =>
 163                 panic!("patterns and types aren't annotatable"),
 164         }
 165     }
 166 }
 167
 168 fn macro_bang_format(path: &ast::Path) -> ExpnFormat {
 169     // We don't want to format a path using pretty-printing,
 170     // `format!("{}", path)`, because that tries to insert
 171     // line-breaks and is slow.
 172     let mut path_str = String::with_capacity(64);
 173     for (i, segment) in path.segments.iter().enumerate() {
 174         if i != 0 {
 175             path_str.push_str("::");
 176         }
 177
 178         if segment.ident.name != keywords::CrateRoot.name() &&
 179             segment.ident.name != keywords::DollarCrate.name()
 180         {
 181             path_str.push_str(&segment.ident.name.as_str())
 182         }
 183     }
 184
 185     MacroBang(Symbol::intern(&path_str))
 186 }
 187
 188 pub struct Invocation {
 189     pub kind: InvocationKind,
 190     expansion_kind: ExpansionKind,
 191     pub expansion_data: ExpansionData,
 192 }
 193
 194 pub enum InvocationKind {
 195     Bang {
 196         mac: ast::Mac,
 197         ident: Option<Ident>,
 198         span: Span,
 199     },
 200     Attr {
 201         attr: Option<ast::Attribute>,
 202         traits: Vec<Path>,
 203         item: Annotatable,
 204     },
 205     Derive {
 206         path: Path,
 207         item: Annotatable,
 208     },
 209 }
 210
 211 impl Invocation {
 212     fn span(&self) -> Span {
 213         match self.kind {
 214             InvocationKind::Bang { span, .. } => span,
 215             InvocationKind::Attr { attr: Some(ref attr), .. } => attr.span,
 216             InvocationKind::Attr { attr: None, .. } => DUMMY_SP,
 217             InvocationKind::Derive { ref path, .. } => path.span,
 218         }
 219     }
 220 }
 221
 222 pub struct MacroExpander<'a, 'b:'a> {
 223     pub cx: &'a mut ExtCtxt<'b>,
 224     monotonic: bool, // c.f. `cx.monotonic_expander()`
 225 }
 226
 227 impl<'a, 'b> MacroExpander<'a, 'b> {
 228     pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self {
 229         MacroExpander { cx: cx, monotonic: monotonic }
 230     }
 231
 232     pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate {
 233         let mut module = ModuleData {
 234             mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)],
 235             directory: match self.cx.codemap().span_to_unmapped_path(krate.span) {
 236                 FileName::Real(path) => path,
 237                 other => PathBuf::from(other.to_string()),
 238             },
 239         };
 240         module.directory.pop();
 241         self.cx.root_path = module.directory.clone();
 242         self.cx.current_expansion.module = Rc::new(module);
 243         self.cx.current_expansion.crate_span = Some(krate.span);
 244
 245         let orig_mod_span = krate.module.inner;
 246
 247         let krate_item = Expansion::Items(SmallVector::one(P(ast::Item {
 248             attrs: krate.attrs,
 249             span: krate.span,
 250             node: ast::ItemKind::Mod(krate.module),
 251             ident: keywords::Invalid.ident(),
 252             id: ast::DUMMY_NODE_ID,
 253             vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public),
 254             tokens: None,
 255         })));
 256
 257         match self.expand(krate_item).make_items().pop().map(P::into_inner) {
 258             Some(ast::Item { attrs, node: ast::ItemKind::Mod(module), .. }) => {
 259                 krate.attrs = attrs;
 260                 krate.module = module;
 261             },
 262             None => {
 263                 // Resolution failed so we return an empty expansion
 264                 krate.attrs = vec![];
 265                 krate.module = ast::Mod {
 266                     inner: orig_mod_span,
 267                     items: vec![],
 268                 };
 269             },
 270             _ => unreachable!(),
 271         };
 272         self.cx.trace_macros_diag();
 273         krate
 274     }
 275
 276     // Fully expand all the invocations in `expansion`.
 277     fn expand(&mut self, expansion: Expansion) -> Expansion {
 278         let orig_expansion_data = self.cx.current_expansion.clone();
 279         self.cx.current_expansion.depth = 0;
 280
 281         let (expansion, mut invocations) = self.collect_invocations(expansion, &[]);
 282         self.resolve_imports();
 283         invocations.reverse();
 284
 285         let mut expansions = Vec::new();
 286         let mut derives = HashMap::new();
 287         let mut undetermined_invocations = Vec::new();
 288         let (mut progress, mut force) = (false, !self.monotonic);
 289         loop {
 290             let mut invoc = if let Some(invoc) = invocations.pop() {
 291                 invoc
 292             } else {
 293                 self.resolve_imports();
 294                 if undetermined_invocations.is_empty() { break }
 295                 invocations = mem::replace(&mut undetermined_invocations, Vec::new());
 296                 force = !mem::replace(&mut progress, false);
 297                 continue
 298             };
 299
 300             let scope =
 301                 if self.monotonic { invoc.expansion_data.mark } else { orig_expansion_data.mark };
 302             let ext = match self.cx.resolver.resolve_invoc(&mut invoc, scope, force) {
 303                 Ok(ext) => Some(ext),
 304                 Err(Determinacy::Determined) => None,
 305                 Err(Determinacy::Undetermined) => {
 306                     undetermined_invocations.push(invoc);
 307                     continue
 308                 }
 309             };
 310
 311             progress = true;
 312             let ExpansionData { depth, mark, .. } = invoc.expansion_data;
 313             self.cx.current_expansion = invoc.expansion_data.clone();
 314
 315             self.cx.current_expansion.mark = scope;
 316             // FIXME(jseyfried): Refactor out the following logic
 317             let (expansion, new_invocations) = if let Some(ext) = ext {
 318                 if let Some(ext) = ext {
 319                     let dummy = invoc.expansion_kind.dummy(invoc.span()).unwrap();
 320                     let expansion = self.expand_invoc(invoc, &*ext).unwrap_or(dummy);
 321                     self.collect_invocations(expansion, &[])
 322                 } else if let InvocationKind::Attr { attr: None, traits, item } = invoc.kind {
 323                     if !item.derive_allowed() {
 324                         let attr = attr::find_by_name(item.attrs(), "derive")
 325                             .expect("`derive` attribute should exist");
 326                         let span = attr.span;
 327                         let mut err = self.cx.mut_span_err(span,
 328                                                            "`derive` may only be applied to \
 329                                                             structs, enums and unions");
 330                         if let ast::AttrStyle::Inner = attr.style {
 331                             let trait_list = traits.iter()
 332                                 .map(|t| format!("{}", t)).collect::<Vec<_>>();
 333                             let suggestion = format!("#[derive({})]", trait_list.join(", "));
 334                             err.span_suggestion(span, "try an outer attribute", suggestion);
 335                         }
 336                         err.emit();
 337                     }
 338
 339                     let item = self.fully_configure(item)
 340                         .map_attrs(|mut attrs| { attrs.retain(|a| a.path != "derive"); attrs });
 341                     let item_with_markers =
 342                         add_derived_markers(&mut self.cx, item.span(), &traits, item.clone());
 343                     let derives = derives.entry(invoc.expansion_data.mark).or_insert_with(Vec::new);
 344
 345                     for path in &traits {
 346                         let mark = Mark::fresh(self.cx.current_expansion.mark);
 347                         derives.push(mark);
 348                         let item = match self.cx.resolver.resolve_macro(
 349                                 Mark::root(), path, MacroKind::Derive, false) {
 350                             Ok(ext) => match *ext {
 351                                 BuiltinDerive(..) => item_with_markers.clone(),
 352                                 _ => item.clone(),
 353                             },
 354                             _ => item.clone(),
 355                         };
 356                         invocations.push(Invocation {
 357                             kind: InvocationKind::Derive { path: path.clone(), item: item },
 358                             expansion_kind: invoc.expansion_kind,
 359                             expansion_data: ExpansionData {
 360                                 mark,
 361                                 ..invoc.expansion_data.clone()
 362                             },
 363                         });
 364                     }
 365                     let expansion = invoc.expansion_kind
 366                         .expect_from_annotatables(::std::iter::once(item_with_markers));
 367                     self.collect_invocations(expansion, derives)
 368                 } else {
 369                     unreachable!()
 370                 }
 371             } else {
 372                 self.collect_invocations(invoc.expansion_kind.dummy(invoc.span()).unwrap(), &[])
 373             };
 374
 375             if expansions.len() < depth {
 376                 expansions.push(Vec::new());
 377             }
 378             expansions[depth - 1].push((mark, expansion));
 379             if !self.cx.ecfg.single_step {
 380                 invocations.extend(new_invocations.into_iter().rev());
 381             }
 382         }
 383
 384         self.cx.current_expansion = orig_expansion_data;
 385
 386         let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic);
 387         while let Some(expansions) = expansions.pop() {
 388             for (mark, expansion) in expansions.into_iter().rev() {
 389                 let derives = derives.remove(&mark).unwrap_or_else(Vec::new);
 390                 placeholder_expander.add(NodeId::placeholder_from_mark(mark), expansion, derives);
 391             }
 392         }
 393
 394         expansion.fold_with(&mut placeholder_expander)
 395     }
 396
 397     fn resolve_imports(&mut self) {
 398         if self.monotonic {
 399             let err_count = self.cx.parse_sess.span_diagnostic.err_count();
 400             self.cx.resolver.resolve_imports();
 401             self.cx.resolve_err_count += self.cx.parse_sess.span_diagnostic.err_count() - err_count;
 402         }
 403     }
 404
 405     fn collect_invocations(&mut self, expansion: Expansion, derives: &[Mark])
 406                            -> (Expansion, Vec<Invocation>) {
 407         let result = {
 408             let mut collector = InvocationCollector {
 409                 cfg: StripUnconfigured {
 410                     should_test: self.cx.ecfg.should_test,
 411                     sess: self.cx.parse_sess,
 412                     features: self.cx.ecfg.features,
 413                 },
 414                 cx: self.cx,
 415                 invocations: Vec::new(),
 416                 monotonic: self.monotonic,
 417             };
 418             (expansion.fold_with(&mut collector), collector.invocations)
 419         };
 420
 421         if self.monotonic {
 422             let err_count = self.cx.parse_sess.span_diagnostic.err_count();
 423             let mark = self.cx.current_expansion.mark;
 424             self.cx.resolver.visit_expansion(mark, &result.0, derives);
 425             self.cx.resolve_err_count += self.cx.parse_sess.span_diagnostic.err_count() - err_count;
 426         }
 427
 428         result
 429     }
 430
 431     fn fully_configure(&mut self, item: Annotatable) -> Annotatable {
 432         let mut cfg = StripUnconfigured {
 433             should_test: self.cx.ecfg.should_test,
 434             sess: self.cx.parse_sess,
 435             features: self.cx.ecfg.features,
 436         };
 437         // Since the item itself has already been configured by the InvocationCollector,
 438         // we know that fold result vector will contain exactly one element
 439         match item {
 440             Annotatable::Item(item) => {
 441                 Annotatable::Item(cfg.fold_item(item).pop().unwrap())
 442             }
 443             Annotatable::TraitItem(item) => {
 444                 Annotatable::TraitItem(item.map(|item| cfg.fold_trait_item(item).pop().unwrap()))
 445             }
 446             Annotatable::ImplItem(item) => {
 447                 Annotatable::ImplItem(item.map(|item| cfg.fold_impl_item(item).pop().unwrap()))
 448             }
 449             Annotatable::ForeignItem(item) => {
 450                 Annotatable::ForeignItem(
 451                     item.map(|item| cfg.fold_foreign_item(item).pop().unwrap())
 452                 )
 453             }
 454             Annotatable::Stmt(stmt) => {
 455                 Annotatable::Stmt(stmt.map(|stmt| cfg.fold_stmt(stmt).pop().unwrap()))
 456             }
 457             Annotatable::Expr(expr) => {
 458                 Annotatable::Expr(cfg.fold_expr(expr))
 459             }
 460         }
 461     }
 462
 463     fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtension) -> Option<Expansion> {
 464         let result = match invoc.kind {
 465             InvocationKind::Bang { .. } => self.expand_bang_invoc(invoc, ext)?,
 466             InvocationKind::Attr { .. } => self.expand_attr_invoc(invoc, ext)?,
 467             InvocationKind::Derive { .. } => self.expand_derive_invoc(invoc, ext)?,
 468         };
 469
 470         if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit {
 471             let info = self.cx.current_expansion.mark.expn_info().unwrap();
 472             let suggested_limit = self.cx.ecfg.recursion_limit * 2;
 473             let mut err = self.cx.struct_span_err(info.call_site,
 474                 &format!("recursion limit reached while expanding the macro `{}`",
 475                          info.callee.name()));
 476             err.help(&format!(
 477                 "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate",
 478                 suggested_limit));
 479             err.emit();
 480             self.cx.trace_macros_diag();
 481             FatalError.raise();
 482         }
 483
 484         Some(result)
 485     }
 486
 487     fn expand_attr_invoc(&mut self,
 488                          invoc: Invocation,
 489                          ext: &SyntaxExtension)
 490                          -> Option<Expansion> {
 491         let Invocation { expansion_kind: kind, .. } = invoc;
 492         let (attr, item) = match invoc.kind {
 493             InvocationKind::Attr { attr, item, .. } => (attr?, item),
 494             _ => unreachable!(),
 495         };
 496
 497         attr::mark_used(&attr);
 498         invoc.expansion_data.mark.set_expn_info(ExpnInfo {
 499             call_site: attr.span,
 500             callee: NameAndSpan {
 501                 format: MacroAttribute(Symbol::intern(&format!("{}", attr.path))),
 502                 span: None,
 503                 allow_internal_unstable: false,
 504                 allow_internal_unsafe: false,
 505             }
 506         });
 507
 508         match *ext {
 509             MultiModifier(ref mac) => {
 510                 let meta = attr.parse_meta(self.cx.parse_sess)
 511                                .map_err(|mut e| { e.emit(); }).ok()?;
 512                 let item = mac.expand(self.cx, attr.span, &meta, item);
 513                 Some(kind.expect_from_annotatables(item))
 514             }
 515             MultiDecorator(ref mac) => {
 516                 let mut items = Vec::new();
 517                 let meta = attr.parse_meta(self.cx.parse_sess)
 518                                .expect("derive meta should already have been parsed");
 519                 mac.expand(self.cx, attr.span, &meta, &item, &mut |item| items.push(item));
 520                 items.push(item);
 521                 Some(kind.expect_from_annotatables(items))
 522             }
 523             AttrProcMacro(ref mac) => {
 524                 self.gate_proc_macro_attr_item(attr.span, &item);
 525                 let item_tok = TokenTree::Token(DUMMY_SP, Token::interpolated(match item {
 526                     Annotatable::Item(item) => token::NtItem(item),
 527                     Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
 528                     Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),
 529                     Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()),
 530                     Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
 531                     Annotatable::Expr(expr) => token::NtExpr(expr),
 532                 })).into();
 533                 let input = self.extract_proc_macro_attr_input(attr.tokens, attr.span);
 534                 let tok_result = mac.expand(self.cx, attr.span, input, item_tok);
 535                 self.parse_expansion(tok_result, kind, &attr.path, attr.span)
 536             }
 537             ProcMacroDerive(..) | BuiltinDerive(..) => {
 538                 self.cx.span_err(attr.span, &format!("`{}` is a derive mode", attr.path));
 539                 self.cx.trace_macros_diag();
 540                 kind.dummy(attr.span)
 541             }
 542             _ => {
 543                 let msg = &format!("macro `{}` may not be used in attributes", attr.path);
 544                 self.cx.span_err(attr.span, msg);
 545                 self.cx.trace_macros_diag();
 546                 kind.dummy(attr.span)
 547             }
 548         }
 549     }
 550
 551     fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
 552         let mut trees = tokens.trees();
 553         match trees.next() {
 554             Some(TokenTree::Delimited(_, delim)) => {
 555                 if trees.next().is_none() {
 556                     return delim.tts.into()
 557                 }
 558             }
 559             Some(TokenTree::Token(..)) => {}
 560             None => return TokenStream::empty(),
 561         }
 562         self.cx.span_err(span, "custom attribute invocations must be \
 563             of the form #[foo] or #[foo(..)], the macro name must only be \
 564             followed by a delimiter token");
 565         TokenStream::empty()
 566     }
 567
 568     fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) {
 569         let (kind, gate) = match *item {
 570             Annotatable::Item(ref item) => {
 571                 match item.node {
 572                     ItemKind::Mod(_) if self.cx.ecfg.proc_macro_mod() => return,
 573                     ItemKind::Mod(_) => ("modules", "proc_macro_mod"),
 574                     _ => return,
 575                 }
 576             }
 577             Annotatable::TraitItem(_) => return,
 578             Annotatable::ImplItem(_) => return,
 579             Annotatable::ForeignItem(_) => return,
 580             Annotatable::Stmt(_) |
 581             Annotatable::Expr(_) if self.cx.ecfg.proc_macro_expr() => return,
 582             Annotatable::Stmt(_) => ("statements", "proc_macro_expr"),
 583             Annotatable::Expr(_) => ("expressions", "proc_macro_expr"),
 584         };
 585         emit_feature_err(
 586             self.cx.parse_sess,
 587             gate,
 588             span,
 589             GateIssue::Language,
 590             &format!("custom attributes cannot be applied to {}", kind),
 591         );
 592     }
 593
 594     /// Expand a macro invocation. Returns the result of expansion.
 595     fn expand_bang_invoc(&mut self,
 596                          invoc: Invocation,
 597                          ext: &SyntaxExtension)
 598                          -> Option<Expansion> {
 599         let (mark, kind) = (invoc.expansion_data.mark, invoc.expansion_kind);
 600         let (mac, ident, span) = match invoc.kind {
 601             InvocationKind::Bang { mac, ident, span } => (mac, ident, span),
 602             _ => unreachable!(),
 603         };
 604         let path = &mac.node.path;
 605
 606         let ident = ident.unwrap_or_else(|| keywords::Invalid.ident());
 607         let validate_and_set_expn_info = |this: &mut Self, // arg instead of capture
 608                                           def_site_span: Option<Span>,
 609                                           allow_internal_unstable,
 610                                           allow_internal_unsafe,
 611                                           // can't infer this type
 612                                           unstable_feature: Option<(Symbol, u32)>| {
 613
 614             // feature-gate the macro invocation
 615             if let Some((feature, issue)) = unstable_feature {
 616                 let crate_span = this.cx.current_expansion.crate_span.unwrap();
 617                 // don't stability-check macros in the same crate
 618                 // (the only time this is null is for syntax extensions registered as macros)
 619                 if def_site_span.map_or(false, |def_span| !crate_span.contains(def_span))
 620                     && !span.allows_unstable() && this.cx.ecfg.features.map_or(true, |feats| {
 621                     // macro features will count as lib features
 622                     !feats.declared_lib_features.iter().any(|&(feat, _)| feat == feature)
 623                 }) {
 624                     let explain = format!("macro {}! is unstable", path);
 625                     emit_feature_err(this.cx.parse_sess, &*feature.as_str(), span,
 626                                      GateIssue::Library(Some(issue)), &explain);
 627                     this.cx.trace_macros_diag();
 628                     return Err(kind.dummy(span));
 629                 }
 630             }
 631
 632             if ident.name != keywords::Invalid.name() {
 633                 let msg = format!("macro {}! expects no ident argument, given '{}'", path, ident);
 634                 this.cx.span_err(path.span, &msg);
 635                 this.cx.trace_macros_diag();
 636                 return Err(kind.dummy(span));
 637             }
 638             mark.set_expn_info(ExpnInfo {
 639                 call_site: span,
 640                 callee: NameAndSpan {
 641                     format: macro_bang_format(path),
 642                     span: def_site_span,
 643                     allow_internal_unstable,
 644                     allow_internal_unsafe,
 645                 },
 646             });
 647             Ok(())
 648         };
 649
 650         let opt_expanded = match *ext {
 651             DeclMacro(ref expand, def_span) => {
 652                 if let Err(dummy_span) = validate_and_set_expn_info(self, def_span.map(|(_, s)| s),
 653                                                                     false, false, None) {
 654                     dummy_span
 655                 } else {
 656                     kind.make_from(expand.expand(self.cx, span, mac.node.stream()))
 657                 }
 658             }
 659
 660             NormalTT {
 661                 ref expander,
 662                 def_info,
 663                 allow_internal_unstable,
 664                 allow_internal_unsafe,
 665                 unstable_feature,
 666             } => {
 667                 if let Err(dummy_span) = validate_and_set_expn_info(self, def_info.map(|(_, s)| s),
 668                                                                     allow_internal_unstable,
 669                                                                     allow_internal_unsafe,
 670                                                                     unstable_feature) {
 671                     dummy_span
 672                 } else {
 673                     kind.make_from(expander.expand(self.cx, span, mac.node.stream()))
 674                 }
 675             }
 676
 677             IdentTT(ref expander, tt_span, allow_internal_unstable) => {
 678                 if ident.name == keywords::Invalid.name() {
 679                     self.cx.span_err(path.span,
 680                                     &format!("macro {}! expects an ident argument", path));
 681                     self.cx.trace_macros_diag();
 682                     kind.dummy(span)
 683                 } else {
 684                     invoc.expansion_data.mark.set_expn_info(ExpnInfo {
 685                         call_site: span,
 686                         callee: NameAndSpan {
 687                             format: macro_bang_format(path),
 688                             span: tt_span,
 689                             allow_internal_unstable,
 690                             allow_internal_unsafe: false,
 691                         }
 692                     });
 693
 694                     let input: Vec<_> = mac.node.stream().into_trees().collect();
 695                     kind.make_from(expander.expand(self.cx, span, ident, input))
 696                 }
 697             }
 698
 699             MultiDecorator(..) | MultiModifier(..) | AttrProcMacro(..) => {
 700                 self.cx.span_err(path.span,
 701                                  &format!("`{}` can only be used in attributes", path));
 702                 self.cx.trace_macros_diag();
 703                 kind.dummy(span)
 704             }
 705
 706             ProcMacroDerive(..) | BuiltinDerive(..) => {
 707                 self.cx.span_err(path.span, &format!("`{}` is a derive mode", path));
 708                 self.cx.trace_macros_diag();
 709                 kind.dummy(span)
 710             }
 711
 712             ProcMacro(ref expandfun) => {
 713                 if ident.name != keywords::Invalid.name() {
 714                     let msg =
 715                         format!("macro {}! expects no ident argument, given '{}'", path, ident);
 716                     self.cx.span_err(path.span, &msg);
 717                     self.cx.trace_macros_diag();
 718                     kind.dummy(span)
 719                 } else {
 720                     self.gate_proc_macro_expansion_kind(span, kind);
 721                     invoc.expansion_data.mark.set_expn_info(ExpnInfo {
 722                         call_site: span,
 723                         callee: NameAndSpan {
 724                             format: macro_bang_format(path),
 725                             // FIXME procedural macros do not have proper span info
 726                             // yet, when they do, we should use it here.
 727                             span: None,
 728                             // FIXME probably want to follow macro_rules macros here.
 729                             allow_internal_unstable: false,
 730                             allow_internal_unsafe: false,
 731                         },
 732                     });
 733
 734                     let tok_result = expandfun.expand(self.cx, span, mac.node.stream());
 735                     self.parse_expansion(tok_result, kind, path, span)
 736                 }
 737             }
 738         };
 739
 740         if opt_expanded.is_some() {
 741             opt_expanded
 742         } else {
 743             let msg = format!("non-{kind} macro in {kind} position: {name}",
 744                               name = path.segments[0].ident.name, kind = kind.name());
 745             self.cx.span_err(path.span, &msg);
 746             self.cx.trace_macros_diag();
 747             kind.dummy(span)
 748         }
 749     }
 750
 751     fn gate_proc_macro_expansion_kind(&self, span: Span, kind: ExpansionKind) {
 752         let kind = match kind {
 753             ExpansionKind::Expr => "expressions",
 754             ExpansionKind::OptExpr => "expressions",
 755             ExpansionKind::Pat => "patterns",
 756             ExpansionKind::Ty => "types",
 757             ExpansionKind::Stmts => "statements",
 758             ExpansionKind::Items => return,
 759             ExpansionKind::TraitItems => return,
 760             ExpansionKind::ImplItems => return,
 761             ExpansionKind::ForeignItems => return,
 762         };
 763         if self.cx.ecfg.proc_macro_non_items() {
 764             return
 765         }
 766         emit_feature_err(
 767             self.cx.parse_sess,
 768             "proc_macro_non_items",
 769             span,
 770             GateIssue::Language,
 771             &format!("procedural macros cannot be expanded to {}", kind),
 772         );
 773     }
 774
 775     /// Expand a derive invocation. Returns the result of expansion.
 776     fn expand_derive_invoc(&mut self,
 777                            invoc: Invocation,
 778                            ext: &SyntaxExtension)
 779                            -> Option<Expansion> {
 780         let Invocation { expansion_kind: kind, .. } = invoc;
 781         let (path, item) = match invoc.kind {
 782             InvocationKind::Derive { path, item } => (path, item),
 783             _ => unreachable!(),
 784         };
 785         if !item.derive_allowed() {
 786             return None;
 787         }
 788
 789         let pretty_name = Symbol::intern(&format!("derive({})", path));
 790         let span = path.span;
 791         let attr = ast::Attribute {
 792             path, span,
 793             tokens: TokenStream::empty(),
 794             // irrelevant:
 795             id: ast::AttrId(0), style: ast::AttrStyle::Outer, is_sugared_doc: false,
 796         };
 797
 798         let mut expn_info = ExpnInfo {
 799             call_site: span,
 800             callee: NameAndSpan {
 801                 format: MacroAttribute(pretty_name),
 802                 span: None,
 803                 allow_internal_unstable: false,
 804                 allow_internal_unsafe: false,
 805             }
 806         };
 807
 808         match *ext {
 809             ProcMacroDerive(ref ext, _) => {
 810                 invoc.expansion_data.mark.set_expn_info(expn_info);
 811                 let span = span.with_ctxt(self.cx.backtrace());
 812                 let dummy = ast::MetaItem { // FIXME(jseyfried) avoid this
 813                     name: Path::from_ident(DUMMY_SP, keywords::Invalid.ident()),
 814                     span: DUMMY_SP,
 815                     node: ast::MetaItemKind::Word,
 816                 };
 817                 Some(kind.expect_from_annotatables(ext.expand(self.cx, span, &dummy, item)))
 818             }
 819             BuiltinDerive(func) => {
 820                 expn_info.callee.allow_internal_unstable = true;
 821                 invoc.expansion_data.mark.set_expn_info(expn_info);
 822                 let span = span.with_ctxt(self.cx.backtrace());
 823                 let mut items = Vec::new();
 824                 func(self.cx, span, &attr.meta()?, &item, &mut |a| items.push(a));
 825                 Some(kind.expect_from_annotatables(items))
 826             }
 827             _ => {
 828                 let msg = &format!("macro `{}` may not be used for derive attributes", attr.path);
 829                 self.cx.span_err(span, msg);
 830                 self.cx.trace_macros_diag();
 831                 kind.dummy(span)
 832             }
 833         }
 834     }
 835
 836     fn parse_expansion(&mut self,
 837                        toks: TokenStream,
 838                        kind: ExpansionKind,
 839                        path: &Path,
 840                        span: Span)
 841                        -> Option<Expansion> {
 842         let mut parser = self.cx.new_parser_from_tts(&toks.into_trees().collect::<Vec<_>>());
 843         match parser.parse_expansion(kind, false) {
 844             Ok(expansion) => {
 845                 parser.ensure_complete_parse(path, kind.name(), span);
 846                 Some(expansion)
 847             }
 848             Err(mut err) => {
 849                 err.set_span(span);
 850                 err.emit();
 851                 self.cx.trace_macros_diag();
 852                 kind.dummy(span)
 853             }
 854         }
 855     }
 856 }
 857
 858 impl<'a> Parser<'a> {
 859     pub fn parse_expansion(&mut self, kind: ExpansionKind, macro_legacy_warnings: bool)
 860                            -> PResult<'a, Expansion> {
 861         Ok(match kind {
 862             ExpansionKind::Items => {
 863                 let mut items = SmallVector::new();
 864                 while let Some(item) = self.parse_item()? {
 865                     items.push(item);
 866                 }
 867                 Expansion::Items(items)
 868             }
 869             ExpansionKind::TraitItems => {
 870                 let mut items = SmallVector::new();
 871                 while self.token != token::Eof {
 872                     items.push(self.parse_trait_item(&mut false)?);
 873                 }
 874                 Expansion::TraitItems(items)
 875             }
 876             ExpansionKind::ImplItems => {
 877                 let mut items = SmallVector::new();
 878                 while self.token != token::Eof {
 879                     items.push(self.parse_impl_item(&mut false)?);
 880                 }
 881                 Expansion::ImplItems(items)
 882             }
 883             ExpansionKind::ForeignItems => {
 884                 let mut items = SmallVector::new();
 885                 while self.token != token::Eof {
 886                     if let Some(item) = self.parse_foreign_item()? {
 887                         items.push(item);
 888                     }
 889                 }
 890                 Expansion::ForeignItems(items)
 891             }
 892             ExpansionKind::Stmts => {
 893                 let mut stmts = SmallVector::new();
 894                 while self.token != token::Eof &&
 895                       // won't make progress on a `}`
 896                       self.token != token::CloseDelim(token::Brace) {
 897                     if let Some(stmt) = self.parse_full_stmt(macro_legacy_warnings)? {
 898                         stmts.push(stmt);
 899                     }
 900                 }
 901                 Expansion::Stmts(stmts)
 902             }
 903             ExpansionKind::Expr => Expansion::Expr(self.parse_expr()?),
 904             ExpansionKind::OptExpr => {
 905                 if self.token != token::Eof {
 906                     Expansion::OptExpr(Some(self.parse_expr()?))
 907                 } else {
 908                     Expansion::OptExpr(None)
 909                 }
 910             },
 911             ExpansionKind::Ty => Expansion::Ty(self.parse_ty()?),
 912             ExpansionKind::Pat => Expansion::Pat(self.parse_pat()?),
 913         })
 914     }
 915
 916     pub fn ensure_complete_parse(&mut self, macro_path: &Path, kind_name: &str, span: Span) {
 917         if self.token != token::Eof {
 918             let msg = format!("macro expansion ignores token `{}` and any following",
 919                               self.this_token_to_string());
 920             // Avoid emitting backtrace info twice.
 921             let def_site_span = self.span.with_ctxt(SyntaxContext::empty());
 922             let mut err = self.diagnostic().struct_span_err(def_site_span, &msg);
 923             let msg = format!("caused by the macro expansion here; the usage \
 924                                of `{}!` is likely invalid in {} context",
 925                                macro_path, kind_name);
 926             err.span_note(span, &msg).emit();
 927         }
 928     }
 929 }
 930
 931 struct InvocationCollector<'a, 'b: 'a> {
 932     cx: &'a mut ExtCtxt<'b>,
 933     cfg: StripUnconfigured<'a>,
 934     invocations: Vec<Invocation>,
 935     monotonic: bool,
 936 }
 937
 938 impl<'a, 'b> InvocationCollector<'a, 'b> {
 939     fn collect(&mut self, expansion_kind: ExpansionKind, kind: InvocationKind) -> Expansion {
 940         let mark = Mark::fresh(self.cx.current_expansion.mark);
 941         self.invocations.push(Invocation {
 942             kind,
 943             expansion_kind,
 944             expansion_data: ExpansionData {
 945                 mark,
 946                 depth: self.cx.current_expansion.depth + 1,
 947                 ..self.cx.current_expansion.clone()
 948             },
 949         });
 950         placeholder(expansion_kind, NodeId::placeholder_from_mark(mark))
 951     }
 952
 953     fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: ExpansionKind) -> Expansion {
 954         self.collect(kind, InvocationKind::Bang { mac: mac, ident: None, span: span })
 955     }
 956
 957     fn collect_attr(&mut self,
 958                     attr: Option<ast::Attribute>,
 959                     traits: Vec<Path>,
 960                     item: Annotatable,
 961                     kind: ExpansionKind)
 962                     -> Expansion {
 963         self.collect(kind, InvocationKind::Attr { attr, traits, item })
 964     }
 965
 966     /// If `item` is an attr invocation, remove and return the macro attribute and derive traits.
 967     fn classify_item<T>(&mut self, mut item: T) -> (Option<ast::Attribute>, Vec<Path>, T)
 968         where T: HasAttrs,
 969     {
 970         let (mut attr, mut traits) = (None, Vec::new());
 971
 972         item = item.map_attrs(|mut attrs| {
 973             if let Some(legacy_attr_invoc) = self.cx.resolver.find_legacy_attr_invoc(&mut attrs,
 974                                                                                      true) {
 975                 attr = Some(legacy_attr_invoc);
 976                 return attrs;
 977             }
 978
 979             if self.cx.ecfg.proc_macro_enabled() {
 980                 attr = find_attr_invoc(&mut attrs);
 981             }
 982             traits = collect_derives(&mut self.cx, &mut attrs);
 983             attrs
 984         });
 985
 986         (attr, traits, item)
 987     }
 988
 989     /// Alternative of `classify_item()` that ignores `#[derive]` so invocations fallthrough
 990     /// to the unused-attributes lint (making it an error on statements and expressions
 991     /// is a breaking change)
 992     fn classify_nonitem<T: HasAttrs>(&mut self, mut item: T) -> (Option<ast::Attribute>, T) {
 993         let mut attr = None;
 994
 995         item = item.map_attrs(|mut attrs| {
 996             if let Some(legacy_attr_invoc) = self.cx.resolver.find_legacy_attr_invoc(&mut attrs,
 997                                                                                      false) {
 998                 attr = Some(legacy_attr_invoc);
 999                 return attrs;
1000             }
1001
1002             if self.cx.ecfg.proc_macro_enabled() {
1003                 attr = find_attr_invoc(&mut attrs);
1004             }
1005             attrs
1006         });
1007
1008         (attr, item)
1009     }
1010
1011     fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> {
1012         self.cfg.configure(node)
1013     }
1014
1015     // Detect use of feature-gated or invalid attributes on macro invocations
1016     // since they will not be detected after macro expansion.
1017     fn check_attributes(&mut self, attrs: &[ast::Attribute]) {
1018         let features = self.cx.ecfg.features.unwrap();
1019         for attr in attrs.iter() {
1020             self.check_attribute_inner(attr, features);
1021
1022             // macros are expanded before any lint passes so this warning has to be hardcoded
1023             if attr.path == "derive" {
1024                 self.cx.struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations")
1025                     .note("this may become a hard error in a future release")
1026                     .emit();
1027             }
1028         }
1029     }
1030
1031     fn check_attribute(&mut self, at: &ast::Attribute) {
1032         let features = self.cx.ecfg.features.unwrap();
1033         self.check_attribute_inner(at, features);
1034     }
1035
1036     fn check_attribute_inner(&mut self, at: &ast::Attribute, features: &Features) {
1037         feature_gate::check_attribute(at, self.cx.parse_sess, features);
1038     }
1039 }
1040
1041 pub fn find_attr_invoc(attrs: &mut Vec<ast::Attribute>) -> Option<ast::Attribute> {
1042     attrs.iter()
1043          .position(|a| !attr::is_known(a) && !is_builtin_attr(a))
1044          .map(|i| attrs.remove(i))
1045 }
1046
1047 impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
1048     fn fold_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> {
1049         let mut expr = self.cfg.configure_expr(expr).into_inner();
1050         expr.node = self.cfg.configure_expr_kind(expr.node);
1051
1052         // ignore derives so they remain unused
1053         let (attr, expr) = self.classify_nonitem(expr);
1054
1055         if attr.is_some() {
1056             // collect the invoc regardless of whether or not attributes are permitted here
1057             // expansion will eat the attribute so it won't error later
1058             attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a));
1059
1060             // ExpansionKind::Expr requires the macro to emit an expression
1061             return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), ExpansionKind::Expr)
1062                 .make_expr();
1063         }
1064
1065         if let ast::ExprKind::Mac(mac) = expr.node {
1066             self.check_attributes(&expr.attrs);
1067             self.collect_bang(mac, expr.span, ExpansionKind::Expr).make_expr()
1068         } else {
1069             P(noop_fold_expr(expr, self))
1070         }
1071     }
1072
1073     fn fold_opt_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
1074         let mut expr = configure!(self, expr).into_inner();
1075         expr.node = self.cfg.configure_expr_kind(expr.node);
1076
1077         // ignore derives so they remain unused
1078         let (attr, expr) = self.classify_nonitem(expr);
1079
1080         if attr.is_some() {
1081             attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a));
1082
1083             return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)),
1084                                      ExpansionKind::OptExpr)
1085                 .make_opt_expr();
1086         }
1087
1088         if let ast::ExprKind::Mac(mac) = expr.node {
1089             self.check_attributes(&expr.attrs);
1090             self.collect_bang(mac, expr.span, ExpansionKind::OptExpr).make_opt_expr()
1091         } else {
1092             Some(P(noop_fold_expr(expr, self)))
1093         }
1094     }
1095
1096     fn fold_pat(&mut self, pat: P<ast::Pat>) -> P<ast::Pat> {
1097         let pat = self.cfg.configure_pat(pat);
1098         match pat.node {
1099             PatKind::Mac(_) => {}
1100             _ => return noop_fold_pat(pat, self),
1101         }
1102
1103         pat.and_then(|pat| match pat.node {
1104             PatKind::Mac(mac) => self.collect_bang(mac, pat.span, ExpansionKind::Pat).make_pat(),
1105             _ => unreachable!(),
1106         })
1107     }
1108
1109     fn fold_stmt(&mut self, stmt: ast::Stmt) -> SmallVector<ast::Stmt> {
1110         let mut stmt = match self.cfg.configure_stmt(stmt) {
1111             Some(stmt) => stmt,
1112             None => return SmallVector::new(),
1113         };
1114
1115         // we'll expand attributes on expressions separately
1116         if !stmt.is_expr() {
1117             let (attr, derives, stmt_) = if stmt.is_item() {
1118                 self.classify_item(stmt)
1119             } else {
1120                 // ignore derives on non-item statements so it falls through
1121                 // to the unused-attributes lint
1122                 let (attr, stmt) = self.classify_nonitem(stmt);
1123                 (attr, vec![], stmt)
1124             };
1125
1126             if attr.is_some() || !derives.is_empty() {
1127                 return self.collect_attr(attr, derives,
1128                                          Annotatable::Stmt(P(stmt_)), ExpansionKind::Stmts)
1129                     .make_stmts();
1130             }
1131
1132             stmt = stmt_;
1133         }
1134
1135         if let StmtKind::Mac(mac) = stmt.node {
1136             let (mac, style, attrs) = mac.into_inner();
1137             self.check_attributes(&attrs);
1138             let mut placeholder = self.collect_bang(mac, stmt.span, ExpansionKind::Stmts)
1139                                         .make_stmts();
1140
1141             // If this is a macro invocation with a semicolon, then apply that
1142             // semicolon to the final statement produced by expansion.
1143             if style == MacStmtStyle::Semicolon {
1144                 if let Some(stmt) = placeholder.pop() {
1145                     placeholder.push(stmt.add_trailing_semicolon());
1146                 }
1147             }
1148
1149             return placeholder;
1150         }
1151
1152         // The placeholder expander gives ids to statements, so we avoid folding the id here.
1153         let ast::Stmt { id, node, span } = stmt;
1154         noop_fold_stmt_kind(node, self).into_iter().map(|node| {
1155             ast::Stmt { id, node, span }
1156         }).collect()
1157
1158     }
1159
1160     fn fold_block(&mut self, block: P<Block>) -> P<Block> {
1161         let old_directory_ownership = self.cx.current_expansion.directory_ownership;
1162         self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock;
1163         let result = noop_fold_block(block, self);
1164         self.cx.current_expansion.directory_ownership = old_directory_ownership;
1165         result
1166     }
1167
1168     fn fold_item(&mut self, item: P<ast::Item>) -> SmallVector<P<ast::Item>> {
1169         let item = configure!(self, item);
1170
1171         let (attr, traits, mut item) = self.classify_item(item);
1172         if attr.is_some() || !traits.is_empty() {
1173             let item = Annotatable::Item(item);
1174             return self.collect_attr(attr, traits, item, ExpansionKind::Items).make_items();
1175         }
1176
1177         match item.node {
1178             ast::ItemKind::Mac(..) => {
1179                 self.check_attributes(&item.attrs);
1180                 item.and_then(|item| match item.node {
1181                     ItemKind::Mac(mac) => {
1182                         self.collect(ExpansionKind::Items, InvocationKind::Bang {
1183                             mac,
1184                             ident: Some(item.ident),
1185                             span: item.span,
1186                         }).make_items()
1187                     }
1188                     _ => unreachable!(),
1189                 })
1190             }
1191             ast::ItemKind::Mod(ast::Mod { inner, .. }) => {
1192                 if item.ident == keywords::Invalid.ident() {
1193                     return noop_fold_item(item, self);
1194                 }
1195
1196                 let orig_directory_ownership = self.cx.current_expansion.directory_ownership;
1197                 let mut module = (*self.cx.current_expansion.module).clone();
1198                 module.mod_path.push(item.ident);
1199
1200                 // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`).
1201                 // In the non-inline case, `inner` is never the dummy span (c.f. `parse_item_mod`).
1202                 // Thus, if `inner` is the dummy span, we know the module is inline.
1203                 let inline_module = item.span.contains(inner) || inner == DUMMY_SP;
1204
1205                 if inline_module {
1206                     if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, "path") {
1207                         self.cx.current_expansion.directory_ownership =
1208                             DirectoryOwnership::Owned { relative: None };
1209                         module.directory.push(&*path.as_str());
1210                     } else {
1211                         module.directory.push(&*item.ident.name.as_str());
1212                     }
1213                 } else {
1214                     let path = self.cx.parse_sess.codemap().span_to_unmapped_path(inner);
1215                     let mut path = match path {
1216                         FileName::Real(path) => path,
1217                         other => PathBuf::from(other.to_string()),
1218                     };
1219                     let directory_ownership = match path.file_name().unwrap().to_str() {
1220                         Some("mod.rs") => DirectoryOwnership::Owned { relative: None },
1221                         Some(_) => DirectoryOwnership::Owned {
1222                             relative: Some(item.ident),
1223                         },
1224                         None => DirectoryOwnership::UnownedViaMod(false),
1225                     };
1226                     path.pop();
1227                     module.directory = path;
1228                     self.cx.current_expansion.directory_ownership = directory_ownership;
1229                 }
1230
1231                 let orig_module =
1232                     mem::replace(&mut self.cx.current_expansion.module, Rc::new(module));
1233                 let result = noop_fold_item(item, self);
1234                 self.cx.current_expansion.module = orig_module;
1235                 self.cx.current_expansion.directory_ownership = orig_directory_ownership;
1236                 result
1237             }
1238             // Ensure that test functions are accessible from the test harness.
1239             ast::ItemKind::Fn(..) if self.cx.ecfg.should_test => {
1240                 if item.attrs.iter().any(|attr| is_test_or_bench(attr)) {
1241                     item = item.map(|mut item| {
1242                         item.vis = respan(item.vis.span, ast::VisibilityKind::Public);
1243                         item
1244                     });
1245                 }
1246                 noop_fold_item(item, self)
1247             }
1248             _ => noop_fold_item(item, self),
1249         }
1250     }
1251
1252     fn fold_trait_item(&mut self, item: ast::TraitItem) -> SmallVector<ast::TraitItem> {
1253         let item = configure!(self, item);
1254
1255         let (attr, traits, item) = self.classify_item(item);
1256         if attr.is_some() || !traits.is_empty() {
1257             let item = Annotatable::TraitItem(P(item));
1258             return self.collect_attr(attr, traits, item, ExpansionKind::TraitItems)
1259                 .make_trait_items()
1260         }
1261
1262         match item.node {
1263             ast::TraitItemKind::Macro(mac) => {
1264                 let ast::TraitItem { attrs, span, .. } = item;
1265                 self.check_attributes(&attrs);
1266                 self.collect_bang(mac, span, ExpansionKind::TraitItems).make_trait_items()
1267             }
1268             _ => fold::noop_fold_trait_item(item, self),
1269         }
1270     }
1271
1272     fn fold_impl_item(&mut self, item: ast::ImplItem) -> SmallVector<ast::ImplItem> {
1273         let item = configure!(self, item);
1274
1275         let (attr, traits, item) = self.classify_item(item);
1276         if attr.is_some() || !traits.is_empty() {
1277             let item = Annotatable::ImplItem(P(item));
1278             return self.collect_attr(attr, traits, item, ExpansionKind::ImplItems)
1279                 .make_impl_items();
1280         }
1281
1282         match item.node {
1283             ast::ImplItemKind::Macro(mac) => {
1284                 let ast::ImplItem { attrs, span, .. } = item;
1285                 self.check_attributes(&attrs);
1286                 self.collect_bang(mac, span, ExpansionKind::ImplItems).make_impl_items()
1287             }
1288             _ => fold::noop_fold_impl_item(item, self),
1289         }
1290     }
1291
1292     fn fold_ty(&mut self, ty: P<ast::Ty>) -> P<ast::Ty> {
1293         let ty = match ty.node {
1294             ast::TyKind::Mac(_) => ty.into_inner(),
1295             _ => return fold::noop_fold_ty(ty, self),
1296         };
1297
1298         match ty.node {
1299             ast::TyKind::Mac(mac) => self.collect_bang(mac, ty.span, ExpansionKind::Ty).make_ty(),
1300             _ => unreachable!(),
1301         }
1302     }
1303
1304     fn fold_foreign_mod(&mut self, foreign_mod: ast::ForeignMod) -> ast::ForeignMod {
1305         noop_fold_foreign_mod(self.cfg.configure_foreign_mod(foreign_mod), self)
1306     }
1307
1308     fn fold_foreign_item(&mut self,
1309                          foreign_item: ast::ForeignItem) -> SmallVector<ast::ForeignItem> {
1310         let (attr, traits, foreign_item) = self.classify_item(foreign_item);
1311
1312         let explain = if self.cx.ecfg.proc_macro_enabled() {
1313             feature_gate::EXPLAIN_PROC_MACROS_IN_EXTERN
1314         } else {
1315             feature_gate::EXPLAIN_MACROS_IN_EXTERN
1316         };
1317
1318         if attr.is_some() || !traits.is_empty()  {
1319             if !self.cx.ecfg.macros_in_extern_enabled() {
1320                 if let Some(ref attr) = attr {
1321                     emit_feature_err(&self.cx.parse_sess, "macros_in_extern", attr.span,
1322                                      GateIssue::Language, explain);
1323                 }
1324             }
1325
1326             let item = Annotatable::ForeignItem(P(foreign_item));
1327             return self.collect_attr(attr, traits, item, ExpansionKind::ForeignItems)
1328                 .make_foreign_items();
1329         }
1330
1331         if let ast::ForeignItemKind::Macro(mac) = foreign_item.node {
1332             self.check_attributes(&foreign_item.attrs);
1333
1334             if !self.cx.ecfg.macros_in_extern_enabled() {
1335                 emit_feature_err(&self.cx.parse_sess, "macros_in_extern", foreign_item.span,
1336                                  GateIssue::Language, explain);
1337             }
1338
1339             return self.collect_bang(mac, foreign_item.span, ExpansionKind::ForeignItems)
1340                 .make_foreign_items();
1341         }
1342
1343         noop_fold_foreign_item(foreign_item, self)
1344     }
1345
1346     fn fold_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
1347         match item {
1348             ast::ItemKind::MacroDef(..) => item,
1349             _ => noop_fold_item_kind(self.cfg.configure_item_kind(item), self),
1350         }
1351     }
1352
1353     fn fold_attribute(&mut self, at: ast::Attribute) -> Option<ast::Attribute> {
1354         // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename",
1355         // contents="file contents")]` attributes
1356         if !at.check_name("doc") {
1357             return noop_fold_attribute(at, self);
1358         }
1359
1360         if let Some(list) = at.meta_item_list() {
1361             if !list.iter().any(|it| it.check_name("include")) {
1362                 return noop_fold_attribute(at, self);
1363             }
1364
1365             let mut items = vec![];
1366
1367             for it in list {
1368                 if !it.check_name("include") {
1369                     items.push(noop_fold_meta_list_item(it, self));
1370                     continue;
1371                 }
1372
1373                 if let Some(file) = it.value_str() {
1374                     let err_count = self.cx.parse_sess.span_diagnostic.err_count();
1375                     self.check_attribute(&at);
1376                     if self.cx.parse_sess.span_diagnostic.err_count() > err_count {
1377                         // avoid loading the file if they haven't enabled the feature
1378                         return noop_fold_attribute(at, self);
1379                     }
1380
1381                     let mut buf = vec![];
1382                     let filename = self.cx.root_path.join(file.to_string());
1383
1384                     match File::open(&filename).and_then(|mut f| f.read_to_end(&mut buf)) {
1385                         Ok(..) => {}
1386                         Err(e) => {
1387                             self.cx.span_err(at.span,
1388                                              &format!("couldn't read {}: {}",
1389                                                       filename.display(),
1390                                                       e));
1391                         }
1392                     }
1393
1394                     match String::from_utf8(buf) {
1395                         Ok(src) => {
1396                             // Add this input file to the code map to make it available as
1397                             // dependency information
1398                             self.cx.codemap().new_filemap_and_lines(&filename, &src);
1399
1400                             let include_info = vec![
1401                                 dummy_spanned(ast::NestedMetaItemKind::MetaItem(
1402                                         attr::mk_name_value_item_str(Ident::from_str("file"),
1403                                                                      dummy_spanned(file)))),
1404                                 dummy_spanned(ast::NestedMetaItemKind::MetaItem(
1405                                         attr::mk_name_value_item_str(Ident::from_str("contents"),
1406                                                             dummy_spanned(Symbol::intern(&src))))),
1407                             ];
1408
1409                             let include_ident = Ident::from_str("include");
1410                             let item = attr::mk_list_item(DUMMY_SP, include_ident, include_info);
1411                             items.push(dummy_spanned(ast::NestedMetaItemKind::MetaItem(item)));
1412                         }
1413                         Err(_) => {
1414                             self.cx.span_err(at.span,
1415                                              &format!("{} wasn't a utf-8 file",
1416                                                       filename.display()));
1417                         }
1418                     }
1419                 } else {
1420                     items.push(noop_fold_meta_list_item(it, self));
1421                 }
1422             }
1423
1424             let meta = attr::mk_list_item(DUMMY_SP, Ident::from_str("doc"), items);
1425             match at.style {
1426                 ast::AttrStyle::Inner =>
1427                     Some(attr::mk_spanned_attr_inner(at.span, at.id, meta)),
1428                 ast::AttrStyle::Outer =>
1429                     Some(attr::mk_spanned_attr_outer(at.span, at.id, meta)),
1430             }
1431         } else {
1432             noop_fold_attribute(at, self)
1433         }
1434     }
1435
1436     fn new_id(&mut self, id: ast::NodeId) -> ast::NodeId {
1437         if self.monotonic {
1438             assert_eq!(id, ast::DUMMY_NODE_ID);
1439             self.cx.resolver.next_node_id()
1440         } else {
1441             id
1442         }
1443     }
1444 }
1445
1446 pub struct ExpansionConfig<'feat> {
1447     pub crate_name: String,
1448     pub features: Option<&'feat Features>,
1449     pub recursion_limit: usize,
1450     pub trace_mac: bool,
1451     pub should_test: bool, // If false, strip `#[test]` nodes
1452     pub single_step: bool,
1453     pub keep_macs: bool,
1454 }
1455
1456 macro_rules! feature_tests {
1457     ($( fn $getter:ident = $field:ident, )*) => {
1458         $(
1459             pub fn $getter(&self) -> bool {
1460                 match self.features {
1461                     Some(&Features { $field: true, .. }) => true,
1462                     _ => false,
1463                 }
1464             }
1465         )*
1466     }
1467 }
1468
1469 impl<'feat> ExpansionConfig<'feat> {
1470     pub fn default(crate_name: String) -> ExpansionConfig<'static> {
1471         ExpansionConfig {
1472             crate_name,
1473             features: None,
1474             recursion_limit: 1024,
1475             trace_mac: false,
1476             should_test: false,
1477             single_step: false,
1478             keep_macs: false,
1479         }
1480     }
1481
1482     feature_tests! {
1483         fn enable_quotes = quote,
1484         fn enable_asm = asm,
1485         fn enable_global_asm = global_asm,
1486         fn enable_log_syntax = log_syntax,
1487         fn enable_concat_idents = concat_idents,
1488         fn enable_trace_macros = trace_macros,
1489         fn enable_allow_internal_unstable = allow_internal_unstable,
1490         fn enable_custom_derive = custom_derive,
1491         fn proc_macro_enabled = proc_macro,
1492         fn macros_in_extern_enabled = macros_in_extern,
1493         fn proc_macro_mod = proc_macro_mod,
1494         fn proc_macro_expr = proc_macro_expr,
1495         fn proc_macro_non_items = proc_macro_non_items,
1496     }
1497 }
1498
1499 // A Marker adds the given mark to the syntax context.
1500 #[derive(Debug)]
1501 pub struct Marker(pub Mark);
1502
1503 impl Folder for Marker {
1504     fn fold_ident(&mut self, mut ident: Ident) -> Ident {
1505         ident.span = ident.span.apply_mark(self.0);
1506         ident
1507     }
1508
1509     fn new_span(&mut self, span: Span) -> Span {
1510         span.apply_mark(self.0)
1511     }
1512
1513     fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
1514         noop_fold_mac(mac, self)
1515     }
1516 }