src/libsyntax_expand/expand.rs

   1 use crate::base::*;
   2 use crate::proc_macro::{collect_derives, MarkAttrs};
   3 use crate::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind};
   4 use crate::mbe::macro_rules::annotate_err_with_kind;
   5 use crate::placeholders::{placeholder, PlaceholderExpander};
   6
   7 use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path};
   8 use syntax::ast::{MacStmtStyle, StmtKind, ItemKind};
   9 use syntax::attr::{self, HasAttrs};
  10 use syntax::source_map::respan;
  11 use syntax::configure;
  12 use syntax::config::StripUnconfigured;
  13 use syntax::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err};
  14 use syntax::mut_visit::*;
  15 use syntax::parse::{DirectoryOwnership, PResult};
  16 use syntax::parse::token;
  17 use syntax::parse::parser::Parser;
  18 use syntax::print::pprust;
  19 use syntax::ptr::P;
  20 use syntax::symbol::{sym, Symbol};
  21 use syntax::tokenstream::{TokenStream, TokenTree};
  22 use syntax::visit::Visitor;
  23 use syntax::util::map_in_place::MapInPlace;
  24
  25 use errors::{Applicability, FatalError};
  26 use smallvec::{smallvec, SmallVec};
  27 use syntax_pos::{Span, DUMMY_SP, FileName};
  28
  29 use rustc_data_structures::sync::Lrc;
  30 use std::io::ErrorKind;
  31 use std::{iter, mem, slice};
  32 use std::ops::DerefMut;
  33 use std::rc::Rc;
  34 use std::path::PathBuf;
  35
  36 macro_rules! ast_fragments {
  37     (
  38         $($Kind:ident($AstTy:ty) {
  39             $kind_name:expr;
  40             $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)?
  41             $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident;)?
  42             fn $make_ast:ident;
  43         })*
  44     ) => {
  45         /// A fragment of AST that can be produced by a single macro expansion.
  46         /// Can also serve as an input and intermediate result for macro expansion operations.
  47         pub enum AstFragment {
  48             OptExpr(Option<P<ast::Expr>>),
  49             $($Kind($AstTy),)*
  50         }
  51
  52         /// "Discriminant" of an AST fragment.
  53         #[derive(Copy, Clone, PartialEq, Eq)]
  54         pub enum AstFragmentKind {
  55             OptExpr,
  56             $($Kind,)*
  57         }
  58
  59         impl AstFragmentKind {
  60             pub fn name(self) -> &'static str {
  61                 match self {
  62                     AstFragmentKind::OptExpr => "expression",
  63                     $(AstFragmentKind::$Kind => $kind_name,)*
  64                 }
  65             }
  66
  67             fn make_from<'a>(self, result: Box<dyn MacResult + 'a>) -> Option<AstFragment> {
  68                 match self {
  69                     AstFragmentKind::OptExpr =>
  70                         result.make_expr().map(Some).map(AstFragment::OptExpr),
  71                     $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)*
  72                 }
  73             }
  74         }
  75
  76         impl AstFragment {
  77             pub fn add_placeholders(&mut self, placeholders: &[NodeId]) {
  78                 if placeholders.is_empty() {
  79                     return;
  80                 }
  81                 match self {
  82                     $($(AstFragment::$Kind(ast) => ast.extend(placeholders.iter().flat_map(|id| {
  83                         // We are repeating through arguments with `many`, to do that we have to
  84                         // mention some macro variable from those arguments even if it's not used.
  85                         #[cfg_attr(bootstrap, allow(unused_macros))]
  86                         macro _repeating($flat_map_ast_elt) {}
  87                         placeholder(AstFragmentKind::$Kind, *id).$make_ast()
  88                     })),)?)*
  89                     _ => panic!("unexpected AST fragment kind")
  90                 }
  91             }
  92
  93             pub fn make_opt_expr(self) -> Option<P<ast::Expr>> {
  94                 match self {
  95                     AstFragment::OptExpr(expr) => expr,
  96                     _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
  97                 }
  98             }
  99
 100             $(pub fn $make_ast(self) -> $AstTy {
 101                 match self {
 102                     AstFragment::$Kind(ast) => ast,
 103                     _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
 104                 }
 105             })*
 106
 107             pub fn mut_visit_with<F: MutVisitor>(&mut self, vis: &mut F) {
 108                 match self {
 109                     AstFragment::OptExpr(opt_expr) => {
 110                         visit_clobber(opt_expr, |opt_expr| {
 111                             if let Some(expr) = opt_expr {
 112                                 vis.filter_map_expr(expr)
 113                             } else {
 114                                 None
 115                             }
 116                         });
 117                     }
 118                     $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)*
 119                     $($(AstFragment::$Kind(ast) =>
 120                         ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)*
 121                 }
 122             }
 123
 124             pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
 125                 match *self {
 126                     AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr),
 127                     AstFragment::OptExpr(None) => {}
 128                     $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)*
 129                     $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] {
 130                         visitor.$visit_ast_elt(ast_elt);
 131                     })?)*
 132                 }
 133             }
 134         }
 135
 136         impl<'a> MacResult for crate::mbe::macro_rules::ParserAnyMacro<'a> {
 137             $(fn $make_ast(self: Box<crate::mbe::macro_rules::ParserAnyMacro<'a>>)
 138                            -> Option<$AstTy> {
 139                 Some(self.make(AstFragmentKind::$Kind).$make_ast())
 140             })*
 141         }
 142     }
 143 }
 144
 145 ast_fragments! {
 146     Expr(P<ast::Expr>) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; }
 147     Pat(P<ast::Pat>) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; }
 148     Ty(P<ast::Ty>) { "type"; one fn visit_ty; fn visit_ty; fn make_ty; }
 149     Stmts(SmallVec<[ast::Stmt; 1]>) {
 150         "statement"; many fn flat_map_stmt; fn visit_stmt; fn make_stmts;
 151     }
 152     Items(SmallVec<[P<ast::Item>; 1]>) {
 153         "item"; many fn flat_map_item; fn visit_item; fn make_items;
 154     }
 155     TraitItems(SmallVec<[ast::TraitItem; 1]>) {
 156         "trait item"; many fn flat_map_trait_item; fn visit_trait_item; fn make_trait_items;
 157     }
 158     ImplItems(SmallVec<[ast::ImplItem; 1]>) {
 159         "impl item"; many fn flat_map_impl_item; fn visit_impl_item; fn make_impl_items;
 160     }
 161     ForeignItems(SmallVec<[ast::ForeignItem; 1]>) {
 162         "foreign item";
 163         many fn flat_map_foreign_item;
 164         fn visit_foreign_item;
 165         fn make_foreign_items;
 166     }
 167     Arms(SmallVec<[ast::Arm; 1]>) {
 168         "match arm"; many fn flat_map_arm; fn visit_arm; fn make_arms;
 169     }
 170     Fields(SmallVec<[ast::Field; 1]>) {
 171         "field expression"; many fn flat_map_field; fn visit_field; fn make_fields;
 172     }
 173     FieldPats(SmallVec<[ast::FieldPat; 1]>) {
 174         "field pattern";
 175         many fn flat_map_field_pattern;
 176         fn visit_field_pattern;
 177         fn make_field_patterns;
 178     }
 179     GenericParams(SmallVec<[ast::GenericParam; 1]>) {
 180         "generic parameter";
 181         many fn flat_map_generic_param;
 182         fn visit_generic_param;
 183         fn make_generic_params;
 184     }
 185     Params(SmallVec<[ast::Param; 1]>) {
 186         "function parameter"; many fn flat_map_param; fn visit_param; fn make_params;
 187     }
 188     StructFields(SmallVec<[ast::StructField; 1]>) {
 189         "field";
 190         many fn flat_map_struct_field;
 191         fn visit_struct_field;
 192         fn make_struct_fields;
 193     }
 194     Variants(SmallVec<[ast::Variant; 1]>) {
 195         "variant"; many fn flat_map_variant; fn visit_variant; fn make_variants;
 196     }
 197 }
 198
 199 impl AstFragmentKind {
 200     fn dummy(self, span: Span) -> AstFragment {
 201         self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment")
 202     }
 203
 204     fn expect_from_annotatables<I: IntoIterator<Item = Annotatable>>(self, items: I)
 205                                                                      -> AstFragment {
 206         let mut items = items.into_iter();
 207         match self {
 208             AstFragmentKind::Arms =>
 209                 AstFragment::Arms(items.map(Annotatable::expect_arm).collect()),
 210             AstFragmentKind::Fields =>
 211                 AstFragment::Fields(items.map(Annotatable::expect_field).collect()),
 212             AstFragmentKind::FieldPats =>
 213                 AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect()),
 214             AstFragmentKind::GenericParams =>
 215                 AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect()),
 216             AstFragmentKind::Params =>
 217                 AstFragment::Params(items.map(Annotatable::expect_param).collect()),
 218             AstFragmentKind::StructFields => AstFragment::StructFields(
 219                 items.map(Annotatable::expect_struct_field).collect()
 220             ),
 221             AstFragmentKind::Variants =>
 222                 AstFragment::Variants(items.map(Annotatable::expect_variant).collect()),
 223             AstFragmentKind::Items =>
 224                 AstFragment::Items(items.map(Annotatable::expect_item).collect()),
 225             AstFragmentKind::ImplItems =>
 226                 AstFragment::ImplItems(items.map(Annotatable::expect_impl_item).collect()),
 227             AstFragmentKind::TraitItems =>
 228                 AstFragment::TraitItems(items.map(Annotatable::expect_trait_item).collect()),
 229             AstFragmentKind::ForeignItems =>
 230                 AstFragment::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()),
 231             AstFragmentKind::Stmts =>
 232                 AstFragment::Stmts(items.map(Annotatable::expect_stmt).collect()),
 233             AstFragmentKind::Expr => AstFragment::Expr(
 234                 items.next().expect("expected exactly one expression").expect_expr()
 235             ),
 236             AstFragmentKind::OptExpr =>
 237                 AstFragment::OptExpr(items.next().map(Annotatable::expect_expr)),
 238             AstFragmentKind::Pat | AstFragmentKind::Ty =>
 239                 panic!("patterns and types aren't annotatable"),
 240         }
 241     }
 242 }
 243
 244 pub struct Invocation {
 245     pub kind: InvocationKind,
 246     pub fragment_kind: AstFragmentKind,
 247     pub expansion_data: ExpansionData,
 248 }
 249
 250 pub enum InvocationKind {
 251     Bang {
 252         mac: ast::Mac,
 253         span: Span,
 254     },
 255     Attr {
 256         attr: ast::Attribute,
 257         item: Annotatable,
 258         // Required for resolving derive helper attributes.
 259         derives: Vec<Path>,
 260         // We temporarily report errors for attribute macros placed after derives
 261         after_derive: bool,
 262     },
 263     Derive {
 264         path: Path,
 265         item: Annotatable,
 266     },
 267     /// "Invocation" that contains all derives from an item,
 268     /// broken into multiple `Derive` invocations when expanded.
 269     /// FIXME: Find a way to remove it.
 270     DeriveContainer {
 271         derives: Vec<Path>,
 272         item: Annotatable,
 273     },
 274 }
 275
 276 impl Invocation {
 277     pub fn span(&self) -> Span {
 278         match &self.kind {
 279             InvocationKind::Bang { span, .. } => *span,
 280             InvocationKind::Attr { attr, .. } => attr.span,
 281             InvocationKind::Derive { path, .. } => path.span,
 282             InvocationKind::DeriveContainer { item, .. } => item.span(),
 283         }
 284     }
 285 }
 286
 287 pub struct MacroExpander<'a, 'b> {
 288     pub cx: &'a mut ExtCtxt<'b>,
 289     monotonic: bool, // cf. `cx.monotonic_expander()`
 290 }
 291
 292 impl<'a, 'b> MacroExpander<'a, 'b> {
 293     pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self {
 294         MacroExpander { cx, monotonic }
 295     }
 296
 297     pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate {
 298         let mut module = ModuleData {
 299             mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)],
 300             directory: match self.cx.source_map().span_to_unmapped_path(krate.span) {
 301                 FileName::Real(path) => path,
 302                 other => PathBuf::from(other.to_string()),
 303             },
 304         };
 305         module.directory.pop();
 306         self.cx.root_path = module.directory.clone();
 307         self.cx.current_expansion.module = Rc::new(module);
 308
 309         let orig_mod_span = krate.module.inner;
 310
 311         let krate_item = AstFragment::Items(smallvec![P(ast::Item {
 312             attrs: krate.attrs,
 313             span: krate.span,
 314             kind: ast::ItemKind::Mod(krate.module),
 315             ident: Ident::invalid(),
 316             id: ast::DUMMY_NODE_ID,
 317             vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public),
 318             tokens: None,
 319         })]);
 320
 321         match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) {
 322             Some(ast::Item { attrs, kind: ast::ItemKind::Mod(module), .. }) => {
 323                 krate.attrs = attrs;
 324                 krate.module = module;
 325             },
 326             None => {
 327                 // Resolution failed so we return an empty expansion
 328                 krate.attrs = vec![];
 329                 krate.module = ast::Mod {
 330                     inner: orig_mod_span,
 331                     items: vec![],
 332                     inline: true,
 333                 };
 334             },
 335             _ => unreachable!(),
 336         };
 337         self.cx.trace_macros_diag();
 338         krate
 339     }
 340
 341     // Recursively expand all macro invocations in this AST fragment.
 342     pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment {
 343         let orig_expansion_data = self.cx.current_expansion.clone();
 344         self.cx.current_expansion.depth = 0;
 345
 346         // Collect all macro invocations and replace them with placeholders.
 347         let (mut fragment_with_placeholders, mut invocations)
 348             = self.collect_invocations(input_fragment, &[]);
 349
 350         // Optimization: if we resolve all imports now,
 351         // we'll be able to immediately resolve most of imported macros.
 352         self.resolve_imports();
 353
 354         // Resolve paths in all invocations and produce output expanded fragments for them, but
 355         // do not insert them into our input AST fragment yet, only store in `expanded_fragments`.
 356         // The output fragments also go through expansion recursively until no invocations are left.
 357         // Unresolved macros produce dummy outputs as a recovery measure.
 358         invocations.reverse();
 359         let mut expanded_fragments = Vec::new();
 360         let mut undetermined_invocations = Vec::new();
 361         let (mut progress, mut force) = (false, !self.monotonic);
 362         loop {
 363             let invoc = if let Some(invoc) = invocations.pop() {
 364                 invoc
 365             } else {
 366                 self.resolve_imports();
 367                 if undetermined_invocations.is_empty() { break }
 368                 invocations = mem::take(&mut undetermined_invocations);
 369                 force = !mem::replace(&mut progress, false);
 370                 continue
 371             };
 372
 373             let eager_expansion_root =
 374                 if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id };
 375             let res = match self.cx.resolver.resolve_macro_invocation(
 376                 &invoc, eager_expansion_root, force
 377             ) {
 378                 Ok(res) => res,
 379                 Err(Indeterminate) => {
 380                     undetermined_invocations.push(invoc);
 381                     continue
 382                 }
 383             };
 384
 385             progress = true;
 386             let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data;
 387             self.cx.current_expansion = invoc.expansion_data.clone();
 388
 389             // FIXME(jseyfried): Refactor out the following logic
 390             let (expanded_fragment, new_invocations) = match res {
 391                 InvocationRes::Single(ext) => {
 392                     let fragment = self.expand_invoc(invoc, &ext.kind);
 393                     self.collect_invocations(fragment, &[])
 394                 }
 395                 InvocationRes::DeriveContainer(exts) => {
 396                     let (derives, item) = match invoc.kind {
 397                         InvocationKind::DeriveContainer { derives, item } => (derives, item),
 398                         _ => unreachable!(),
 399                     };
 400                     if !item.derive_allowed() {
 401                         let attr = attr::find_by_name(item.attrs(), sym::derive)
 402                             .expect("`derive` attribute should exist");
 403                         let span = attr.span;
 404                         let mut err = self.cx.struct_span_err(span,
 405                             "`derive` may only be applied to structs, enums and unions");
 406                         if let ast::AttrStyle::Inner = attr.style {
 407                             let trait_list = derives.iter()
 408                                 .map(|t| pprust::path_to_string(t))
 409                                 .collect::<Vec<_>>();
 410                             let suggestion = format!("#[derive({})]", trait_list.join(", "));
 411                             err.span_suggestion(
 412                                 span, "try an outer attribute", suggestion,
 413                                 // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT
 414                                 Applicability::MaybeIncorrect
 415                             );
 416                         }
 417                         err.emit();
 418                     }
 419
 420                     let mut item = self.fully_configure(item);
 421                     item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive));
 422                     let mut helper_attrs = Vec::new();
 423                     let mut has_copy = false;
 424                     for ext in exts {
 425                         helper_attrs.extend(&ext.helper_attrs);
 426                         has_copy |= ext.is_derive_copy;
 427                     }
 428                     // Mark derive helpers inside this item as known and used.
 429                     // FIXME: This is a hack, derive helpers should be integrated with regular name
 430                     // resolution instead. For example, helpers introduced by a derive container
 431                     // can be in scope for all code produced by that container's expansion.
 432                     item.visit_with(&mut MarkAttrs(&helper_attrs));
 433                     if has_copy {
 434                         self.cx.resolver.add_derives(invoc.expansion_data.id, SpecialDerives::COPY);
 435                     }
 436
 437                     let mut derive_placeholders = Vec::with_capacity(derives.len());
 438                     invocations.reserve(derives.len());
 439                     for path in derives {
 440                         let expn_id = ExpnId::fresh(None);
 441                         derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id));
 442                         invocations.push(Invocation {
 443                             kind: InvocationKind::Derive { path, item: item.clone() },
 444                             fragment_kind: invoc.fragment_kind,
 445                             expansion_data: ExpansionData {
 446                                 id: expn_id,
 447                                 ..invoc.expansion_data.clone()
 448                             },
 449                         });
 450                     }
 451                     let fragment = invoc.fragment_kind
 452                         .expect_from_annotatables(::std::iter::once(item));
 453                     self.collect_invocations(fragment, &derive_placeholders)
 454                 }
 455             };
 456
 457             if expanded_fragments.len() < depth {
 458                 expanded_fragments.push(Vec::new());
 459             }
 460             expanded_fragments[depth - 1].push((expn_id, expanded_fragment));
 461             if !self.cx.ecfg.single_step {
 462                 invocations.extend(new_invocations.into_iter().rev());
 463             }
 464         }
 465
 466         self.cx.current_expansion = orig_expansion_data;
 467
 468         // Finally incorporate all the expanded macros into the input AST fragment.
 469         let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic);
 470         while let Some(expanded_fragments) = expanded_fragments.pop() {
 471             for (expn_id, expanded_fragment) in expanded_fragments.into_iter().rev() {
 472                 placeholder_expander.add(NodeId::placeholder_from_expn_id(expn_id),
 473                                          expanded_fragment);
 474             }
 475         }
 476         fragment_with_placeholders.mut_visit_with(&mut placeholder_expander);
 477         fragment_with_placeholders
 478     }
 479
 480     fn resolve_imports(&mut self) {
 481         if self.monotonic {
 482             self.cx.resolver.resolve_imports();
 483         }
 484     }
 485
 486     /// Collects all macro invocations reachable at this time in this AST fragment, and replace
 487     /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s.
 488     /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and
 489     /// prepares data for resolving paths of macro invocations.
 490     fn collect_invocations(&mut self, mut fragment: AstFragment, extra_placeholders: &[NodeId])
 491                            -> (AstFragment, Vec<Invocation>) {
 492         // Resolve `$crate`s in the fragment for pretty-printing.
 493         self.cx.resolver.resolve_dollar_crates();
 494
 495         let invocations = {
 496             let mut collector = InvocationCollector {
 497                 cfg: StripUnconfigured {
 498                     sess: self.cx.parse_sess,
 499                     features: self.cx.ecfg.features,
 500                 },
 501                 cx: self.cx,
 502                 invocations: Vec::new(),
 503                 monotonic: self.monotonic,
 504             };
 505             fragment.mut_visit_with(&mut collector);
 506             fragment.add_placeholders(extra_placeholders);
 507             collector.invocations
 508         };
 509
 510         if self.monotonic {
 511             self.cx.resolver.visit_ast_fragment_with_placeholders(
 512                 self.cx.current_expansion.id, &fragment
 513             );
 514         }
 515
 516         (fragment, invocations)
 517     }
 518
 519     fn fully_configure(&mut self, item: Annotatable) -> Annotatable {
 520         let mut cfg = StripUnconfigured {
 521             sess: self.cx.parse_sess,
 522             features: self.cx.ecfg.features,
 523         };
 524         // Since the item itself has already been configured by the InvocationCollector,
 525         // we know that fold result vector will contain exactly one element
 526         match item {
 527             Annotatable::Item(item) => {
 528                 Annotatable::Item(cfg.flat_map_item(item).pop().unwrap())
 529             }
 530             Annotatable::TraitItem(item) => {
 531                 Annotatable::TraitItem(
 532                     item.map(|item| cfg.flat_map_trait_item(item).pop().unwrap()))
 533             }
 534             Annotatable::ImplItem(item) => {
 535                 Annotatable::ImplItem(item.map(|item| cfg.flat_map_impl_item(item).pop().unwrap()))
 536             }
 537             Annotatable::ForeignItem(item) => {
 538                 Annotatable::ForeignItem(
 539                     item.map(|item| cfg.flat_map_foreign_item(item).pop().unwrap())
 540                 )
 541             }
 542             Annotatable::Stmt(stmt) => {
 543                 Annotatable::Stmt(stmt.map(|stmt| cfg.flat_map_stmt(stmt).pop().unwrap()))
 544             }
 545             Annotatable::Expr(mut expr) => {
 546                 Annotatable::Expr({ cfg.visit_expr(&mut expr); expr })
 547             }
 548             Annotatable::Arm(arm) => {
 549                 Annotatable::Arm(cfg.flat_map_arm(arm).pop().unwrap())
 550             }
 551             Annotatable::Field(field) => {
 552                 Annotatable::Field(cfg.flat_map_field(field).pop().unwrap())
 553             }
 554             Annotatable::FieldPat(fp) => {
 555                 Annotatable::FieldPat(cfg.flat_map_field_pattern(fp).pop().unwrap())
 556             }
 557             Annotatable::GenericParam(param) => {
 558                 Annotatable::GenericParam(cfg.flat_map_generic_param(param).pop().unwrap())
 559             }
 560             Annotatable::Param(param) => {
 561                 Annotatable::Param(cfg.flat_map_param(param).pop().unwrap())
 562             }
 563             Annotatable::StructField(sf) => {
 564                 Annotatable::StructField(cfg.flat_map_struct_field(sf).pop().unwrap())
 565             }
 566             Annotatable::Variant(v) => {
 567                 Annotatable::Variant(cfg.flat_map_variant(v).pop().unwrap())
 568             }
 569         }
 570     }
 571
 572     fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment {
 573         if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit {
 574             let expn_data = self.cx.current_expansion.id.expn_data();
 575             let suggested_limit = self.cx.ecfg.recursion_limit * 2;
 576             let mut err = self.cx.struct_span_err(expn_data.call_site,
 577                 &format!("recursion limit reached while expanding the macro `{}`",
 578                          expn_data.kind.descr()));
 579             err.help(&format!(
 580                 "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate",
 581                 suggested_limit));
 582             err.emit();
 583             self.cx.trace_macros_diag();
 584             FatalError.raise();
 585         }
 586
 587         let (fragment_kind, span) = (invoc.fragment_kind, invoc.span());
 588         match invoc.kind {
 589             InvocationKind::Bang { mac, .. } => match ext {
 590                 SyntaxExtensionKind::Bang(expander) => {
 591                     self.gate_proc_macro_expansion_kind(span, fragment_kind);
 592                     let tok_result = expander.expand(self.cx, span, mac.stream());
 593                     self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span)
 594                 }
 595                 SyntaxExtensionKind::LegacyBang(expander) => {
 596                     let prev = self.cx.current_expansion.prior_type_ascription;
 597                     self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription;
 598                     let tok_result = expander.expand(self.cx, span, mac.stream());
 599                     let result = if let Some(result) = fragment_kind.make_from(tok_result) {
 600                         result
 601                     } else {
 602                         let msg = format!(
 603                             "non-{kind} macro in {kind} position: {path}",
 604                             kind = fragment_kind.name(),
 605                             path = pprust::path_to_string(&mac.path),
 606                         );
 607                         self.cx.span_err(span, &msg);
 608                         self.cx.trace_macros_diag();
 609                         fragment_kind.dummy(span)
 610                     };
 611                     self.cx.current_expansion.prior_type_ascription = prev;
 612                     result
 613                 }
 614                 _ => unreachable!()
 615             }
 616             InvocationKind::Attr { attr, mut item, .. } => match ext {
 617                 SyntaxExtensionKind::Attr(expander) => {
 618                     self.gate_proc_macro_attr_item(span, &item);
 619                     let item_tok = TokenTree::token(token::Interpolated(Lrc::new(match item {
 620                         Annotatable::Item(item) => token::NtItem(item),
 621                         Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
 622                         Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),
 623                         Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()),
 624                         Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
 625                         Annotatable::Expr(expr) => token::NtExpr(expr),
 626                         Annotatable::Arm(..)
 627                         | Annotatable::Field(..)
 628                         | Annotatable::FieldPat(..)
 629                         | Annotatable::GenericParam(..)
 630                         | Annotatable::Param(..)
 631                         | Annotatable::StructField(..)
 632                         | Annotatable::Variant(..)
 633                             => panic!("unexpected annotatable"),
 634                     })), DUMMY_SP).into();
 635                     let input = self.extract_proc_macro_attr_input(attr.item.tokens, span);
 636                     let tok_result = expander.expand(self.cx, span, input, item_tok);
 637                     self.parse_ast_fragment(tok_result, fragment_kind, &attr.item.path, span)
 638                 }
 639                 SyntaxExtensionKind::LegacyAttr(expander) => {
 640                     match attr.parse_meta(self.cx.parse_sess) {
 641                         Ok(meta) => {
 642                             let item = expander.expand(self.cx, span, &meta, item);
 643                             fragment_kind.expect_from_annotatables(item)
 644                         }
 645                         Err(mut err) => {
 646                             err.emit();
 647                             fragment_kind.dummy(span)
 648                         }
 649                     }
 650                 }
 651                 SyntaxExtensionKind::NonMacroAttr { mark_used } => {
 652                     attr::mark_known(&attr);
 653                     if *mark_used {
 654                         attr::mark_used(&attr);
 655                     }
 656                     item.visit_attrs(|attrs| attrs.push(attr));
 657                     fragment_kind.expect_from_annotatables(iter::once(item))
 658                 }
 659                 _ => unreachable!()
 660             }
 661             InvocationKind::Derive { path, item } => match ext {
 662                 SyntaxExtensionKind::Derive(expander) |
 663                 SyntaxExtensionKind::LegacyDerive(expander) => {
 664                     if !item.derive_allowed() {
 665                         return fragment_kind.dummy(span);
 666                     }
 667                     let meta = ast::MetaItem { kind: ast::MetaItemKind::Word, span, path };
 668                     let items = expander.expand(self.cx, span, &meta, item);
 669                     fragment_kind.expect_from_annotatables(items)
 670                 }
 671                 _ => unreachable!()
 672             }
 673             InvocationKind::DeriveContainer { .. } => unreachable!()
 674         }
 675     }
 676
 677     fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
 678         let mut trees = tokens.trees();
 679         match trees.next() {
 680             Some(TokenTree::Delimited(_, _, tts)) => {
 681                 if trees.next().is_none() {
 682                     return tts.into()
 683                 }
 684             }
 685             Some(TokenTree::Token(..)) => {}
 686             None => return TokenStream::default(),
 687         }
 688         self.cx.span_err(span, "custom attribute invocations must be \
 689             of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \
 690             followed by a delimiter token");
 691         TokenStream::default()
 692     }
 693
 694     fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) {
 695         let (kind, gate) = match *item {
 696             Annotatable::Item(ref item) => {
 697                 match item.kind {
 698                     ItemKind::Mod(_) if self.cx.ecfg.proc_macro_hygiene() => return,
 699                     ItemKind::Mod(_) => ("modules", sym::proc_macro_hygiene),
 700                     _ => return,
 701                 }
 702             }
 703             Annotatable::TraitItem(_) => return,
 704             Annotatable::ImplItem(_) => return,
 705             Annotatable::ForeignItem(_) => return,
 706             Annotatable::Stmt(_) |
 707             Annotatable::Expr(_) if self.cx.ecfg.proc_macro_hygiene() => return,
 708             Annotatable::Stmt(_) => ("statements", sym::proc_macro_hygiene),
 709             Annotatable::Expr(_) => ("expressions", sym::proc_macro_hygiene),
 710             Annotatable::Arm(..)
 711             | Annotatable::Field(..)
 712             | Annotatable::FieldPat(..)
 713             | Annotatable::GenericParam(..)
 714             | Annotatable::Param(..)
 715             | Annotatable::StructField(..)
 716             | Annotatable::Variant(..)
 717             => panic!("unexpected annotatable"),
 718         };
 719         emit_feature_err(
 720             self.cx.parse_sess,
 721             gate,
 722             span,
 723             GateIssue::Language,
 724             &format!("custom attributes cannot be applied to {}", kind),
 725         );
 726     }
 727
 728     fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) {
 729         let kind = match kind {
 730             AstFragmentKind::Expr |
 731             AstFragmentKind::OptExpr => "expressions",
 732             AstFragmentKind::Pat => "patterns",
 733             AstFragmentKind::Stmts => "statements",
 734             AstFragmentKind::Ty |
 735             AstFragmentKind::Items |
 736             AstFragmentKind::TraitItems |
 737             AstFragmentKind::ImplItems |
 738             AstFragmentKind::ForeignItems => return,
 739             AstFragmentKind::Arms
 740             | AstFragmentKind::Fields
 741             | AstFragmentKind::FieldPats
 742             | AstFragmentKind::GenericParams
 743             | AstFragmentKind::Params
 744             | AstFragmentKind::StructFields
 745             | AstFragmentKind::Variants
 746                 => panic!("unexpected AST fragment kind"),
 747         };
 748         if self.cx.ecfg.proc_macro_hygiene() {
 749             return
 750         }
 751         emit_feature_err(
 752             self.cx.parse_sess,
 753             sym::proc_macro_hygiene,
 754             span,
 755             GateIssue::Language,
 756             &format!("procedural macros cannot be expanded to {}", kind),
 757         );
 758     }
 759
 760     fn parse_ast_fragment(
 761         &mut self,
 762         toks: TokenStream,
 763         kind: AstFragmentKind,
 764         path: &Path,
 765         span: Span,
 766     ) -> AstFragment {
 767         let mut parser = self.cx.new_parser_from_tts(toks);
 768         match parse_ast_fragment(&mut parser, kind, false) {
 769             Ok(fragment) => {
 770                 ensure_complete_parse(&mut parser, path, kind.name(), span);
 771                 fragment
 772             }
 773             Err(mut err) => {
 774                 err.set_span(span);
 775                 annotate_err_with_kind(&mut err, kind, span);
 776                 err.emit();
 777                 self.cx.trace_macros_diag();
 778                 kind.dummy(span)
 779             }
 780         }
 781     }
 782 }
 783
 784 pub fn parse_ast_fragment<'a>(
 785     this: &mut Parser<'a>,
 786     kind: AstFragmentKind,
 787     macro_legacy_warnings: bool,
 788 ) -> PResult<'a, AstFragment> {
 789     Ok(match kind {
 790         AstFragmentKind::Items => {
 791             let mut items = SmallVec::new();
 792             while let Some(item) = this.parse_item()? {
 793                 items.push(item);
 794             }
 795             AstFragment::Items(items)
 796         }
 797         AstFragmentKind::TraitItems => {
 798             let mut items = SmallVec::new();
 799             while this.token != token::Eof {
 800                 items.push(this.parse_trait_item(&mut false)?);
 801             }
 802             AstFragment::TraitItems(items)
 803         }
 804         AstFragmentKind::ImplItems => {
 805             let mut items = SmallVec::new();
 806             while this.token != token::Eof {
 807                 items.push(this.parse_impl_item(&mut false)?);
 808             }
 809             AstFragment::ImplItems(items)
 810         }
 811         AstFragmentKind::ForeignItems => {
 812             let mut items = SmallVec::new();
 813             while this.token != token::Eof {
 814                 items.push(this.parse_foreign_item(DUMMY_SP)?);
 815             }
 816             AstFragment::ForeignItems(items)
 817         }
 818         AstFragmentKind::Stmts => {
 819             let mut stmts = SmallVec::new();
 820             while this.token != token::Eof &&
 821                     // won't make progress on a `}`
 822                     this.token != token::CloseDelim(token::Brace) {
 823                 if let Some(stmt) = this.parse_full_stmt(macro_legacy_warnings)? {
 824                     stmts.push(stmt);
 825                 }
 826             }
 827             AstFragment::Stmts(stmts)
 828         }
 829         AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?),
 830         AstFragmentKind::OptExpr => {
 831             if this.token != token::Eof {
 832                 AstFragment::OptExpr(Some(this.parse_expr()?))
 833             } else {
 834                 AstFragment::OptExpr(None)
 835             }
 836         },
 837         AstFragmentKind::Ty => AstFragment::Ty(this.parse_ty()?),
 838         AstFragmentKind::Pat => AstFragment::Pat(this.parse_pat(None)?),
 839         AstFragmentKind::Arms
 840         | AstFragmentKind::Fields
 841         | AstFragmentKind::FieldPats
 842         | AstFragmentKind::GenericParams
 843         | AstFragmentKind::Params
 844         | AstFragmentKind::StructFields
 845         | AstFragmentKind::Variants
 846             => panic!("unexpected AST fragment kind"),
 847     })
 848 }
 849
 850 pub fn ensure_complete_parse<'a>(
 851     this: &mut Parser<'a>,
 852     macro_path: &Path,
 853     kind_name: &str,
 854     span: Span,
 855 ) {
 856     if this.token != token::Eof {
 857         let msg = format!("macro expansion ignores token `{}` and any following",
 858                             this.this_token_to_string());
 859         // Avoid emitting backtrace info twice.
 860         let def_site_span = this.token.span.with_ctxt(SyntaxContext::root());
 861         let mut err = this.struct_span_err(def_site_span, &msg);
 862         err.span_label(span, "caused by the macro expansion here");
 863         let msg = format!(
 864             "the usage of `{}!` is likely invalid in {} context",
 865             pprust::path_to_string(macro_path),
 866             kind_name,
 867         );
 868         err.note(&msg);
 869         let semi_span = this.sess.source_map().next_point(span);
 870
 871         let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span));
 872         match this.sess.source_map().span_to_snippet(semi_full_span) {
 873             Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => {
 874                 err.span_suggestion(
 875                     semi_span,
 876                     "you might be missing a semicolon here",
 877                     ";".to_owned(),
 878                     Applicability::MaybeIncorrect,
 879                 );
 880             }
 881             _ => {}
 882         }
 883         err.emit();
 884     }
 885 }
 886
 887 struct InvocationCollector<'a, 'b> {
 888     cx: &'a mut ExtCtxt<'b>,
 889     cfg: StripUnconfigured<'a>,
 890     invocations: Vec<Invocation>,
 891     monotonic: bool,
 892 }
 893
 894 impl<'a, 'b> InvocationCollector<'a, 'b> {
 895     fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment {
 896         // Expansion data for all the collected invocations is set upon their resolution,
 897         // with exception of the derive container case which is not resolved and can get
 898         // its expansion data immediately.
 899         let expn_data = match &kind {
 900             InvocationKind::DeriveContainer { item, .. } => Some(ExpnData {
 901                 parent: self.cx.current_expansion.id,
 902                 ..ExpnData::default(
 903                     ExpnKind::Macro(MacroKind::Attr, sym::derive),
 904                     item.span(), self.cx.parse_sess.edition,
 905                 )
 906             }),
 907             _ => None,
 908         };
 909         let expn_id = ExpnId::fresh(expn_data);
 910         self.invocations.push(Invocation {
 911             kind,
 912             fragment_kind,
 913             expansion_data: ExpansionData {
 914                 id: expn_id,
 915                 depth: self.cx.current_expansion.depth + 1,
 916                 ..self.cx.current_expansion.clone()
 917             },
 918         });
 919         placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id))
 920     }
 921
 922     fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment {
 923         self.collect(kind, InvocationKind::Bang { mac, span })
 924     }
 925
 926     fn collect_attr(&mut self,
 927                     attr: Option<ast::Attribute>,
 928                     derives: Vec<Path>,
 929                     item: Annotatable,
 930                     kind: AstFragmentKind,
 931                     after_derive: bool)
 932                     -> AstFragment {
 933         self.collect(kind, match attr {
 934             Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive },
 935             None => InvocationKind::DeriveContainer { derives, item },
 936         })
 937     }
 938
 939     fn find_attr_invoc(&self, attrs: &mut Vec<ast::Attribute>, after_derive: &mut bool)
 940                        -> Option<ast::Attribute> {
 941         let attr = attrs.iter()
 942                         .position(|a| {
 943                             if a.path == sym::derive {
 944                                 *after_derive = true;
 945                             }
 946                             !attr::is_known(a) && !is_builtin_attr(a)
 947                         })
 948                         .map(|i| attrs.remove(i));
 949         if let Some(attr) = &attr {
 950             if !self.cx.ecfg.custom_inner_attributes() &&
 951                attr.style == ast::AttrStyle::Inner && attr.path != sym::test {
 952                 emit_feature_err(&self.cx.parse_sess, sym::custom_inner_attributes,
 953                                  attr.span, GateIssue::Language,
 954                                  "non-builtin inner attributes are unstable");
 955             }
 956         }
 957         attr
 958     }
 959
 960     /// If `item` is an attr invocation, remove and return the macro attribute and derive traits.
 961     fn classify_item<T>(&mut self, item: &mut T)
 962                         -> (Option<ast::Attribute>, Vec<Path>, /* after_derive */ bool)
 963         where T: HasAttrs,
 964     {
 965         let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false);
 966
 967         item.visit_attrs(|mut attrs| {
 968             attr = self.find_attr_invoc(&mut attrs, &mut after_derive);
 969             traits = collect_derives(&mut self.cx, &mut attrs);
 970         });
 971
 972         (attr, traits, after_derive)
 973     }
 974
 975     /// Alternative to `classify_item()` that ignores `#[derive]` so invocations fallthrough
 976     /// to the unused-attributes lint (making it an error on statements and expressions
 977     /// is a breaking change)
 978     fn classify_nonitem<T: HasAttrs>(&mut self, nonitem: &mut T)
 979                                      -> (Option<ast::Attribute>, /* after_derive */ bool) {
 980         let (mut attr, mut after_derive) = (None, false);
 981
 982         nonitem.visit_attrs(|mut attrs| {
 983             attr = self.find_attr_invoc(&mut attrs, &mut after_derive);
 984         });
 985
 986         (attr, after_derive)
 987     }
 988
 989     fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> {
 990         self.cfg.configure(node)
 991     }
 992
 993     // Detect use of feature-gated or invalid attributes on macro invocations
 994     // since they will not be detected after macro expansion.
 995     fn check_attributes(&mut self, attrs: &[ast::Attribute]) {
 996         let features = self.cx.ecfg.features.unwrap();
 997         for attr in attrs.iter() {
 998             feature_gate::check_attribute(attr, self.cx.parse_sess, features);
 999
1000             // macros are expanded before any lint passes so this warning has to be hardcoded
1001             if attr.path == sym::derive {
1002                 self.cx.struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations")
1003                     .note("this may become a hard error in a future release")
1004                     .emit();
1005             }
1006         }
1007     }
1008 }
1009
1010 impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
1011     fn visit_expr(&mut self, expr: &mut P<ast::Expr>) {
1012         self.cfg.configure_expr(expr);
1013         visit_clobber(expr.deref_mut(), |mut expr| {
1014             self.cfg.configure_expr_kind(&mut expr.kind);
1015
1016             // ignore derives so they remain unused
1017             let (attr, after_derive) = self.classify_nonitem(&mut expr);
1018
1019             if attr.is_some() {
1020                 // Collect the invoc regardless of whether or not attributes are permitted here
1021                 // expansion will eat the attribute so it won't error later.
1022                 attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a));
1023
1024                 // AstFragmentKind::Expr requires the macro to emit an expression.
1025                 return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)),
1026                                           AstFragmentKind::Expr, after_derive)
1027                     .make_expr()
1028                     .into_inner()
1029             }
1030
1031             if let ast::ExprKind::Mac(mac) = expr.kind {
1032                 self.check_attributes(&expr.attrs);
1033                 self.collect_bang(mac, expr.span, AstFragmentKind::Expr)
1034                     .make_expr()
1035                     .into_inner()
1036             } else {
1037                 noop_visit_expr(&mut expr, self);
1038                 expr
1039             }
1040         });
1041     }
1042
1043     fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> {
1044         let mut arm = configure!(self, arm);
1045
1046         let (attr, traits, after_derive) = self.classify_item(&mut arm);
1047         if attr.is_some() || !traits.is_empty() {
1048             return self.collect_attr(attr, traits, Annotatable::Arm(arm),
1049                                      AstFragmentKind::Arms, after_derive)
1050                                      .make_arms();
1051         }
1052
1053         noop_flat_map_arm(arm, self)
1054     }
1055
1056     fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> {
1057         let mut field = configure!(self, field);
1058
1059         let (attr, traits, after_derive) = self.classify_item(&mut field);
1060         if attr.is_some() || !traits.is_empty() {
1061             return self.collect_attr(attr, traits, Annotatable::Field(field),
1062                                      AstFragmentKind::Fields, after_derive)
1063                                      .make_fields();
1064         }
1065
1066         noop_flat_map_field(field, self)
1067     }
1068
1069     fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> {
1070         let mut fp = configure!(self, fp);
1071
1072         let (attr, traits, after_derive) = self.classify_item(&mut fp);
1073         if attr.is_some() || !traits.is_empty() {
1074             return self.collect_attr(attr, traits, Annotatable::FieldPat(fp),
1075                                      AstFragmentKind::FieldPats, after_derive)
1076                                      .make_field_patterns();
1077         }
1078
1079         noop_flat_map_field_pattern(fp, self)
1080     }
1081
1082     fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> {
1083         let mut p = configure!(self, p);
1084
1085         let (attr, traits, after_derive) = self.classify_item(&mut p);
1086         if attr.is_some() || !traits.is_empty() {
1087             return self.collect_attr(attr, traits, Annotatable::Param(p),
1088                                      AstFragmentKind::Params, after_derive)
1089                                      .make_params();
1090         }
1091
1092         noop_flat_map_param(p, self)
1093     }
1094
1095     fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> {
1096         let mut sf = configure!(self, sf);
1097
1098         let (attr, traits, after_derive) = self.classify_item(&mut sf);
1099         if attr.is_some() || !traits.is_empty() {
1100             return self.collect_attr(attr, traits, Annotatable::StructField(sf),
1101                                      AstFragmentKind::StructFields, after_derive)
1102                                      .make_struct_fields();
1103         }
1104
1105         noop_flat_map_struct_field(sf, self)
1106     }
1107
1108     fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> {
1109         let mut variant = configure!(self, variant);
1110
1111         let (attr, traits, after_derive) = self.classify_item(&mut variant);
1112         if attr.is_some() || !traits.is_empty() {
1113             return self.collect_attr(attr, traits, Annotatable::Variant(variant),
1114                                      AstFragmentKind::Variants, after_derive)
1115                                      .make_variants();
1116         }
1117
1118         noop_flat_map_variant(variant, self)
1119     }
1120
1121     fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
1122         let expr = configure!(self, expr);
1123         expr.filter_map(|mut expr| {
1124             self.cfg.configure_expr_kind(&mut expr.kind);
1125
1126             // Ignore derives so they remain unused.
1127             let (attr, after_derive) = self.classify_nonitem(&mut expr);
1128
1129             if attr.is_some() {
1130                 attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a));
1131
1132                 return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)),
1133                                          AstFragmentKind::OptExpr, after_derive)
1134                     .make_opt_expr()
1135                     .map(|expr| expr.into_inner())
1136             }
1137
1138             if let ast::ExprKind::Mac(mac) = expr.kind {
1139                 self.check_attributes(&expr.attrs);
1140                 self.collect_bang(mac, expr.span, AstFragmentKind::OptExpr)
1141                     .make_opt_expr()
1142                     .map(|expr| expr.into_inner())
1143             } else {
1144                 Some({ noop_visit_expr(&mut expr, self); expr })
1145             }
1146         })
1147     }
1148
1149     fn visit_pat(&mut self, pat: &mut P<ast::Pat>) {
1150         self.cfg.configure_pat(pat);
1151         match pat.kind {
1152             PatKind::Mac(_) => {}
1153             _ => return noop_visit_pat(pat, self),
1154         }
1155
1156         visit_clobber(pat, |mut pat| {
1157             match mem::replace(&mut pat.kind, PatKind::Wild) {
1158                 PatKind::Mac(mac) =>
1159                     self.collect_bang(mac, pat.span, AstFragmentKind::Pat).make_pat(),
1160                 _ => unreachable!(),
1161             }
1162         });
1163     }
1164
1165     fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> {
1166         let mut stmt = configure!(self, stmt);
1167
1168         // we'll expand attributes on expressions separately
1169         if !stmt.is_expr() {
1170             let (attr, derives, after_derive) = if stmt.is_item() {
1171                 self.classify_item(&mut stmt)
1172             } else {
1173                 // ignore derives on non-item statements so it falls through
1174                 // to the unused-attributes lint
1175                 let (attr, after_derive) = self.classify_nonitem(&mut stmt);
1176                 (attr, vec![], after_derive)
1177             };
1178
1179             if attr.is_some() || !derives.is_empty() {
1180                 return self.collect_attr(attr, derives, Annotatable::Stmt(P(stmt)),
1181                                          AstFragmentKind::Stmts, after_derive).make_stmts();
1182             }
1183         }
1184
1185         if let StmtKind::Mac(mac) = stmt.kind {
1186             let (mac, style, attrs) = mac.into_inner();
1187             self.check_attributes(&attrs);
1188             let mut placeholder = self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts)
1189                                         .make_stmts();
1190
1191             // If this is a macro invocation with a semicolon, then apply that
1192             // semicolon to the final statement produced by expansion.
1193             if style == MacStmtStyle::Semicolon {
1194                 if let Some(stmt) = placeholder.pop() {
1195                     placeholder.push(stmt.add_trailing_semicolon());
1196                 }
1197             }
1198
1199             return placeholder;
1200         }
1201
1202         // The placeholder expander gives ids to statements, so we avoid folding the id here.
1203         let ast::Stmt { id, kind, span } = stmt;
1204         noop_flat_map_stmt_kind(kind, self).into_iter().map(|kind| {
1205             ast::Stmt { id, kind, span }
1206         }).collect()
1207
1208     }
1209
1210     fn visit_block(&mut self, block: &mut P<Block>) {
1211         let old_directory_ownership = self.cx.current_expansion.directory_ownership;
1212         self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock;
1213         noop_visit_block(block, self);
1214         self.cx.current_expansion.directory_ownership = old_directory_ownership;
1215     }
1216
1217     fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
1218         let mut item = configure!(self, item);
1219
1220         let (attr, traits, after_derive) = self.classify_item(&mut item);
1221         if attr.is_some() || !traits.is_empty() {
1222             return self.collect_attr(attr, traits, Annotatable::Item(item),
1223                                      AstFragmentKind::Items, after_derive).make_items();
1224         }
1225
1226         match item.kind {
1227             ast::ItemKind::Mac(..) => {
1228                 self.check_attributes(&item.attrs);
1229                 item.and_then(|item| match item.kind {
1230                     ItemKind::Mac(mac) => self.collect(
1231                         AstFragmentKind::Items, InvocationKind::Bang { mac, span: item.span }
1232                     ).make_items(),
1233                     _ => unreachable!(),
1234                 })
1235             }
1236             ast::ItemKind::Mod(ast::Mod { inner, .. }) => {
1237                 if item.ident == Ident::invalid() {
1238                     return noop_flat_map_item(item, self);
1239                 }
1240
1241                 let orig_directory_ownership = self.cx.current_expansion.directory_ownership;
1242                 let mut module = (*self.cx.current_expansion.module).clone();
1243                 module.mod_path.push(item.ident);
1244
1245                 // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`).
1246                 // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`).
1247                 // Thus, if `inner` is the dummy span, we know the module is inline.
1248                 let inline_module = item.span.contains(inner) || inner.is_dummy();
1249
1250                 if inline_module {
1251                     if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) {
1252                         self.cx.current_expansion.directory_ownership =
1253                             DirectoryOwnership::Owned { relative: None };
1254                         module.directory.push(&*path.as_str());
1255                     } else {
1256                         module.directory.push(&*item.ident.as_str());
1257                     }
1258                 } else {
1259                     let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner);
1260                     let mut path = match path {
1261                         FileName::Real(path) => path,
1262                         other => PathBuf::from(other.to_string()),
1263                     };
1264                     let directory_ownership = match path.file_name().unwrap().to_str() {
1265                         Some("mod.rs") => DirectoryOwnership::Owned { relative: None },
1266                         Some(_) => DirectoryOwnership::Owned {
1267                             relative: Some(item.ident),
1268                         },
1269                         None => DirectoryOwnership::UnownedViaMod(false),
1270                     };
1271                     path.pop();
1272                     module.directory = path;
1273                     self.cx.current_expansion.directory_ownership = directory_ownership;
1274                 }
1275
1276                 let orig_module =
1277                     mem::replace(&mut self.cx.current_expansion.module, Rc::new(module));
1278                 let result = noop_flat_map_item(item, self);
1279                 self.cx.current_expansion.module = orig_module;
1280                 self.cx.current_expansion.directory_ownership = orig_directory_ownership;
1281                 result
1282             }
1283
1284             _ => noop_flat_map_item(item, self),
1285         }
1286     }
1287
1288     fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> {
1289         let mut item = configure!(self, item);
1290
1291         let (attr, traits, after_derive) = self.classify_item(&mut item);
1292         if attr.is_some() || !traits.is_empty() {
1293             return self.collect_attr(attr, traits, Annotatable::TraitItem(P(item)),
1294                                      AstFragmentKind::TraitItems, after_derive).make_trait_items()
1295         }
1296
1297         match item.kind {
1298             ast::TraitItemKind::Macro(mac) => {
1299                 let ast::TraitItem { attrs, span, .. } = item;
1300                 self.check_attributes(&attrs);
1301                 self.collect_bang(mac, span, AstFragmentKind::TraitItems).make_trait_items()
1302             }
1303             _ => noop_flat_map_trait_item(item, self),
1304         }
1305     }
1306
1307     fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> {
1308         let mut item = configure!(self, item);
1309
1310         let (attr, traits, after_derive) = self.classify_item(&mut item);
1311         if attr.is_some() || !traits.is_empty() {
1312             return self.collect_attr(attr, traits, Annotatable::ImplItem(P(item)),
1313                                      AstFragmentKind::ImplItems, after_derive).make_impl_items();
1314         }
1315
1316         match item.kind {
1317             ast::ImplItemKind::Macro(mac) => {
1318                 let ast::ImplItem { attrs, span, .. } = item;
1319                 self.check_attributes(&attrs);
1320                 self.collect_bang(mac, span, AstFragmentKind::ImplItems).make_impl_items()
1321             }
1322             _ => noop_flat_map_impl_item(item, self),
1323         }
1324     }
1325
1326     fn visit_ty(&mut self, ty: &mut P<ast::Ty>) {
1327         match ty.kind {
1328             ast::TyKind::Mac(_) => {}
1329             _ => return noop_visit_ty(ty, self),
1330         };
1331
1332         visit_clobber(ty, |mut ty| {
1333             match mem::replace(&mut ty.kind, ast::TyKind::Err) {
1334                 ast::TyKind::Mac(mac) =>
1335                     self.collect_bang(mac, ty.span, AstFragmentKind::Ty).make_ty(),
1336                 _ => unreachable!(),
1337             }
1338         });
1339     }
1340
1341     fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) {
1342         self.cfg.configure_foreign_mod(foreign_mod);
1343         noop_visit_foreign_mod(foreign_mod, self);
1344     }
1345
1346     fn flat_map_foreign_item(&mut self, mut foreign_item: ast::ForeignItem)
1347         -> SmallVec<[ast::ForeignItem; 1]>
1348     {
1349         let (attr, traits, after_derive) = self.classify_item(&mut foreign_item);
1350
1351         if attr.is_some() || !traits.is_empty() {
1352             return self.collect_attr(attr, traits, Annotatable::ForeignItem(P(foreign_item)),
1353                                      AstFragmentKind::ForeignItems, after_derive)
1354                                      .make_foreign_items();
1355         }
1356
1357         if let ast::ForeignItemKind::Macro(mac) = foreign_item.kind {
1358             self.check_attributes(&foreign_item.attrs);
1359             return self.collect_bang(mac, foreign_item.span, AstFragmentKind::ForeignItems)
1360                 .make_foreign_items();
1361         }
1362
1363         noop_flat_map_foreign_item(foreign_item, self)
1364     }
1365
1366     fn visit_item_kind(&mut self, item: &mut ast::ItemKind) {
1367         match item {
1368             ast::ItemKind::MacroDef(..) => {}
1369             _ => {
1370                 self.cfg.configure_item_kind(item);
1371                 noop_visit_item_kind(item, self);
1372             }
1373         }
1374     }
1375
1376     fn flat_map_generic_param(
1377         &mut self,
1378         param: ast::GenericParam
1379     ) -> SmallVec<[ast::GenericParam; 1]>
1380     {
1381         let mut param = configure!(self, param);
1382
1383         let (attr, traits, after_derive) = self.classify_item(&mut param);
1384         if attr.is_some() || !traits.is_empty() {
1385             return self.collect_attr(attr, traits, Annotatable::GenericParam(param),
1386                                      AstFragmentKind::GenericParams, after_derive)
1387                                      .make_generic_params();
1388         }
1389
1390         noop_flat_map_generic_param(param, self)
1391     }
1392
1393     fn visit_attribute(&mut self, at: &mut ast::Attribute) {
1394         // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename",
1395         // contents="file contents")]` attributes
1396         if !at.check_name(sym::doc) {
1397             return noop_visit_attribute(at, self);
1398         }
1399
1400         if let Some(list) = at.meta_item_list() {
1401             if !list.iter().any(|it| it.check_name(sym::include)) {
1402                 return noop_visit_attribute(at, self);
1403             }
1404
1405             let mut items = vec![];
1406
1407             for mut it in list {
1408                 if !it.check_name(sym::include) {
1409                     items.push({ noop_visit_meta_list_item(&mut it, self); it });
1410                     continue;
1411                 }
1412
1413                 if let Some(file) = it.value_str() {
1414                     let err_count = self.cx.parse_sess.span_diagnostic.err_count();
1415                     self.check_attributes(slice::from_ref(at));
1416                     if self.cx.parse_sess.span_diagnostic.err_count() > err_count {
1417                         // avoid loading the file if they haven't enabled the feature
1418                         return noop_visit_attribute(at, self);
1419                     }
1420
1421                     let filename = match self.cx.resolve_path(&*file.as_str(), it.span()) {
1422                         Ok(filename) => filename,
1423                         Err(mut err) => {
1424                             err.emit();
1425                             continue;
1426                         }
1427                     };
1428
1429                     match self.cx.source_map().load_file(&filename) {
1430                         Ok(source_file) => {
1431                             let src = source_file.src.as_ref()
1432                                 .expect("freshly loaded file should have a source");
1433                             let src_interned = Symbol::intern(src.as_str());
1434
1435                             let include_info = vec![
1436                                 ast::NestedMetaItem::MetaItem(
1437                                     attr::mk_name_value_item_str(
1438                                         Ident::with_dummy_span(sym::file),
1439                                         file,
1440                                         DUMMY_SP,
1441                                     ),
1442                                 ),
1443                                 ast::NestedMetaItem::MetaItem(
1444                                     attr::mk_name_value_item_str(
1445                                         Ident::with_dummy_span(sym::contents),
1446                                         src_interned,
1447                                         DUMMY_SP,
1448                                     ),
1449                                 ),
1450                             ];
1451
1452                             let include_ident = Ident::with_dummy_span(sym::include);
1453                             let item = attr::mk_list_item(include_ident, include_info);
1454                             items.push(ast::NestedMetaItem::MetaItem(item));
1455                         }
1456                         Err(e) => {
1457                             let lit = it
1458                                 .meta_item()
1459                                 .and_then(|item| item.name_value_literal())
1460                                 .unwrap();
1461
1462                             if e.kind() == ErrorKind::InvalidData {
1463                                 self.cx
1464                                     .struct_span_err(
1465                                         lit.span,
1466                                         &format!("{} wasn't a utf-8 file", filename.display()),
1467                                     )
1468                                     .span_label(lit.span, "contains invalid utf-8")
1469                                     .emit();
1470                             } else {
1471                                 let mut err = self.cx.struct_span_err(
1472                                     lit.span,
1473                                     &format!("couldn't read {}: {}", filename.display(), e),
1474                                 );
1475                                 err.span_label(lit.span, "couldn't read file");
1476
1477                                 err.emit();
1478                             }
1479                         }
1480                     }
1481                 } else {
1482                     let mut err = self.cx.struct_span_err(
1483                         it.span(),
1484                         &format!("expected path to external documentation"),
1485                     );
1486
1487                     // Check if the user erroneously used `doc(include(...))` syntax.
1488                     let literal = it.meta_item_list().and_then(|list| {
1489                         if list.len() == 1 {
1490                             list[0].literal().map(|literal| &literal.kind)
1491                         } else {
1492                             None
1493                         }
1494                     });
1495
1496                     let (path, applicability) = match &literal {
1497                         Some(LitKind::Str(path, ..)) => {
1498                             (path.to_string(), Applicability::MachineApplicable)
1499                         }
1500                         _ => (String::from("<path>"), Applicability::HasPlaceholders),
1501                     };
1502
1503                     err.span_suggestion(
1504                         it.span(),
1505                         "provide a file path with `=`",
1506                         format!("include = \"{}\"", path),
1507                         applicability,
1508                     );
1509
1510                     err.emit();
1511                 }
1512             }
1513
1514             let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items);
1515             *at = attr::Attribute {
1516                 item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) },
1517                 span: at.span,
1518                 id: at.id,
1519                 style: at.style,
1520                 is_sugared_doc: false,
1521             };
1522         } else {
1523             noop_visit_attribute(at, self)
1524         }
1525     }
1526
1527     fn visit_id(&mut self, id: &mut ast::NodeId) {
1528         if self.monotonic {
1529             debug_assert_eq!(*id, ast::DUMMY_NODE_ID);
1530             *id = self.cx.resolver.next_node_id()
1531         }
1532     }
1533
1534     fn visit_fn_decl(&mut self, mut fn_decl: &mut P<ast::FnDecl>) {
1535         self.cfg.configure_fn_decl(&mut fn_decl);
1536         noop_visit_fn_decl(fn_decl, self);
1537     }
1538 }
1539
1540 pub struct ExpansionConfig<'feat> {
1541     pub crate_name: String,
1542     pub features: Option<&'feat Features>,
1543     pub recursion_limit: usize,
1544     pub trace_mac: bool,
1545     pub should_test: bool, // If false, strip `#[test]` nodes
1546     pub single_step: bool,
1547     pub keep_macs: bool,
1548 }
1549
1550 impl<'feat> ExpansionConfig<'feat> {
1551     pub fn default(crate_name: String) -> ExpansionConfig<'static> {
1552         ExpansionConfig {
1553             crate_name,
1554             features: None,
1555             recursion_limit: 1024,
1556             trace_mac: false,
1557             should_test: false,
1558             single_step: false,
1559             keep_macs: false,
1560         }
1561     }
1562
1563     fn proc_macro_hygiene(&self) -> bool {
1564         self.features.map_or(false, |features| features.proc_macro_hygiene)
1565     }
1566     fn custom_inner_attributes(&self) -> bool {
1567         self.features.map_or(false, |features| features.custom_inner_attributes)
1568     }
1569 }