src/librustc_expand/expand.rs

   1 use crate::base::*;
   2 use crate::config::StripUnconfigured;
   3 use crate::hygiene::{ExpnData, ExpnId, ExpnKind, SyntaxContext};
   4 use crate::mbe::macro_rules::annotate_err_with_kind;
   5 use crate::placeholders::{placeholder, PlaceholderExpander};
   6 use crate::proc_macro::collect_derives;
   7
   8 use rustc_ast_pretty::pprust;
   9 use rustc_attr::{self as attr, is_builtin_attr, HasAttrs};
  10 use rustc_data_structures::sync::Lrc;
  11 use rustc_errors::{Applicability, FatalError, PResult};
  12 use rustc_feature::Features;
  13 use rustc_parse::configure;
  14 use rustc_parse::parser::Parser;
  15 use rustc_parse::validate_attr;
  16 use rustc_parse::DirectoryOwnership;
  17 use rustc_session::parse::{feature_err, ParseSess};
  18 use rustc_span::source_map::respan;
  19 use rustc_span::symbol::{sym, Symbol};
  20 use rustc_span::{FileName, Span, DUMMY_SP};
  21 use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path};
  22 use syntax::ast::{ItemKind, MacArgs, MacStmtStyle, StmtKind};
  23 use syntax::mut_visit::*;
  24 use syntax::ptr::P;
  25 use syntax::token;
  26 use syntax::tokenstream::{TokenStream, TokenTree};
  27 use syntax::util::map_in_place::MapInPlace;
  28 use syntax::visit::{self, Visitor};
  29
  30 use smallvec::{smallvec, SmallVec};
  31 use std::io::ErrorKind;
  32 use std::ops::DerefMut;
  33 use std::path::PathBuf;
  34 use std::rc::Rc;
  35 use std::{iter, mem, slice};
  36
  37 macro_rules! ast_fragments {
  38     (
  39         $($Kind:ident($AstTy:ty) {
  40             $kind_name:expr;
  41             $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)?
  42             $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident;)?
  43             fn $make_ast:ident;
  44         })*
  45     ) => {
  46         /// A fragment of AST that can be produced by a single macro expansion.
  47         /// Can also serve as an input and intermediate result for macro expansion operations.
  48         pub enum AstFragment {
  49             OptExpr(Option<P<ast::Expr>>),
  50             $($Kind($AstTy),)*
  51         }
  52
  53         /// "Discriminant" of an AST fragment.
  54         #[derive(Copy, Clone, PartialEq, Eq)]
  55         pub enum AstFragmentKind {
  56             OptExpr,
  57             $($Kind,)*
  58         }
  59
  60         impl AstFragmentKind {
  61             pub fn name(self) -> &'static str {
  62                 match self {
  63                     AstFragmentKind::OptExpr => "expression",
  64                     $(AstFragmentKind::$Kind => $kind_name,)*
  65                 }
  66             }
  67
  68             fn make_from<'a>(self, result: Box<dyn MacResult + 'a>) -> Option<AstFragment> {
  69                 match self {
  70                     AstFragmentKind::OptExpr =>
  71                         result.make_expr().map(Some).map(AstFragment::OptExpr),
  72                     $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)*
  73                 }
  74             }
  75         }
  76
  77         impl AstFragment {
  78             pub fn add_placeholders(&mut self, placeholders: &[NodeId]) {
  79                 if placeholders.is_empty() {
  80                     return;
  81                 }
  82                 match self {
  83                     $($(AstFragment::$Kind(ast) => ast.extend(placeholders.iter().flat_map(|id| {
  84                         // We are repeating through arguments with `many`, to do that we have to
  85                         // mention some macro variable from those arguments even if it's not used.
  86                         macro _repeating($flat_map_ast_elt) {}
  87                         placeholder(AstFragmentKind::$Kind, *id, None).$make_ast()
  88                     })),)?)*
  89                     _ => panic!("unexpected AST fragment kind")
  90                 }
  91             }
  92
  93             pub fn make_opt_expr(self) -> Option<P<ast::Expr>> {
  94                 match self {
  95                     AstFragment::OptExpr(expr) => expr,
  96                     _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
  97                 }
  98             }
  99
 100             $(pub fn $make_ast(self) -> $AstTy {
 101                 match self {
 102                     AstFragment::$Kind(ast) => ast,
 103                     _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
 104                 }
 105             })*
 106
 107             pub fn mut_visit_with<F: MutVisitor>(&mut self, vis: &mut F) {
 108                 match self {
 109                     AstFragment::OptExpr(opt_expr) => {
 110                         visit_clobber(opt_expr, |opt_expr| {
 111                             if let Some(expr) = opt_expr {
 112                                 vis.filter_map_expr(expr)
 113                             } else {
 114                                 None
 115                             }
 116                         });
 117                     }
 118                     $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)*
 119                     $($(AstFragment::$Kind(ast) =>
 120                         ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)*
 121                 }
 122             }
 123
 124             pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
 125                 match *self {
 126                     AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr),
 127                     AstFragment::OptExpr(None) => {}
 128                     $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)*
 129                     $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] {
 130                         visitor.$visit_ast_elt(ast_elt);
 131                     })?)*
 132                 }
 133             }
 134         }
 135
 136         impl<'a> MacResult for crate::mbe::macro_rules::ParserAnyMacro<'a> {
 137             $(fn $make_ast(self: Box<crate::mbe::macro_rules::ParserAnyMacro<'a>>)
 138                            -> Option<$AstTy> {
 139                 Some(self.make(AstFragmentKind::$Kind).$make_ast())
 140             })*
 141         }
 142     }
 143 }
 144
 145 ast_fragments! {
 146     Expr(P<ast::Expr>) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; }
 147     Pat(P<ast::Pat>) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; }
 148     Ty(P<ast::Ty>) { "type"; one fn visit_ty; fn visit_ty; fn make_ty; }
 149     Stmts(SmallVec<[ast::Stmt; 1]>) {
 150         "statement"; many fn flat_map_stmt; fn visit_stmt; fn make_stmts;
 151     }
 152     Items(SmallVec<[P<ast::Item>; 1]>) {
 153         "item"; many fn flat_map_item; fn visit_item; fn make_items;
 154     }
 155     TraitItems(SmallVec<[P<ast::AssocItem>; 1]>) {
 156         "trait item"; many fn flat_map_trait_item; fn visit_trait_item; fn make_trait_items;
 157     }
 158     ImplItems(SmallVec<[P<ast::AssocItem>; 1]>) {
 159         "impl item"; many fn flat_map_impl_item; fn visit_impl_item; fn make_impl_items;
 160     }
 161     ForeignItems(SmallVec<[P<ast::ForeignItem>; 1]>) {
 162         "foreign item";
 163         many fn flat_map_foreign_item;
 164         fn visit_foreign_item;
 165         fn make_foreign_items;
 166     }
 167     Arms(SmallVec<[ast::Arm; 1]>) {
 168         "match arm"; many fn flat_map_arm; fn visit_arm; fn make_arms;
 169     }
 170     Fields(SmallVec<[ast::Field; 1]>) {
 171         "field expression"; many fn flat_map_field; fn visit_field; fn make_fields;
 172     }
 173     FieldPats(SmallVec<[ast::FieldPat; 1]>) {
 174         "field pattern";
 175         many fn flat_map_field_pattern;
 176         fn visit_field_pattern;
 177         fn make_field_patterns;
 178     }
 179     GenericParams(SmallVec<[ast::GenericParam; 1]>) {
 180         "generic parameter";
 181         many fn flat_map_generic_param;
 182         fn visit_generic_param;
 183         fn make_generic_params;
 184     }
 185     Params(SmallVec<[ast::Param; 1]>) {
 186         "function parameter"; many fn flat_map_param; fn visit_param; fn make_params;
 187     }
 188     StructFields(SmallVec<[ast::StructField; 1]>) {
 189         "field";
 190         many fn flat_map_struct_field;
 191         fn visit_struct_field;
 192         fn make_struct_fields;
 193     }
 194     Variants(SmallVec<[ast::Variant; 1]>) {
 195         "variant"; many fn flat_map_variant; fn visit_variant; fn make_variants;
 196     }
 197 }
 198
 199 impl AstFragmentKind {
 200     fn dummy(self, span: Span) -> AstFragment {
 201         self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment")
 202     }
 203
 204     fn expect_from_annotatables<I: IntoIterator<Item = Annotatable>>(
 205         self,
 206         items: I,
 207     ) -> AstFragment {
 208         let mut items = items.into_iter();
 209         match self {
 210             AstFragmentKind::Arms => {
 211                 AstFragment::Arms(items.map(Annotatable::expect_arm).collect())
 212             }
 213             AstFragmentKind::Fields => {
 214                 AstFragment::Fields(items.map(Annotatable::expect_field).collect())
 215             }
 216             AstFragmentKind::FieldPats => {
 217                 AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect())
 218             }
 219             AstFragmentKind::GenericParams => {
 220                 AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect())
 221             }
 222             AstFragmentKind::Params => {
 223                 AstFragment::Params(items.map(Annotatable::expect_param).collect())
 224             }
 225             AstFragmentKind::StructFields => {
 226                 AstFragment::StructFields(items.map(Annotatable::expect_struct_field).collect())
 227             }
 228             AstFragmentKind::Variants => {
 229                 AstFragment::Variants(items.map(Annotatable::expect_variant).collect())
 230             }
 231             AstFragmentKind::Items => {
 232                 AstFragment::Items(items.map(Annotatable::expect_item).collect())
 233             }
 234             AstFragmentKind::ImplItems => {
 235                 AstFragment::ImplItems(items.map(Annotatable::expect_impl_item).collect())
 236             }
 237             AstFragmentKind::TraitItems => {
 238                 AstFragment::TraitItems(items.map(Annotatable::expect_trait_item).collect())
 239             }
 240             AstFragmentKind::ForeignItems => {
 241                 AstFragment::ForeignItems(items.map(Annotatable::expect_foreign_item).collect())
 242             }
 243             AstFragmentKind::Stmts => {
 244                 AstFragment::Stmts(items.map(Annotatable::expect_stmt).collect())
 245             }
 246             AstFragmentKind::Expr => AstFragment::Expr(
 247                 items.next().expect("expected exactly one expression").expect_expr(),
 248             ),
 249             AstFragmentKind::OptExpr => {
 250                 AstFragment::OptExpr(items.next().map(Annotatable::expect_expr))
 251             }
 252             AstFragmentKind::Pat | AstFragmentKind::Ty => {
 253                 panic!("patterns and types aren't annotatable")
 254             }
 255         }
 256     }
 257 }
 258
 259 pub struct Invocation {
 260     pub kind: InvocationKind,
 261     pub fragment_kind: AstFragmentKind,
 262     pub expansion_data: ExpansionData,
 263 }
 264
 265 pub enum InvocationKind {
 266     Bang {
 267         mac: ast::Mac,
 268         span: Span,
 269     },
 270     Attr {
 271         attr: ast::Attribute,
 272         item: Annotatable,
 273         // Required for resolving derive helper attributes.
 274         derives: Vec<Path>,
 275         // We temporarily report errors for attribute macros placed after derives
 276         after_derive: bool,
 277     },
 278     Derive {
 279         path: Path,
 280         item: Annotatable,
 281     },
 282     /// "Invocation" that contains all derives from an item,
 283     /// broken into multiple `Derive` invocations when expanded.
 284     /// FIXME: Find a way to remove it.
 285     DeriveContainer {
 286         derives: Vec<Path>,
 287         item: Annotatable,
 288     },
 289 }
 290
 291 impl InvocationKind {
 292     fn placeholder_visibility(&self) -> Option<ast::Visibility> {
 293         // HACK: For unnamed fields placeholders should have the same visibility as the actual
 294         // fields because for tuple structs/variants resolve determines visibilities of their
 295         // constructor using these field visibilities before attributes on them are are expanded.
 296         // The assumption is that the attribute expansion cannot change field visibilities,
 297         // and it holds because only inert attributes are supported in this position.
 298         match self {
 299             InvocationKind::Attr { item: Annotatable::StructField(field), .. }
 300             | InvocationKind::Derive { item: Annotatable::StructField(field), .. }
 301             | InvocationKind::DeriveContainer { item: Annotatable::StructField(field), .. }
 302                 if field.ident.is_none() =>
 303             {
 304                 Some(field.vis.clone())
 305             }
 306             _ => None,
 307         }
 308     }
 309 }
 310
 311 impl Invocation {
 312     pub fn span(&self) -> Span {
 313         match &self.kind {
 314             InvocationKind::Bang { span, .. } => *span,
 315             InvocationKind::Attr { attr, .. } => attr.span,
 316             InvocationKind::Derive { path, .. } => path.span,
 317             InvocationKind::DeriveContainer { item, .. } => item.span(),
 318         }
 319     }
 320 }
 321
 322 pub struct MacroExpander<'a, 'b> {
 323     pub cx: &'a mut ExtCtxt<'b>,
 324     monotonic: bool, // cf. `cx.monotonic_expander()`
 325 }
 326
 327 impl<'a, 'b> MacroExpander<'a, 'b> {
 328     pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self {
 329         MacroExpander { cx, monotonic }
 330     }
 331
 332     pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate {
 333         let mut module = ModuleData {
 334             mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)],
 335             directory: match self.cx.source_map().span_to_unmapped_path(krate.span) {
 336                 FileName::Real(path) => path,
 337                 other => PathBuf::from(other.to_string()),
 338             },
 339         };
 340         module.directory.pop();
 341         self.cx.root_path = module.directory.clone();
 342         self.cx.current_expansion.module = Rc::new(module);
 343
 344         let orig_mod_span = krate.module.inner;
 345
 346         let krate_item = AstFragment::Items(smallvec![P(ast::Item {
 347             attrs: krate.attrs,
 348             span: krate.span,
 349             kind: ast::ItemKind::Mod(krate.module),
 350             ident: Ident::invalid(),
 351             id: ast::DUMMY_NODE_ID,
 352             vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public),
 353             tokens: None,
 354         })]);
 355
 356         match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) {
 357             Some(ast::Item { attrs, kind: ast::ItemKind::Mod(module), .. }) => {
 358                 krate.attrs = attrs;
 359                 krate.module = module;
 360             }
 361             None => {
 362                 // Resolution failed so we return an empty expansion
 363                 krate.attrs = vec![];
 364                 krate.module = ast::Mod { inner: orig_mod_span, items: vec![], inline: true };
 365             }
 366             Some(ast::Item { span, kind, .. }) => {
 367                 krate.attrs = vec![];
 368                 krate.module = ast::Mod { inner: orig_mod_span, items: vec![], inline: true };
 369                 self.cx.span_err(
 370                     span,
 371                     &format!(
 372                         "expected crate top-level item to be a module after macro expansion, found a {}",
 373                         kind.descriptive_variant()
 374                     ),
 375                 );
 376             }
 377         };
 378         self.cx.trace_macros_diag();
 379         krate
 380     }
 381
 382     // Recursively expand all macro invocations in this AST fragment.
 383     pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment {
 384         let orig_expansion_data = self.cx.current_expansion.clone();
 385         self.cx.current_expansion.depth = 0;
 386
 387         // Collect all macro invocations and replace them with placeholders.
 388         let (mut fragment_with_placeholders, mut invocations) =
 389             self.collect_invocations(input_fragment, &[]);
 390
 391         // Optimization: if we resolve all imports now,
 392         // we'll be able to immediately resolve most of imported macros.
 393         self.resolve_imports();
 394
 395         // Resolve paths in all invocations and produce output expanded fragments for them, but
 396         // do not insert them into our input AST fragment yet, only store in `expanded_fragments`.
 397         // The output fragments also go through expansion recursively until no invocations are left.
 398         // Unresolved macros produce dummy outputs as a recovery measure.
 399         invocations.reverse();
 400         let mut expanded_fragments = Vec::new();
 401         let mut undetermined_invocations = Vec::new();
 402         let (mut progress, mut force) = (false, !self.monotonic);
 403         loop {
 404             let invoc = if let Some(invoc) = invocations.pop() {
 405                 invoc
 406             } else {
 407                 self.resolve_imports();
 408                 if undetermined_invocations.is_empty() {
 409                     break;
 410                 }
 411                 invocations = mem::take(&mut undetermined_invocations);
 412                 force = !mem::replace(&mut progress, false);
 413                 continue;
 414             };
 415
 416             let eager_expansion_root =
 417                 if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id };
 418             let res = match self.cx.resolver.resolve_macro_invocation(
 419                 &invoc,
 420                 eager_expansion_root,
 421                 force,
 422             ) {
 423                 Ok(res) => res,
 424                 Err(Indeterminate) => {
 425                     undetermined_invocations.push(invoc);
 426                     continue;
 427                 }
 428             };
 429
 430             progress = true;
 431             let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data;
 432             self.cx.current_expansion = invoc.expansion_data.clone();
 433
 434             // FIXME(jseyfried): Refactor out the following logic
 435             let (expanded_fragment, new_invocations) = match res {
 436                 InvocationRes::Single(ext) => {
 437                     let fragment = self.expand_invoc(invoc, &ext.kind);
 438                     self.collect_invocations(fragment, &[])
 439                 }
 440                 InvocationRes::DeriveContainer(_exts) => {
 441                     // FIXME: Consider using the derive resolutions (`_exts`) immediately,
 442                     // instead of enqueuing the derives to be resolved again later.
 443                     let (derives, item) = match invoc.kind {
 444                         InvocationKind::DeriveContainer { derives, item } => (derives, item),
 445                         _ => unreachable!(),
 446                     };
 447                     if !item.derive_allowed() {
 448                         let attr = attr::find_by_name(item.attrs(), sym::derive)
 449                             .expect("`derive` attribute should exist");
 450                         let span = attr.span;
 451                         let mut err = self.cx.struct_span_err(
 452                             span,
 453                             "`derive` may only be applied to structs, enums and unions",
 454                         );
 455                         if let ast::AttrStyle::Inner = attr.style {
 456                             let trait_list = derives
 457                                 .iter()
 458                                 .map(|t| pprust::path_to_string(t))
 459                                 .collect::<Vec<_>>();
 460                             let suggestion = format!("#[derive({})]", trait_list.join(", "));
 461                             err.span_suggestion(
 462                                 span,
 463                                 "try an outer attribute",
 464                                 suggestion,
 465                                 // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT
 466                                 Applicability::MaybeIncorrect,
 467                             );
 468                         }
 469                         err.emit();
 470                     }
 471
 472                     let mut item = self.fully_configure(item);
 473                     item.visit_attrs(|attrs| attrs.retain(|a| !a.has_name(sym::derive)));
 474
 475                     let mut derive_placeholders = Vec::with_capacity(derives.len());
 476                     invocations.reserve(derives.len());
 477                     for path in derives {
 478                         let expn_id = ExpnId::fresh(None);
 479                         derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id));
 480                         invocations.push(Invocation {
 481                             kind: InvocationKind::Derive { path, item: item.clone() },
 482                             fragment_kind: invoc.fragment_kind,
 483                             expansion_data: ExpansionData {
 484                                 id: expn_id,
 485                                 ..invoc.expansion_data.clone()
 486                             },
 487                         });
 488                     }
 489                     let fragment =
 490                         invoc.fragment_kind.expect_from_annotatables(::std::iter::once(item));
 491                     self.collect_invocations(fragment, &derive_placeholders)
 492                 }
 493             };
 494
 495             if expanded_fragments.len() < depth {
 496                 expanded_fragments.push(Vec::new());
 497             }
 498             expanded_fragments[depth - 1].push((expn_id, expanded_fragment));
 499             if !self.cx.ecfg.single_step {
 500                 invocations.extend(new_invocations.into_iter().rev());
 501             }
 502         }
 503
 504         self.cx.current_expansion = orig_expansion_data;
 505
 506         // Finally incorporate all the expanded macros into the input AST fragment.
 507         let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic);
 508         while let Some(expanded_fragments) = expanded_fragments.pop() {
 509             for (expn_id, expanded_fragment) in expanded_fragments.into_iter().rev() {
 510                 placeholder_expander
 511                     .add(NodeId::placeholder_from_expn_id(expn_id), expanded_fragment);
 512             }
 513         }
 514         fragment_with_placeholders.mut_visit_with(&mut placeholder_expander);
 515         fragment_with_placeholders
 516     }
 517
 518     fn resolve_imports(&mut self) {
 519         if self.monotonic {
 520             self.cx.resolver.resolve_imports();
 521         }
 522     }
 523
 524     /// Collects all macro invocations reachable at this time in this AST fragment, and replace
 525     /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s.
 526     /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and
 527     /// prepares data for resolving paths of macro invocations.
 528     fn collect_invocations(
 529         &mut self,
 530         mut fragment: AstFragment,
 531         extra_placeholders: &[NodeId],
 532     ) -> (AstFragment, Vec<Invocation>) {
 533         // Resolve `$crate`s in the fragment for pretty-printing.
 534         self.cx.resolver.resolve_dollar_crates();
 535
 536         let invocations = {
 537             let mut collector = InvocationCollector {
 538                 cfg: StripUnconfigured {
 539                     sess: self.cx.parse_sess,
 540                     features: self.cx.ecfg.features,
 541                 },
 542                 cx: self.cx,
 543                 invocations: Vec::new(),
 544                 monotonic: self.monotonic,
 545             };
 546             fragment.mut_visit_with(&mut collector);
 547             fragment.add_placeholders(extra_placeholders);
 548             collector.invocations
 549         };
 550
 551         if self.monotonic {
 552             self.cx
 553                 .resolver
 554                 .visit_ast_fragment_with_placeholders(self.cx.current_expansion.id, &fragment);
 555         }
 556
 557         (fragment, invocations)
 558     }
 559
 560     fn fully_configure(&mut self, item: Annotatable) -> Annotatable {
 561         let mut cfg =
 562             StripUnconfigured { sess: self.cx.parse_sess, features: self.cx.ecfg.features };
 563         // Since the item itself has already been configured by the InvocationCollector,
 564         // we know that fold result vector will contain exactly one element
 565         match item {
 566             Annotatable::Item(item) => Annotatable::Item(cfg.flat_map_item(item).pop().unwrap()),
 567             Annotatable::TraitItem(item) => {
 568                 Annotatable::TraitItem(cfg.flat_map_trait_item(item).pop().unwrap())
 569             }
 570             Annotatable::ImplItem(item) => {
 571                 Annotatable::ImplItem(cfg.flat_map_impl_item(item).pop().unwrap())
 572             }
 573             Annotatable::ForeignItem(item) => {
 574                 Annotatable::ForeignItem(cfg.flat_map_foreign_item(item).pop().unwrap())
 575             }
 576             Annotatable::Stmt(stmt) => {
 577                 Annotatable::Stmt(stmt.map(|stmt| cfg.flat_map_stmt(stmt).pop().unwrap()))
 578             }
 579             Annotatable::Expr(mut expr) => Annotatable::Expr({
 580                 cfg.visit_expr(&mut expr);
 581                 expr
 582             }),
 583             Annotatable::Arm(arm) => Annotatable::Arm(cfg.flat_map_arm(arm).pop().unwrap()),
 584             Annotatable::Field(field) => {
 585                 Annotatable::Field(cfg.flat_map_field(field).pop().unwrap())
 586             }
 587             Annotatable::FieldPat(fp) => {
 588                 Annotatable::FieldPat(cfg.flat_map_field_pattern(fp).pop().unwrap())
 589             }
 590             Annotatable::GenericParam(param) => {
 591                 Annotatable::GenericParam(cfg.flat_map_generic_param(param).pop().unwrap())
 592             }
 593             Annotatable::Param(param) => {
 594                 Annotatable::Param(cfg.flat_map_param(param).pop().unwrap())
 595             }
 596             Annotatable::StructField(sf) => {
 597                 Annotatable::StructField(cfg.flat_map_struct_field(sf).pop().unwrap())
 598             }
 599             Annotatable::Variant(v) => Annotatable::Variant(cfg.flat_map_variant(v).pop().unwrap()),
 600         }
 601     }
 602
 603     fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment {
 604         if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit {
 605             let expn_data = self.cx.current_expansion.id.expn_data();
 606             let suggested_limit = self.cx.ecfg.recursion_limit * 2;
 607             let mut err = self.cx.struct_span_err(
 608                 expn_data.call_site,
 609                 &format!("recursion limit reached while expanding `{}`", expn_data.kind.descr()),
 610             );
 611             err.help(&format!(
 612                 "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate (`{}`)",
 613                 suggested_limit, self.cx.ecfg.crate_name,
 614             ));
 615             err.emit();
 616             self.cx.trace_macros_diag();
 617             FatalError.raise();
 618         }
 619
 620         let (fragment_kind, span) = (invoc.fragment_kind, invoc.span());
 621         match invoc.kind {
 622             InvocationKind::Bang { mac, .. } => match ext {
 623                 SyntaxExtensionKind::Bang(expander) => {
 624                     self.gate_proc_macro_expansion_kind(span, fragment_kind);
 625                     let tok_result = expander.expand(self.cx, span, mac.args.inner_tokens());
 626                     self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span)
 627                 }
 628                 SyntaxExtensionKind::LegacyBang(expander) => {
 629                     let prev = self.cx.current_expansion.prior_type_ascription;
 630                     self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription;
 631                     let tok_result = expander.expand(self.cx, span, mac.args.inner_tokens());
 632                     let result = if let Some(result) = fragment_kind.make_from(tok_result) {
 633                         result
 634                     } else {
 635                         let msg = format!(
 636                             "non-{kind} macro in {kind} position: {path}",
 637                             kind = fragment_kind.name(),
 638                             path = pprust::path_to_string(&mac.path),
 639                         );
 640                         self.cx.span_err(span, &msg);
 641                         self.cx.trace_macros_diag();
 642                         fragment_kind.dummy(span)
 643                     };
 644                     self.cx.current_expansion.prior_type_ascription = prev;
 645                     result
 646                 }
 647                 _ => unreachable!(),
 648             },
 649             InvocationKind::Attr { attr, mut item, .. } => match ext {
 650                 SyntaxExtensionKind::Attr(expander) => {
 651                     self.gate_proc_macro_input(&item);
 652                     self.gate_proc_macro_attr_item(span, &item);
 653                     let item_tok = TokenTree::token(
 654                         token::Interpolated(Lrc::new(match item {
 655                             Annotatable::Item(item) => token::NtItem(item),
 656                             Annotatable::TraitItem(item) => token::NtTraitItem(item),
 657                             Annotatable::ImplItem(item) => token::NtImplItem(item),
 658                             Annotatable::ForeignItem(item) => token::NtForeignItem(item),
 659                             Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
 660                             Annotatable::Expr(expr) => token::NtExpr(expr),
 661                             Annotatable::Arm(..)
 662                             | Annotatable::Field(..)
 663                             | Annotatable::FieldPat(..)
 664                             | Annotatable::GenericParam(..)
 665                             | Annotatable::Param(..)
 666                             | Annotatable::StructField(..)
 667                             | Annotatable::Variant(..) => panic!("unexpected annotatable"),
 668                         })),
 669                         DUMMY_SP,
 670                     )
 671                     .into();
 672                     let item = attr.unwrap_normal_item();
 673                     if let MacArgs::Eq(..) = item.args {
 674                         self.cx.span_err(span, "key-value macro attributes are not supported");
 675                     }
 676                     let tok_result =
 677                         expander.expand(self.cx, span, item.args.inner_tokens(), item_tok);
 678                     self.parse_ast_fragment(tok_result, fragment_kind, &item.path, span)
 679                 }
 680                 SyntaxExtensionKind::LegacyAttr(expander) => {
 681                     match validate_attr::parse_meta(self.cx.parse_sess, &attr) {
 682                         Ok(meta) => {
 683                             let item = expander.expand(self.cx, span, &meta, item);
 684                             fragment_kind.expect_from_annotatables(item)
 685                         }
 686                         Err(mut err) => {
 687                             err.emit();
 688                             fragment_kind.dummy(span)
 689                         }
 690                     }
 691                 }
 692                 SyntaxExtensionKind::NonMacroAttr { mark_used } => {
 693                     attr::mark_known(&attr);
 694                     if *mark_used {
 695                         attr::mark_used(&attr);
 696                     }
 697                     item.visit_attrs(|attrs| attrs.push(attr));
 698                     fragment_kind.expect_from_annotatables(iter::once(item))
 699                 }
 700                 _ => unreachable!(),
 701             },
 702             InvocationKind::Derive { path, item } => match ext {
 703                 SyntaxExtensionKind::Derive(expander)
 704                 | SyntaxExtensionKind::LegacyDerive(expander) => {
 705                     if !item.derive_allowed() {
 706                         return fragment_kind.dummy(span);
 707                     }
 708                     if let SyntaxExtensionKind::Derive(..) = ext {
 709                         self.gate_proc_macro_input(&item);
 710                     }
 711                     let meta = ast::MetaItem { kind: ast::MetaItemKind::Word, span, path };
 712                     let items = expander.expand(self.cx, span, &meta, item);
 713                     fragment_kind.expect_from_annotatables(items)
 714                 }
 715                 _ => unreachable!(),
 716             },
 717             InvocationKind::DeriveContainer { .. } => unreachable!(),
 718         }
 719     }
 720
 721     fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) {
 722         let kind = match item {
 723             Annotatable::Item(_)
 724             | Annotatable::TraitItem(_)
 725             | Annotatable::ImplItem(_)
 726             | Annotatable::ForeignItem(_) => return,
 727             Annotatable::Stmt(_) => "statements",
 728             Annotatable::Expr(_) => "expressions",
 729             Annotatable::Arm(..)
 730             | Annotatable::Field(..)
 731             | Annotatable::FieldPat(..)
 732             | Annotatable::GenericParam(..)
 733             | Annotatable::Param(..)
 734             | Annotatable::StructField(..)
 735             | Annotatable::Variant(..) => panic!("unexpected annotatable"),
 736         };
 737         if self.cx.ecfg.proc_macro_hygiene() {
 738             return;
 739         }
 740         feature_err(
 741             self.cx.parse_sess,
 742             sym::proc_macro_hygiene,
 743             span,
 744             &format!("custom attributes cannot be applied to {}", kind),
 745         )
 746         .emit();
 747     }
 748
 749     fn gate_proc_macro_input(&self, annotatable: &Annotatable) {
 750         struct GateProcMacroInput<'a> {
 751             parse_sess: &'a ParseSess,
 752         }
 753
 754         impl<'ast, 'a> Visitor<'ast> for GateProcMacroInput<'a> {
 755             fn visit_item(&mut self, item: &'ast ast::Item) {
 756                 match &item.kind {
 757                     ast::ItemKind::Mod(module) if !module.inline => {
 758                         feature_err(
 759                             self.parse_sess,
 760                             sym::proc_macro_hygiene,
 761                             item.span,
 762                             "non-inline modules in proc macro input are unstable",
 763                         )
 764                         .emit();
 765                     }
 766                     _ => {}
 767                 }
 768
 769                 visit::walk_item(self, item);
 770             }
 771
 772             fn visit_mac(&mut self, _: &'ast ast::Mac) {}
 773         }
 774
 775         if !self.cx.ecfg.proc_macro_hygiene() {
 776             annotatable.visit_with(&mut GateProcMacroInput { parse_sess: self.cx.parse_sess });
 777         }
 778     }
 779
 780     fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) {
 781         let kind = match kind {
 782             AstFragmentKind::Expr | AstFragmentKind::OptExpr => "expressions",
 783             AstFragmentKind::Pat => "patterns",
 784             AstFragmentKind::Stmts => "statements",
 785             AstFragmentKind::Ty
 786             | AstFragmentKind::Items
 787             | AstFragmentKind::TraitItems
 788             | AstFragmentKind::ImplItems
 789             | AstFragmentKind::ForeignItems => return,
 790             AstFragmentKind::Arms
 791             | AstFragmentKind::Fields
 792             | AstFragmentKind::FieldPats
 793             | AstFragmentKind::GenericParams
 794             | AstFragmentKind::Params
 795             | AstFragmentKind::StructFields
 796             | AstFragmentKind::Variants => panic!("unexpected AST fragment kind"),
 797         };
 798         if self.cx.ecfg.proc_macro_hygiene() {
 799             return;
 800         }
 801         feature_err(
 802             self.cx.parse_sess,
 803             sym::proc_macro_hygiene,
 804             span,
 805             &format!("procedural macros cannot be expanded to {}", kind),
 806         )
 807         .emit();
 808     }
 809
 810     fn parse_ast_fragment(
 811         &mut self,
 812         toks: TokenStream,
 813         kind: AstFragmentKind,
 814         path: &Path,
 815         span: Span,
 816     ) -> AstFragment {
 817         let mut parser = self.cx.new_parser_from_tts(toks);
 818         match parse_ast_fragment(&mut parser, kind, false) {
 819             Ok(fragment) => {
 820                 ensure_complete_parse(&mut parser, path, kind.name(), span);
 821                 fragment
 822             }
 823             Err(mut err) => {
 824                 err.set_span(span);
 825                 annotate_err_with_kind(&mut err, kind, span);
 826                 err.emit();
 827                 self.cx.trace_macros_diag();
 828                 kind.dummy(span)
 829             }
 830         }
 831     }
 832 }
 833
 834 pub fn parse_ast_fragment<'a>(
 835     this: &mut Parser<'a>,
 836     kind: AstFragmentKind,
 837     macro_legacy_warnings: bool,
 838 ) -> PResult<'a, AstFragment> {
 839     Ok(match kind {
 840         AstFragmentKind::Items => {
 841             let mut items = SmallVec::new();
 842             while let Some(item) = this.parse_item()? {
 843                 items.push(item);
 844             }
 845             AstFragment::Items(items)
 846         }
 847         AstFragmentKind::TraitItems => {
 848             let mut items = SmallVec::new();
 849             while this.token != token::Eof {
 850                 items.push(this.parse_trait_item(&mut false)?);
 851             }
 852             AstFragment::TraitItems(items)
 853         }
 854         AstFragmentKind::ImplItems => {
 855             let mut items = SmallVec::new();
 856             while this.token != token::Eof {
 857                 items.push(this.parse_impl_item(&mut false)?);
 858             }
 859             AstFragment::ImplItems(items)
 860         }
 861         AstFragmentKind::ForeignItems => {
 862             let mut items = SmallVec::new();
 863             while this.token != token::Eof {
 864                 items.push(this.parse_foreign_item(DUMMY_SP)?);
 865             }
 866             AstFragment::ForeignItems(items)
 867         }
 868         AstFragmentKind::Stmts => {
 869             let mut stmts = SmallVec::new();
 870             while this.token != token::Eof &&
 871                     // won't make progress on a `}`
 872                     this.token != token::CloseDelim(token::Brace)
 873             {
 874                 if let Some(stmt) = this.parse_full_stmt(macro_legacy_warnings)? {
 875                     stmts.push(stmt);
 876                 }
 877             }
 878             AstFragment::Stmts(stmts)
 879         }
 880         AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?),
 881         AstFragmentKind::OptExpr => {
 882             if this.token != token::Eof {
 883                 AstFragment::OptExpr(Some(this.parse_expr()?))
 884             } else {
 885                 AstFragment::OptExpr(None)
 886             }
 887         }
 888         AstFragmentKind::Ty => AstFragment::Ty(this.parse_ty()?),
 889         AstFragmentKind::Pat => AstFragment::Pat(this.parse_pat(None)?),
 890         AstFragmentKind::Arms
 891         | AstFragmentKind::Fields
 892         | AstFragmentKind::FieldPats
 893         | AstFragmentKind::GenericParams
 894         | AstFragmentKind::Params
 895         | AstFragmentKind::StructFields
 896         | AstFragmentKind::Variants => panic!("unexpected AST fragment kind"),
 897     })
 898 }
 899
 900 pub fn ensure_complete_parse<'a>(
 901     this: &mut Parser<'a>,
 902     macro_path: &Path,
 903     kind_name: &str,
 904     span: Span,
 905 ) {
 906     if this.token != token::Eof {
 907         let token = pprust::token_to_string(&this.token);
 908         let msg = format!("macro expansion ignores token `{}` and any following", token);
 909         // Avoid emitting backtrace info twice.
 910         let def_site_span = this.token.span.with_ctxt(SyntaxContext::root());
 911         let mut err = this.struct_span_err(def_site_span, &msg);
 912         err.span_label(span, "caused by the macro expansion here");
 913         let msg = format!(
 914             "the usage of `{}!` is likely invalid in {} context",
 915             pprust::path_to_string(macro_path),
 916             kind_name,
 917         );
 918         err.note(&msg);
 919         let semi_span = this.sess.source_map().next_point(span);
 920
 921         let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span));
 922         match this.sess.source_map().span_to_snippet(semi_full_span) {
 923             Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => {
 924                 err.span_suggestion(
 925                     semi_span,
 926                     "you might be missing a semicolon here",
 927                     ";".to_owned(),
 928                     Applicability::MaybeIncorrect,
 929                 );
 930             }
 931             _ => {}
 932         }
 933         err.emit();
 934     }
 935 }
 936
 937 struct InvocationCollector<'a, 'b> {
 938     cx: &'a mut ExtCtxt<'b>,
 939     cfg: StripUnconfigured<'a>,
 940     invocations: Vec<Invocation>,
 941     monotonic: bool,
 942 }
 943
 944 impl<'a, 'b> InvocationCollector<'a, 'b> {
 945     fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment {
 946         // Expansion data for all the collected invocations is set upon their resolution,
 947         // with exception of the derive container case which is not resolved and can get
 948         // its expansion data immediately.
 949         let expn_data = match &kind {
 950             InvocationKind::DeriveContainer { item, .. } => Some(ExpnData {
 951                 parent: self.cx.current_expansion.id,
 952                 ..ExpnData::default(
 953                     ExpnKind::Macro(MacroKind::Attr, sym::derive),
 954                     item.span(),
 955                     self.cx.parse_sess.edition,
 956                 )
 957             }),
 958             _ => None,
 959         };
 960         let expn_id = ExpnId::fresh(expn_data);
 961         let vis = kind.placeholder_visibility();
 962         self.invocations.push(Invocation {
 963             kind,
 964             fragment_kind,
 965             expansion_data: ExpansionData {
 966                 id: expn_id,
 967                 depth: self.cx.current_expansion.depth + 1,
 968                 ..self.cx.current_expansion.clone()
 969             },
 970         });
 971         placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id), vis)
 972     }
 973
 974     fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment {
 975         self.collect(kind, InvocationKind::Bang { mac, span })
 976     }
 977
 978     fn collect_attr(
 979         &mut self,
 980         attr: Option<ast::Attribute>,
 981         derives: Vec<Path>,
 982         item: Annotatable,
 983         kind: AstFragmentKind,
 984         after_derive: bool,
 985     ) -> AstFragment {
 986         self.collect(
 987             kind,
 988             match attr {
 989                 Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive },
 990                 None => InvocationKind::DeriveContainer { derives, item },
 991             },
 992         )
 993     }
 994
 995     fn find_attr_invoc(
 996         &self,
 997         attrs: &mut Vec<ast::Attribute>,
 998         after_derive: &mut bool,
 999     ) -> Option<ast::Attribute> {
1000         let attr = attrs
1001             .iter()
1002             .position(|a| {
1003                 if a.has_name(sym::derive) {
1004                     *after_derive = true;
1005                 }
1006                 !attr::is_known(a) && !is_builtin_attr(a)
1007             })
1008             .map(|i| attrs.remove(i));
1009         if let Some(attr) = &attr {
1010             if !self.cx.ecfg.custom_inner_attributes()
1011                 && attr.style == ast::AttrStyle::Inner
1012                 && !attr.has_name(sym::test)
1013             {
1014                 feature_err(
1015                     &self.cx.parse_sess,
1016                     sym::custom_inner_attributes,
1017                     attr.span,
1018                     "non-builtin inner attributes are unstable",
1019                 )
1020                 .emit();
1021             }
1022         }
1023         attr
1024     }
1025
1026     /// If `item` is an attr invocation, remove and return the macro attribute and derive traits.
1027     fn classify_item<T>(
1028         &mut self,
1029         item: &mut T,
1030     ) -> (Option<ast::Attribute>, Vec<Path>, /* after_derive */ bool)
1031     where
1032         T: HasAttrs,
1033     {
1034         let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false);
1035
1036         item.visit_attrs(|mut attrs| {
1037             attr = self.find_attr_invoc(&mut attrs, &mut after_derive);
1038             traits = collect_derives(&mut self.cx, &mut attrs);
1039         });
1040
1041         (attr, traits, after_derive)
1042     }
1043
1044     /// Alternative to `classify_item()` that ignores `#[derive]` so invocations fallthrough
1045     /// to the unused-attributes lint (making it an error on statements and expressions
1046     /// is a breaking change)
1047     fn classify_nonitem<T: HasAttrs>(
1048         &mut self,
1049         nonitem: &mut T,
1050     ) -> (Option<ast::Attribute>, /* after_derive */ bool) {
1051         let (mut attr, mut after_derive) = (None, false);
1052
1053         nonitem.visit_attrs(|mut attrs| {
1054             attr = self.find_attr_invoc(&mut attrs, &mut after_derive);
1055         });
1056
1057         (attr, after_derive)
1058     }
1059
1060     fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> {
1061         self.cfg.configure(node)
1062     }
1063
1064     // Detect use of feature-gated or invalid attributes on macro invocations
1065     // since they will not be detected after macro expansion.
1066     fn check_attributes(&mut self, attrs: &[ast::Attribute]) {
1067         let features = self.cx.ecfg.features.unwrap();
1068         for attr in attrs.iter() {
1069             rustc_ast_passes::feature_gate::check_attribute(attr, self.cx.parse_sess, features);
1070             validate_attr::check_meta(self.cx.parse_sess, attr);
1071
1072             // macros are expanded before any lint passes so this warning has to be hardcoded
1073             if attr.has_name(sym::derive) {
1074                 self.cx
1075                     .struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations")
1076                     .note("this may become a hard error in a future release")
1077                     .emit();
1078             }
1079         }
1080     }
1081 }
1082
1083 impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
1084     fn visit_expr(&mut self, expr: &mut P<ast::Expr>) {
1085         self.cfg.configure_expr(expr);
1086         visit_clobber(expr.deref_mut(), |mut expr| {
1087             self.cfg.configure_expr_kind(&mut expr.kind);
1088
1089             // ignore derives so they remain unused
1090             let (attr, after_derive) = self.classify_nonitem(&mut expr);
1091
1092             if attr.is_some() {
1093                 // Collect the invoc regardless of whether or not attributes are permitted here
1094                 // expansion will eat the attribute so it won't error later.
1095                 attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a));
1096
1097                 // AstFragmentKind::Expr requires the macro to emit an expression.
1098                 return self
1099                     .collect_attr(
1100                         attr,
1101                         vec![],
1102                         Annotatable::Expr(P(expr)),
1103                         AstFragmentKind::Expr,
1104                         after_derive,
1105                     )
1106                     .make_expr()
1107                     .into_inner();
1108             }
1109
1110             if let ast::ExprKind::Mac(mac) = expr.kind {
1111                 self.check_attributes(&expr.attrs);
1112                 self.collect_bang(mac, expr.span, AstFragmentKind::Expr).make_expr().into_inner()
1113             } else {
1114                 noop_visit_expr(&mut expr, self);
1115                 expr
1116             }
1117         });
1118     }
1119
1120     fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> {
1121         let mut arm = configure!(self, arm);
1122
1123         let (attr, traits, after_derive) = self.classify_item(&mut arm);
1124         if attr.is_some() || !traits.is_empty() {
1125             return self
1126                 .collect_attr(
1127                     attr,
1128                     traits,
1129                     Annotatable::Arm(arm),
1130                     AstFragmentKind::Arms,
1131                     after_derive,
1132                 )
1133                 .make_arms();
1134         }
1135
1136         noop_flat_map_arm(arm, self)
1137     }
1138
1139     fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> {
1140         let mut field = configure!(self, field);
1141
1142         let (attr, traits, after_derive) = self.classify_item(&mut field);
1143         if attr.is_some() || !traits.is_empty() {
1144             return self
1145                 .collect_attr(
1146                     attr,
1147                     traits,
1148                     Annotatable::Field(field),
1149                     AstFragmentKind::Fields,
1150                     after_derive,
1151                 )
1152                 .make_fields();
1153         }
1154
1155         noop_flat_map_field(field, self)
1156     }
1157
1158     fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> {
1159         let mut fp = configure!(self, fp);
1160
1161         let (attr, traits, after_derive) = self.classify_item(&mut fp);
1162         if attr.is_some() || !traits.is_empty() {
1163             return self
1164                 .collect_attr(
1165                     attr,
1166                     traits,
1167                     Annotatable::FieldPat(fp),
1168                     AstFragmentKind::FieldPats,
1169                     after_derive,
1170                 )
1171                 .make_field_patterns();
1172         }
1173
1174         noop_flat_map_field_pattern(fp, self)
1175     }
1176
1177     fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> {
1178         let mut p = configure!(self, p);
1179
1180         let (attr, traits, after_derive) = self.classify_item(&mut p);
1181         if attr.is_some() || !traits.is_empty() {
1182             return self
1183                 .collect_attr(
1184                     attr,
1185                     traits,
1186                     Annotatable::Param(p),
1187                     AstFragmentKind::Params,
1188                     after_derive,
1189                 )
1190                 .make_params();
1191         }
1192
1193         noop_flat_map_param(p, self)
1194     }
1195
1196     fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> {
1197         let mut sf = configure!(self, sf);
1198
1199         let (attr, traits, after_derive) = self.classify_item(&mut sf);
1200         if attr.is_some() || !traits.is_empty() {
1201             return self
1202                 .collect_attr(
1203                     attr,
1204                     traits,
1205                     Annotatable::StructField(sf),
1206                     AstFragmentKind::StructFields,
1207                     after_derive,
1208                 )
1209                 .make_struct_fields();
1210         }
1211
1212         noop_flat_map_struct_field(sf, self)
1213     }
1214
1215     fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> {
1216         let mut variant = configure!(self, variant);
1217
1218         let (attr, traits, after_derive) = self.classify_item(&mut variant);
1219         if attr.is_some() || !traits.is_empty() {
1220             return self
1221                 .collect_attr(
1222                     attr,
1223                     traits,
1224                     Annotatable::Variant(variant),
1225                     AstFragmentKind::Variants,
1226                     after_derive,
1227                 )
1228                 .make_variants();
1229         }
1230
1231         noop_flat_map_variant(variant, self)
1232     }
1233
1234     fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
1235         let expr = configure!(self, expr);
1236         expr.filter_map(|mut expr| {
1237             self.cfg.configure_expr_kind(&mut expr.kind);
1238
1239             // Ignore derives so they remain unused.
1240             let (attr, after_derive) = self.classify_nonitem(&mut expr);
1241
1242             if attr.is_some() {
1243                 attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a));
1244
1245                 return self
1246                     .collect_attr(
1247                         attr,
1248                         vec![],
1249                         Annotatable::Expr(P(expr)),
1250                         AstFragmentKind::OptExpr,
1251                         after_derive,
1252                     )
1253                     .make_opt_expr()
1254                     .map(|expr| expr.into_inner());
1255             }
1256
1257             if let ast::ExprKind::Mac(mac) = expr.kind {
1258                 self.check_attributes(&expr.attrs);
1259                 self.collect_bang(mac, expr.span, AstFragmentKind::OptExpr)
1260                     .make_opt_expr()
1261                     .map(|expr| expr.into_inner())
1262             } else {
1263                 Some({
1264                     noop_visit_expr(&mut expr, self);
1265                     expr
1266                 })
1267             }
1268         })
1269     }
1270
1271     fn visit_pat(&mut self, pat: &mut P<ast::Pat>) {
1272         self.cfg.configure_pat(pat);
1273         match pat.kind {
1274             PatKind::Mac(_) => {}
1275             _ => return noop_visit_pat(pat, self),
1276         }
1277
1278         visit_clobber(pat, |mut pat| match mem::replace(&mut pat.kind, PatKind::Wild) {
1279             PatKind::Mac(mac) => self.collect_bang(mac, pat.span, AstFragmentKind::Pat).make_pat(),
1280             _ => unreachable!(),
1281         });
1282     }
1283
1284     fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> {
1285         let mut stmt = configure!(self, stmt);
1286
1287         // we'll expand attributes on expressions separately
1288         if !stmt.is_expr() {
1289             let (attr, derives, after_derive) = if stmt.is_item() {
1290                 self.classify_item(&mut stmt)
1291             } else {
1292                 // ignore derives on non-item statements so it falls through
1293                 // to the unused-attributes lint
1294                 let (attr, after_derive) = self.classify_nonitem(&mut stmt);
1295                 (attr, vec![], after_derive)
1296             };
1297
1298             if attr.is_some() || !derives.is_empty() {
1299                 return self
1300                     .collect_attr(
1301                         attr,
1302                         derives,
1303                         Annotatable::Stmt(P(stmt)),
1304                         AstFragmentKind::Stmts,
1305                         after_derive,
1306                     )
1307                     .make_stmts();
1308             }
1309         }
1310
1311         if let StmtKind::Mac(mac) = stmt.kind {
1312             let (mac, style, attrs) = mac.into_inner();
1313             self.check_attributes(&attrs);
1314             let mut placeholder =
1315                 self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts).make_stmts();
1316
1317             // If this is a macro invocation with a semicolon, then apply that
1318             // semicolon to the final statement produced by expansion.
1319             if style == MacStmtStyle::Semicolon {
1320                 if let Some(stmt) = placeholder.pop() {
1321                     placeholder.push(stmt.add_trailing_semicolon());
1322                 }
1323             }
1324
1325             return placeholder;
1326         }
1327
1328         // The placeholder expander gives ids to statements, so we avoid folding the id here.
1329         let ast::Stmt { id, kind, span } = stmt;
1330         noop_flat_map_stmt_kind(kind, self)
1331             .into_iter()
1332             .map(|kind| ast::Stmt { id, kind, span })
1333             .collect()
1334     }
1335
1336     fn visit_block(&mut self, block: &mut P<Block>) {
1337         let old_directory_ownership = self.cx.current_expansion.directory_ownership;
1338         self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock;
1339         noop_visit_block(block, self);
1340         self.cx.current_expansion.directory_ownership = old_directory_ownership;
1341     }
1342
1343     fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
1344         let mut item = configure!(self, item);
1345
1346         let (attr, traits, after_derive) = self.classify_item(&mut item);
1347         if attr.is_some() || !traits.is_empty() {
1348             return self
1349                 .collect_attr(
1350                     attr,
1351                     traits,
1352                     Annotatable::Item(item),
1353                     AstFragmentKind::Items,
1354                     after_derive,
1355                 )
1356                 .make_items();
1357         }
1358
1359         match item.kind {
1360             ast::ItemKind::Mac(..) => {
1361                 self.check_attributes(&item.attrs);
1362                 item.and_then(|item| match item.kind {
1363                     ItemKind::Mac(mac) => self
1364                         .collect(
1365                             AstFragmentKind::Items,
1366                             InvocationKind::Bang { mac, span: item.span },
1367                         )
1368                         .make_items(),
1369                     _ => unreachable!(),
1370                 })
1371             }
1372             ast::ItemKind::Mod(ast::Mod { inner, .. }) => {
1373                 if item.ident == Ident::invalid() {
1374                     return noop_flat_map_item(item, self);
1375                 }
1376
1377                 let orig_directory_ownership = self.cx.current_expansion.directory_ownership;
1378                 let mut module = (*self.cx.current_expansion.module).clone();
1379                 module.mod_path.push(item.ident);
1380
1381                 // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`).
1382                 // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`).
1383                 // Thus, if `inner` is the dummy span, we know the module is inline.
1384                 let inline_module = item.span.contains(inner) || inner.is_dummy();
1385
1386                 if inline_module {
1387                     if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) {
1388                         self.cx.current_expansion.directory_ownership =
1389                             DirectoryOwnership::Owned { relative: None };
1390                         module.directory.push(&*path.as_str());
1391                     } else {
1392                         module.directory.push(&*item.ident.as_str());
1393                     }
1394                 } else {
1395                     let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner);
1396                     let mut path = match path {
1397                         FileName::Real(path) => path,
1398                         other => PathBuf::from(other.to_string()),
1399                     };
1400                     let directory_ownership = match path.file_name().unwrap().to_str() {
1401                         Some("mod.rs") => DirectoryOwnership::Owned { relative: None },
1402                         Some(_) => DirectoryOwnership::Owned { relative: Some(item.ident) },
1403                         None => DirectoryOwnership::UnownedViaMod,
1404                     };
1405                     path.pop();
1406                     module.directory = path;
1407                     self.cx.current_expansion.directory_ownership = directory_ownership;
1408                 }
1409
1410                 let orig_module =
1411                     mem::replace(&mut self.cx.current_expansion.module, Rc::new(module));
1412                 let result = noop_flat_map_item(item, self);
1413                 self.cx.current_expansion.module = orig_module;
1414                 self.cx.current_expansion.directory_ownership = orig_directory_ownership;
1415                 result
1416             }
1417
1418             _ => noop_flat_map_item(item, self),
1419         }
1420     }
1421
1422     fn flat_map_trait_item(&mut self, item: P<ast::AssocItem>) -> SmallVec<[P<ast::AssocItem>; 1]> {
1423         let mut item = configure!(self, item);
1424
1425         let (attr, traits, after_derive) = self.classify_item(&mut item);
1426         if attr.is_some() || !traits.is_empty() {
1427             return self
1428                 .collect_attr(
1429                     attr,
1430                     traits,
1431                     Annotatable::TraitItem(item),
1432                     AstFragmentKind::TraitItems,
1433                     after_derive,
1434                 )
1435                 .make_trait_items();
1436         }
1437
1438         match item.kind {
1439             ast::AssocItemKind::Macro(..) => {
1440                 self.check_attributes(&item.attrs);
1441                 item.and_then(|item| match item.kind {
1442                     ast::AssocItemKind::Macro(mac) => self
1443                         .collect_bang(mac, item.span, AstFragmentKind::TraitItems)
1444                         .make_trait_items(),
1445                     _ => unreachable!(),
1446                 })
1447             }
1448             _ => noop_flat_map_assoc_item(item, self),
1449         }
1450     }
1451
1452     fn flat_map_impl_item(&mut self, item: P<ast::AssocItem>) -> SmallVec<[P<ast::AssocItem>; 1]> {
1453         let mut item = configure!(self, item);
1454
1455         let (attr, traits, after_derive) = self.classify_item(&mut item);
1456         if attr.is_some() || !traits.is_empty() {
1457             return self
1458                 .collect_attr(
1459                     attr,
1460                     traits,
1461                     Annotatable::ImplItem(item),
1462                     AstFragmentKind::ImplItems,
1463                     after_derive,
1464                 )
1465                 .make_impl_items();
1466         }
1467
1468         match item.kind {
1469             ast::AssocItemKind::Macro(..) => {
1470                 self.check_attributes(&item.attrs);
1471                 item.and_then(|item| match item.kind {
1472                     ast::AssocItemKind::Macro(mac) => self
1473                         .collect_bang(mac, item.span, AstFragmentKind::ImplItems)
1474                         .make_impl_items(),
1475                     _ => unreachable!(),
1476                 })
1477             }
1478             _ => noop_flat_map_assoc_item(item, self),
1479         }
1480     }
1481
1482     fn visit_ty(&mut self, ty: &mut P<ast::Ty>) {
1483         match ty.kind {
1484             ast::TyKind::Mac(_) => {}
1485             _ => return noop_visit_ty(ty, self),
1486         };
1487
1488         visit_clobber(ty, |mut ty| match mem::replace(&mut ty.kind, ast::TyKind::Err) {
1489             ast::TyKind::Mac(mac) => self.collect_bang(mac, ty.span, AstFragmentKind::Ty).make_ty(),
1490             _ => unreachable!(),
1491         });
1492     }
1493
1494     fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) {
1495         self.cfg.configure_foreign_mod(foreign_mod);
1496         noop_visit_foreign_mod(foreign_mod, self);
1497     }
1498
1499     fn flat_map_foreign_item(
1500         &mut self,
1501         mut foreign_item: P<ast::ForeignItem>,
1502     ) -> SmallVec<[P<ast::ForeignItem>; 1]> {
1503         let (attr, traits, after_derive) = self.classify_item(&mut foreign_item);
1504
1505         if attr.is_some() || !traits.is_empty() {
1506             return self
1507                 .collect_attr(
1508                     attr,
1509                     traits,
1510                     Annotatable::ForeignItem(foreign_item),
1511                     AstFragmentKind::ForeignItems,
1512                     after_derive,
1513                 )
1514                 .make_foreign_items();
1515         }
1516
1517         match foreign_item.kind {
1518             ast::ForeignItemKind::Macro(..) => {
1519                 self.check_attributes(&foreign_item.attrs);
1520                 foreign_item.and_then(|item| match item.kind {
1521                     ast::ForeignItemKind::Macro(mac) => self
1522                         .collect_bang(mac, item.span, AstFragmentKind::ForeignItems)
1523                         .make_foreign_items(),
1524                     _ => unreachable!(),
1525                 })
1526             }
1527             _ => noop_flat_map_foreign_item(foreign_item, self),
1528         }
1529     }
1530
1531     fn visit_item_kind(&mut self, item: &mut ast::ItemKind) {
1532         match item {
1533             ast::ItemKind::MacroDef(..) => {}
1534             _ => {
1535                 self.cfg.configure_item_kind(item);
1536                 noop_visit_item_kind(item, self);
1537             }
1538         }
1539     }
1540
1541     fn flat_map_generic_param(
1542         &mut self,
1543         param: ast::GenericParam,
1544     ) -> SmallVec<[ast::GenericParam; 1]> {
1545         let mut param = configure!(self, param);
1546
1547         let (attr, traits, after_derive) = self.classify_item(&mut param);
1548         if attr.is_some() || !traits.is_empty() {
1549             return self
1550                 .collect_attr(
1551                     attr,
1552                     traits,
1553                     Annotatable::GenericParam(param),
1554                     AstFragmentKind::GenericParams,
1555                     after_derive,
1556                 )
1557                 .make_generic_params();
1558         }
1559
1560         noop_flat_map_generic_param(param, self)
1561     }
1562
1563     fn visit_attribute(&mut self, at: &mut ast::Attribute) {
1564         // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename",
1565         // contents="file contents")]` attributes
1566         if !at.check_name(sym::doc) {
1567             return noop_visit_attribute(at, self);
1568         }
1569
1570         if let Some(list) = at.meta_item_list() {
1571             if !list.iter().any(|it| it.check_name(sym::include)) {
1572                 return noop_visit_attribute(at, self);
1573             }
1574
1575             let mut items = vec![];
1576
1577             for mut it in list {
1578                 if !it.check_name(sym::include) {
1579                     items.push({
1580                         noop_visit_meta_list_item(&mut it, self);
1581                         it
1582                     });
1583                     continue;
1584                 }
1585
1586                 if let Some(file) = it.value_str() {
1587                     let err_count = self.cx.parse_sess.span_diagnostic.err_count();
1588                     self.check_attributes(slice::from_ref(at));
1589                     if self.cx.parse_sess.span_diagnostic.err_count() > err_count {
1590                         // avoid loading the file if they haven't enabled the feature
1591                         return noop_visit_attribute(at, self);
1592                     }
1593
1594                     let filename = match self.cx.resolve_path(&*file.as_str(), it.span()) {
1595                         Ok(filename) => filename,
1596                         Err(mut err) => {
1597                             err.emit();
1598                             continue;
1599                         }
1600                     };
1601
1602                     match self.cx.source_map().load_file(&filename) {
1603                         Ok(source_file) => {
1604                             let src = source_file
1605                                 .src
1606                                 .as_ref()
1607                                 .expect("freshly loaded file should have a source");
1608                             let src_interned = Symbol::intern(src.as_str());
1609
1610                             let include_info = vec![
1611                                 ast::NestedMetaItem::MetaItem(attr::mk_name_value_item_str(
1612                                     Ident::with_dummy_span(sym::file),
1613                                     file,
1614                                     DUMMY_SP,
1615                                 )),
1616                                 ast::NestedMetaItem::MetaItem(attr::mk_name_value_item_str(
1617                                     Ident::with_dummy_span(sym::contents),
1618                                     src_interned,
1619                                     DUMMY_SP,
1620                                 )),
1621                             ];
1622
1623                             let include_ident = Ident::with_dummy_span(sym::include);
1624                             let item = attr::mk_list_item(include_ident, include_info);
1625                             items.push(ast::NestedMetaItem::MetaItem(item));
1626                         }
1627                         Err(e) => {
1628                             let lit =
1629                                 it.meta_item().and_then(|item| item.name_value_literal()).unwrap();
1630
1631                             if e.kind() == ErrorKind::InvalidData {
1632                                 self.cx
1633                                     .struct_span_err(
1634                                         lit.span,
1635                                         &format!("{} wasn't a utf-8 file", filename.display()),
1636                                     )
1637                                     .span_label(lit.span, "contains invalid utf-8")
1638                                     .emit();
1639                             } else {
1640                                 let mut err = self.cx.struct_span_err(
1641                                     lit.span,
1642                                     &format!("couldn't read {}: {}", filename.display(), e),
1643                                 );
1644                                 err.span_label(lit.span, "couldn't read file");
1645
1646                                 err.emit();
1647                             }
1648                         }
1649                     }
1650                 } else {
1651                     let mut err = self.cx.struct_span_err(
1652                         it.span(),
1653                         &format!("expected path to external documentation"),
1654                     );
1655
1656                     // Check if the user erroneously used `doc(include(...))` syntax.
1657                     let literal = it.meta_item_list().and_then(|list| {
1658                         if list.len() == 1 {
1659                             list[0].literal().map(|literal| &literal.kind)
1660                         } else {
1661                             None
1662                         }
1663                     });
1664
1665                     let (path, applicability) = match &literal {
1666                         Some(LitKind::Str(path, ..)) => {
1667                             (path.to_string(), Applicability::MachineApplicable)
1668                         }
1669                         _ => (String::from("<path>"), Applicability::HasPlaceholders),
1670                     };
1671
1672                     err.span_suggestion(
1673                         it.span(),
1674                         "provide a file path with `=`",
1675                         format!("include = \"{}\"", path),
1676                         applicability,
1677                     );
1678
1679                     err.emit();
1680                 }
1681             }
1682
1683             let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items);
1684             *at = ast::Attribute {
1685                 kind: ast::AttrKind::Normal(AttrItem {
1686                     path: meta.path,
1687                     args: meta.kind.mac_args(meta.span),
1688                 }),
1689                 span: at.span,
1690                 id: at.id,
1691                 style: at.style,
1692             };
1693         } else {
1694             noop_visit_attribute(at, self)
1695         }
1696     }
1697
1698     fn visit_id(&mut self, id: &mut ast::NodeId) {
1699         if self.monotonic {
1700             debug_assert_eq!(*id, ast::DUMMY_NODE_ID);
1701             *id = self.cx.resolver.next_node_id()
1702         }
1703     }
1704
1705     fn visit_fn_decl(&mut self, mut fn_decl: &mut P<ast::FnDecl>) {
1706         self.cfg.configure_fn_decl(&mut fn_decl);
1707         noop_visit_fn_decl(fn_decl, self);
1708     }
1709 }
1710
1711 pub struct ExpansionConfig<'feat> {
1712     pub crate_name: String,
1713     pub features: Option<&'feat Features>,
1714     pub recursion_limit: usize,
1715     pub trace_mac: bool,
1716     pub should_test: bool, // If false, strip `#[test]` nodes
1717     pub single_step: bool,
1718     pub keep_macs: bool,
1719 }
1720
1721 impl<'feat> ExpansionConfig<'feat> {
1722     pub fn default(crate_name: String) -> ExpansionConfig<'static> {
1723         ExpansionConfig {
1724             crate_name,
1725             features: None,
1726             recursion_limit: 1024,
1727             trace_mac: false,
1728             should_test: false,
1729             single_step: false,
1730             keep_macs: false,
1731         }
1732     }
1733
1734     fn proc_macro_hygiene(&self) -> bool {
1735         self.features.map_or(false, |features| features.proc_macro_hygiene)
1736     }
1737     fn custom_inner_attributes(&self) -> bool {
1738         self.features.map_or(false, |features| features.custom_inner_attributes)
1739     }
1740 }