Auto merge of #40220 - jseyfried:ast_macro_def, r=nrc

syntax: add `ast::ItemKind::MacroDef`, simplify hygiene info

This PR
 - adds a new variant `MacroDef` to `ast::ItemKind` for `macro_rules!` and eventually `macro` items,
 - [breaking-change] forbids macro defs without a name (`macro_rules! { () => {} }` compiles today),
 - removes `ast::MacroDef`, and
 - no longer uses `Mark` and `Invocation` to identify and characterize macro definitions.
   - We used to apply (at least) two `Mark`s to an expanded identifier's `SyntaxContext` -- the definition mark(s) and the expansion mark(s). We now only apply the latter.

r? @nrc

diff --combined src/librustc_driver/driver.rs
index 86722f667413a1d6645535308e9581fc1f12d417,e458d45bbd6e8cdd6a4ca1b19f356756b38afe80..2126a5a7c71bd5e5f201713fe1770df063b6585f
--- 1/src/librustc_driver/driver.rs
--- 2/src/librustc_driver/driver.rs
+++ b/src/librustc_driver/driver.rs
@@@ -43,7 -43,6 +43,6 @@@ use super::Compilation
  use serialize::json;
  
  use std::env;
- use std::mem;
  use std::ffi::{OsString, OsStr};
  use std::fs;
  use std::io::{self, Write};
@@@ -125,10 -124,6 +124,10 @@@ pub fn compile_input(sess: &Session
          };
  
          write_out_deps(sess, &outputs, &crate_name);
 +        if sess.opts.output_types.contains_key(&OutputType::DepInfo) &&
 +            sess.opts.output_types.keys().count() == 1 {
 +            return Ok(())
 +        }
  
          let arena = DroplessArena::new();
          let arenas = GlobalArenas::new();
@@@ -608,7 -603,7 +607,7 @@@ pub fn phase_2_configure_and_expand<F>(
  
      let whitelisted_legacy_custom_derives = registry.take_whitelisted_custom_derives();
      let Registry { syntax_exts, early_lint_passes, late_lint_passes, lint_groups,
 -                   llvm_passes, attributes, mir_passes, .. } = registry;
 +                   llvm_passes, attributes, .. } = registry;
  
      sess.track_errors(|| {
          let mut ls = sess.lint_store.borrow_mut();
@@@ -624,6 -619,7 +623,6 @@@
          }
  
          *sess.plugin_llvm_passes.borrow_mut() = llvm_passes;
 -        sess.mir_passes.borrow_mut().extend(mir_passes);
          *sess.plugin_attributes.borrow_mut() = attributes.clone();
      })?;
  
@@@ -708,8 -704,6 +707,6 @@@
          krate
      });
  
-     krate.exported_macros = mem::replace(&mut resolver.exported_macros, Vec::new());
- 
      krate = time(time_passes, "maybe building test harness", || {
          syntax::test::modify_for_testing(&sess.parse_sess,
                                           &mut resolver,
@@@ -1051,7 -1045,6 +1048,7 @@@ pub fn phase_4_translate_to_llvm<'a, 't
          passes.push_pass(box mir::transform::simplify::SimplifyCfg::new("elaborate-drops"));
  
          // No lifetime analysis based on borrowing can be done from here on out.
 +        passes.push_pass(box mir::transform::inline::Inline);
          passes.push_pass(box mir::transform::instcombine::InstCombine::new());
          passes.push_pass(box mir::transform::deaggregator::Deaggregator);
          passes.push_pass(box mir::transform::copy_prop::CopyPropagation);

diff --combined src/libsyntax/ext/expand.rs
index f7dcd00e40976d7ba8f16464ca4428dcdc95ce1c,96fcea7148bfa67a529c0063ae9d73f978149bba..10168f010a077c9b94f2ec1d8e130fc108426252
--- 1/src/libsyntax/ext/expand.rs
--- 2/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@@ -154,7 -154,7 +154,7 @@@ impl ExpansionKind 
  pub struct Invocation {
      pub kind: InvocationKind,
      expansion_kind: ExpansionKind,
-     expansion_data: ExpansionData,
+     pub expansion_data: ExpansionData,
  }
  
  pub enum InvocationKind {
@@@ -251,7 -251,7 +251,7 @@@ impl<'a, 'b> MacroExpander<'a, 'b> 
  
              let scope =
                  if self.monotonic { invoc.expansion_data.mark } else { orig_expansion_data.mark };
-             let ext = match self.resolve_invoc(&mut invoc, scope, force) {
+             let ext = match self.cx.resolver.resolve_invoc(&mut invoc, scope, force) {
                  Ok(ext) => Some(ext),
                  Err(Determinacy::Determined) => None,
                  Err(Determinacy::Undetermined) => {
@@@ -364,64 -364,6 +364,6 @@@
          result
      }
  
-     fn resolve_invoc(&mut self, invoc: &mut Invocation, scope: Mark, force: bool)
-                      -> Result<Option<Rc<SyntaxExtension>>, Determinacy> {
-         let (attr, traits, item) = match invoc.kind {
-             InvocationKind::Bang { ref mac, .. } => {
-                 return self.cx.resolver.resolve_macro(scope, &mac.node.path,
-                                                       MacroKind::Bang, force).map(Some);
-             }
-             InvocationKind::Attr { attr: None, .. } => return Ok(None),
-             InvocationKind::Derive { name, span, .. } => {
-                 let path = ast::Path::from_ident(span, Ident::with_empty_ctxt(name));
-                 return self.cx.resolver.resolve_macro(scope, &path,
-                                                       MacroKind::Derive, force).map(Some)
-             }
-             InvocationKind::Attr { ref mut attr, ref traits, ref mut item } => (attr, traits, item),
-         };
- 
-         let (attr_name, path) = {
-             let attr = attr.as_ref().unwrap();
-             (attr.name(), ast::Path::from_ident(attr.span, Ident::with_empty_ctxt(attr.name())))
-         };
- 
-         let mut determined = true;
-         match self.cx.resolver.resolve_macro(scope, &path, MacroKind::Attr, force) {
-             Ok(ext) => return Ok(Some(ext)),
-             Err(Determinacy::Undetermined) => determined = false,
-             Err(Determinacy::Determined) if force => return Err(Determinacy::Determined),
-             _ => {}
-         }
- 
-         for &(name, span) in traits {
-             let path = ast::Path::from_ident(span, Ident::with_empty_ctxt(name));
-             match self.cx.resolver.resolve_macro(scope, &path, MacroKind::Derive, force) {
-                 Ok(ext) => if let SyntaxExtension::ProcMacroDerive(_, ref inert_attrs) = *ext {
-                     if inert_attrs.contains(&attr_name) {
-                         // FIXME(jseyfried) Avoid `mem::replace` here.
-                         let dummy_item = placeholder(ExpansionKind::Items, ast::DUMMY_NODE_ID)
-                             .make_items().pop().unwrap();
-                         *item = mem::replace(item, Annotatable::Item(dummy_item))
-                             .map_attrs(|mut attrs| {
-                                 let inert_attr = attr.take().unwrap();
-                                 attr::mark_known(&inert_attr);
-                                 if self.cx.ecfg.proc_macro_enabled() {
-                                     *attr = find_attr_invoc(&mut attrs);
-                                 }
-                                 attrs.push(inert_attr);
-                                 attrs
-                             });
-                     }
-                     return Err(Determinacy::Undetermined);
-                 },
-                 Err(Determinacy::Undetermined) => determined = false,
-                 Err(Determinacy::Determined) => {}
-             }
-         }
- 
-         Err(if determined { Determinacy::Determined } else { Determinacy::Undetermined })
-     }
- 
      fn expand_invoc(&mut self, invoc: Invocation, ext: Rc<SyntaxExtension>) -> Expansion {
          match invoc.kind {
              InvocationKind::Bang { .. } => self.expand_bang_invoc(invoc, ext),
@@@ -464,20 -406,8 +406,20 @@@
                  let attr_toks = stream_for_attr_args(&attr, &self.cx.parse_sess);
                  let item_toks = stream_for_item(&item, &self.cx.parse_sess);
  
 +                let span = Span {
 +                    expn_id: self.cx.codemap().record_expansion(ExpnInfo {
 +                        call_site: attr.span,
 +                        callee: NameAndSpan {
 +                            format: MacroAttribute(name),
 +                            span: None,
 +                            allow_internal_unstable: false,
 +                        },
 +                    }),
 +                    ..attr.span
 +                };
 +
                  let tok_result = mac.expand(self.cx, attr.span, attr_toks, item_toks);
 -                self.parse_expansion(tok_result, kind, name, attr.span)
 +                self.parse_expansion(tok_result, kind, name, span)
              }
              SyntaxExtension::ProcMacroDerive(..) | SyntaxExtension::BuiltinDerive(..) => {
                  self.cx.span_err(attr.span, &format!("`{}` is a derive mode", name));
@@@ -502,7 -432,8 +444,8 @@@
  
          let extname = path.segments.last().unwrap().identifier.name;
          let ident = ident.unwrap_or(keywords::Invalid.ident());
-         let marked_tts = mark_tts(mac.node.stream(), mark);
+         let marked_tts =
+             noop_fold_tts(mac.node.stream(), &mut Marker { mark: mark, expn_id: None });
          let opt_expanded = match *ext {
              NormalTT(ref expandfun, exp_span, allow_internal_unstable) => {
                  if ident.name != keywords::Invalid.name() {
@@@ -814,7 -745,7 +757,7 @@@ impl<'a, 'b> InvocationCollector<'a, 'b
      }
  }
  
- fn find_attr_invoc(attrs: &mut Vec<ast::Attribute>) -> Option<ast::Attribute> {
+ pub fn find_attr_invoc(attrs: &mut Vec<ast::Attribute>) -> Option<ast::Attribute> {
      for i in 0 .. attrs.len() {
          if !attr::is_known(&attrs[i]) && !is_builtin_attr(&attrs[i]) {
               return Some(attrs.remove(i));
@@@ -960,17 -891,7 +903,7 @@@ impl<'a, 'b> Folder for InvocationColle
          match item.node {
              ast::ItemKind::Mac(..) => {
                  self.check_attributes(&item.attrs);
-                 let is_macro_def = if let ItemKind::Mac(ref mac) = item.node {
-                     mac.node.path.segments[0].identifier.name == "macro_rules"
-                 } else {
-                     unreachable!()
-                 };
- 
-                 item.and_then(|mut item| match item.node {
-                     ItemKind::Mac(_) if is_macro_def => {
-                         item.id = Mark::fresh().as_placeholder_id();
-                         SmallVector::one(P(item))
-                     }
+                 item.and_then(|item| match item.node {
                      ItemKind::Mac(mac) => {
                          self.collect(ExpansionKind::Items, InvocationKind::Bang {
                              mac: mac,
@@@ -1090,7 -1011,10 +1023,10 @@@
      }
  
      fn fold_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
-         noop_fold_item_kind(self.cfg.configure_item_kind(item), self)
+         match item {
+             ast::ItemKind::MacroDef(..) => item,
+             _ => noop_fold_item_kind(self.cfg.configure_item_kind(item), self),
+         }
      }
  
      fn new_id(&mut self, id: ast::NodeId) -> ast::NodeId {
@@@ -1171,8 -1095,3 +1107,3 @@@ impl Folder for Marker 
          span
      }
  }
- 
- // apply a given mark to the given token trees. Used prior to expansion of a macro.
- pub fn mark_tts(tts: TokenStream, m: Mark) -> TokenStream {
-     noop_fold_tts(tts, &mut Marker{mark:m, expn_id: None})
- }