auto merge of #8745 : brson/rust/metadata, r=cmr

This does two things: 1) stops compressing metadata, 2) stops copying the metadata section, instead holding a reference to the buffer returned by the LLVM section iterator.

Not compressing metadata requires something like 7x the storage space, but makes running tests about 9% faster. This has been a time improvement on all platforms I've tested, including windows. I considered leaving compression as an option but it doesn't seem to be worth the complexity since we don't currently have any use cases where we need to save that space.

In order to avoid copying the metadata section I had to hack up extra::ebml a bit to support unsafe buffers. We should probably move it into librustc so that it can evolve to support the compiler without worrying about having a crummy interface.

r? @graydon

diff --combined src/librustc/driver/driver.rs
index e0c5a729b3f88116804e629eb3c7df6b9d890cdc,dd57be823ab692bb0481fcba73ca08d439b6f51e..1771a11adef139ed23cdeb366b9fff87b76a7e88
--- 1/src/librustc/driver/driver.rs
--- 2/src/librustc/driver/driver.rs
+++ b/src/librustc/driver/driver.rs
@@@ -333,31 -333,21 +333,31 @@@ pub fn phase_5_run_llvm_passes(sess: Se
                                 trans: &CrateTranslation,
                                 outputs: &OutputFilenames) {
  
 -    // NB: Android hack
 -    if sess.targ_cfg.os == session::os_android &&
 +    // On Windows, LLVM integrated assembler emits bad stack unwind tables when
 +    // segmented stacks are enabled.  However, unwind info directives in assembly
 +    // output are OK, so we generate assembly first and then run it through
 +    // an external assembler.
 +    // Same for Android.
 +    if (sess.targ_cfg.os == session::os_android ||
 +        sess.targ_cfg.os == session::os_win32) &&
          (sess.opts.output_type == link::output_type_object ||
           sess.opts.output_type == link::output_type_exe) {
          let output_type = link::output_type_assembly;
 -        let obj_filename = outputs.obj_filename.with_filetype("s");
 +        let asm_filename = outputs.obj_filename.with_filetype("s");
  
          time(sess.time_passes(), ~"LLVM passes", ||
              link::write::run_passes(sess,
                                      trans.context,
                                      trans.module,
                                      output_type,
 -                                    &obj_filename));
 +                                    &asm_filename));
  
 -        link::write::run_ndk(sess, &obj_filename, &outputs.obj_filename);
 +        link::write::run_assembler(sess, &asm_filename, &outputs.obj_filename);
 +
 +        // Remove assembly source unless --save-temps was specified
 +        if !sess.opts.save_temps {
 +            os::remove_file(&asm_filename);
 +        }
      } else {
          time(sess.time_passes(), ~"LLVM passes", ||
              link::write::run_passes(sess,
@@@ -570,12 -560,11 +570,12 @@@ pub fn build_target_config(sopts: @sess
        abi::Arm => (ast::ty_i32, ast::ty_u32, ast::ty_f64),
        abi::Mips => (ast::ty_i32, ast::ty_u32, ast::ty_f64)
      };
 +    let target_triple = sopts.target_triple.clone();
      let target_strs = match arch {
 -      abi::X86 => x86::get_target_strs(os),
 -      abi::X86_64 => x86_64::get_target_strs(os),
 -      abi::Arm => arm::get_target_strs(os),
 -      abi::Mips => mips::get_target_strs(os)
 +      abi::X86 => x86::get_target_strs(target_triple, os),
 +      abi::X86_64 => x86_64::get_target_strs(target_triple, os),
 +      abi::Arm => arm::get_target_strs(target_triple, os),
 +      abi::Mips => mips::get_target_strs(target_triple, os)
      };
      let target_cfg = @session::config {
          os: os,
@@@ -763,7 -752,7 +763,7 @@@ pub fn build_session_options(binary: @s
          parse_only: parse_only,
          no_trans: no_trans,
          debugging_opts: debugging_opts,
-         android_cross_path: android_cross_path
+         android_cross_path: android_cross_path,
      };
      return sopts;
  }

diff --combined src/librustc/metadata/decoder.rs
index 9eb09806bc07dc0ea6e12bae27951225772564a3,39ba64c5e26cb8f97d2037237a57f5e290c37449..f3476a9963d0cca7aec912812dfc24cd2c56af10
--- 1/src/librustc/metadata/decoder.rs
--- 2/src/librustc/metadata/decoder.rs
+++ b/src/librustc/metadata/decoder.rs
@@@ -20,6 -20,7 +20,7 @@@ use metadata::decoder
  use metadata::tydecode::{parse_ty_data, parse_def_id,
                           parse_type_param_def_data,
                           parse_bare_fn_ty_data, parse_trait_ref_data};
+ use metadata::loader::{MetadataSection, UnsafeSection};
  use middle::ty;
  use middle::typeck;
  use middle::astencode::vtable_decoder_helpers;
@@@ -56,16 -57,16 +57,16 @@@ fn lookup_hash(d: ebml::Doc, eq_fn: &fn
      let index = reader::get_doc(d, tag_index);
      let table = reader::get_doc(index, tag_index_table);
      let hash_pos = table.start + (hash % 256 * 4) as uint;
-     let pos = io::u64_from_be_bytes(*d.data, hash_pos, 4) as uint;
-     let tagged_doc = reader::doc_at(d.data, pos);
+     let pos = io::u64_from_be_bytes(d.data.as_slice(), hash_pos, 4) as uint;
+     let tagged_doc = reader::doc_at(&d.data, pos);
  
      let belt = tag_index_buckets_bucket_elt;
  
      let mut ret = None;
      do reader::tagged_docs(tagged_doc.doc, belt) |elt| {
-         let pos = io::u64_from_be_bytes(*elt.data, elt.start, 4) as uint;
+         let pos = io::u64_from_be_bytes(elt.data.as_slice(), elt.start, 4) as uint;
          if eq_fn(elt.data.slice(elt.start + 4, elt.end)) {
-             ret = Some(reader::doc_at(d.data, pos).doc);
+             ret = Some(reader::doc_at(&d.data, pos).doc);
              false
          } else {
              true
@@@ -96,8 -97,8 +97,8 @@@ fn find_item(item_id: int, items: ebml:
  
  // Looks up an item in the given metadata and returns an ebml doc pointing
  // to the item data.
- fn lookup_item(item_id: int, data: @~[u8]) -> ebml::Doc {
-     let items = reader::get_doc(reader::Doc(data), tag_items);
+ fn lookup_item(item_id: int, data: MetadataSection) -> ebml::Doc {
+     let items = reader::get_doc(section_to_ebml_doc(data), tag_items);
      find_item(item_id, items)
  }
  
@@@ -215,13 -216,13 +216,13 @@@ fn variant_disr_val(d: ebml::Doc) -> Op
  
  fn doc_type(doc: ebml::Doc, tcx: ty::ctxt, cdata: cmd) -> ty::t {
      let tp = reader::get_doc(doc, tag_items_data_item_type);
-     parse_ty_data(*tp.data, cdata.cnum, tp.start, tcx,
+     parse_ty_data(tp.data.as_slice(), cdata.cnum, tp.start, tcx,
                    |_, did| translate_def_id(cdata, did))
  }
  
  fn doc_method_fty(doc: ebml::Doc, tcx: ty::ctxt, cdata: cmd) -> ty::BareFnTy {
      let tp = reader::get_doc(doc, tag_item_method_fty);
-     parse_bare_fn_ty_data(*tp.data, cdata.cnum, tp.start, tcx,
+     parse_bare_fn_ty_data(tp.data.as_slice(), cdata.cnum, tp.start, tcx,
                            |_, did| translate_def_id(cdata, did))
  }
  
@@@ -230,7 -231,7 +231,7 @@@ fn doc_transformed_self_ty(doc: ebml::D
                             cdata: cmd) -> Option<ty::t>
  {
      do reader::maybe_get_doc(doc, tag_item_method_transformed_self_ty).map |tp| {
-         parse_ty_data(*tp.data, cdata.cnum, tp.start, tcx,
+         parse_ty_data(tp.data.as_slice(), cdata.cnum, tp.start, tcx,
                        |_, did| translate_def_id(cdata, did))
      }
  }
@@@ -241,7 -242,7 +242,7 @@@ pub fn item_type(_item_id: ast::def_id
  }
  
  fn doc_trait_ref(doc: ebml::Doc, tcx: ty::ctxt, cdata: cmd) -> ty::TraitRef {
-     parse_trait_ref_data(*doc.data, cdata.cnum, doc.start, tcx,
+     parse_trait_ref_data(doc.data.as_slice(), cdata.cnum, doc.start, tcx,
                           |_, did| translate_def_id(cdata, did))
  }
  
@@@ -256,7 -257,7 +257,7 @@@ fn item_ty_param_defs(item: ebml::Doc, 
      let mut bounds = ~[];
      do reader::tagged_docs(item, tag) |p| {
          let bd = parse_type_param_def_data(
-             *p.data, p.start, cdata.cnum, tcx,
+             p.data.as_slice(), p.start, cdata.cnum, tcx,
              |_, did| translate_def_id(cdata, did));
          bounds.push(bd);
          true
@@@ -359,7 -360,7 +360,7 @@@ fn item_to_def_like(item: ebml::Doc, di
      }
  }
  
- pub fn lookup_def(cnum: ast::CrateNum, data: @~[u8], did_: ast::def_id) ->
+ pub fn lookup_def(cnum: ast::CrateNum, data: MetadataSection, did_: ast::def_id) ->
     ast::def {
      let item = lookup_item(did_.node, data);
      let did = ast::def_id { crate: cnum, node: did_.node };
@@@ -375,21 -376,9 +376,21 @@@ pub fn get_trait_def(cdata: cmd
      let tp_defs = item_ty_param_defs(item_doc, tcx, cdata,
                                       tag_items_data_item_ty_param_bounds);
      let rp = item_ty_region_param(item_doc);
 +    let mut bounds = ty::EmptyBuiltinBounds();
 +    // Collect the builtin bounds from the encoded supertraits.
 +    // FIXME(#8559): They should be encoded directly.
 +    do reader::tagged_docs(item_doc, tag_item_super_trait_ref) |trait_doc| {
 +        // NB. Bypasses real supertraits. See get_supertraits() if you wanted them.
 +        let trait_ref = doc_trait_ref(trait_doc, tcx, cdata);
 +        do tcx.lang_items.to_builtin_kind(trait_ref.def_id).map_move |bound| {
 +            bounds.add(bound);
 +        };
 +        true
 +    };
      ty::TraitDef {
          generics: ty::Generics {type_param_defs: tp_defs,
                                  region_param: rp},
 +        bounds: bounds,
          trait_ref: @item_trait_ref(item_doc, tcx, cdata)
      }
  }
@@@ -418,7 -407,7 +419,7 @@@ pub fn get_region_param(cdata: cmd, id
      return item_ty_region_param(item);
  }
  
- pub fn get_type_param_count(data: @~[u8], id: ast::NodeId) -> uint {
+ pub fn get_type_param_count(data: MetadataSection, id: ast::NodeId) -> uint {
      item_ty_param_count(lookup_item(id, data))
  }
  
@@@ -449,7 -438,7 +450,7 @@@ pub fn get_impl_vtables(cdata: cmd
  
  pub fn get_impl_method(intr: @ident_interner, cdata: cmd, id: ast::NodeId,
                         name: ast::ident) -> Option<ast::def_id> {
-     let items = reader::get_doc(reader::Doc(cdata.data), tag_items);
+     let items = reader::get_doc(section_to_ebml_doc(cdata.data), tag_items);
      let mut found = None;
      do reader::tagged_docs(find_item(id, items), tag_item_impl_method) |mid| {
          let m_did = reader::with_doc_data(mid, parse_def_id);
@@@ -461,7 -450,7 +462,7 @@@
      found
  }
  
- pub fn get_symbol(data: @~[u8], id: ast::NodeId) -> ~str {
+ pub fn get_symbol(data: MetadataSection, id: ast::NodeId) -> ~str {
      return item_symbol(lookup_item(id, data));
  }
  
@@@ -482,7 -471,7 +483,7 @@@ fn def_like_to_def(def_like: def_like) 
  
  /// Iterates over the language items in the given crate.
  pub fn each_lang_item(cdata: cmd, f: &fn(ast::NodeId, uint) -> bool) -> bool {
-     let root = reader::Doc(cdata.data);
+     let root = section_to_ebml_doc(cdata.data);
      let lang_items = reader::get_doc(root, tag_lang_items);
      do reader::tagged_docs(lang_items, tag_lang_items_item) |item_doc| {
          let id_doc = reader::get_doc(item_doc, tag_lang_items_item_id);
@@@ -577,10 -566,10 +578,10 @@@ impl<'self> EachItemContext<'self> 
      fn each_item_of_module(&mut self, def_id: ast::def_id) -> bool {
          // This item might not be in this crate. If it's not, look it up.
          let items = if def_id.crate == self.cdata.cnum {
-             reader::get_doc(reader::Doc(self.cdata.data), tag_items)
+             reader::get_doc(section_to_ebml_doc(self.cdata.data), tag_items)
          } else {
              let crate_data = (self.get_crate_data)(def_id.crate);
-             let root = reader::Doc(crate_data.data);
+             let root = section_to_ebml_doc(crate_data.data);
              reader::get_doc(root, tag_items)
          };
  
@@@ -606,10 -595,10 +607,10 @@@
              // a reexport.
              let other_crates_items = if child_def_id.crate ==
                      self.cdata.cnum {
-                 reader::get_doc(reader::Doc(self.cdata.data), tag_items)
+                 reader::get_doc(section_to_ebml_doc(self.cdata.data), tag_items)
              } else {
                  let crate_data = (self.get_crate_data)(child_def_id.crate);
-                 let root = reader::Doc(crate_data.data);
+                 let root = section_to_ebml_doc(crate_data.data);
                  reader::get_doc(root, tag_items)
              };
  
@@@ -673,10 -662,10 +674,10 @@@
  
              // This reexport may be in yet another crate.
              let other_crates_items = if def_id.crate == self.cdata.cnum {
-                 reader::get_doc(reader::Doc(self.cdata.data), tag_items)
+                 reader::get_doc(section_to_ebml_doc(self.cdata.data), tag_items)
              } else {
                  let crate_data = (self.get_crate_data)(def_id.crate);
-                 let root = reader::Doc(crate_data.data);
+                 let root = section_to_ebml_doc(crate_data.data);
                  reader::get_doc(root, tag_items)
              };
  
@@@ -708,7 -697,7 +709,7 @@@ pub fn each_path(intr: @ident_interner
      // make fast. It's the source of most of the performance problems when
      // compiling small crates.
  
-     let root_doc = reader::Doc(cdata.data);
+     let root_doc = section_to_ebml_doc(cdata.data);
      let misc_info_doc = reader::get_doc(root_doc, tag_misc_info);
      let crate_items_doc = reader::get_doc(misc_info_doc,
                                            tag_misc_info_crate_items);
@@@ -768,7 -757,7 +769,7 @@@ pub fn maybe_get_item_ast(cdata: cmd, t
  pub fn get_enum_variants(intr: @ident_interner, cdata: cmd, id: ast::NodeId,
                       tcx: ty::ctxt) -> ~[@ty::VariantInfo] {
      let data = cdata.data;
-     let items = reader::get_doc(reader::Doc(data), tag_items);
+     let items = reader::get_doc(section_to_ebml_doc(data), tag_items);
      let item = find_item(id, items);
      let mut infos: ~[@ty::VariantInfo] = ~[];
      let variant_ids = enum_variant_ids(item, cdata);
@@@ -941,13 -930,7 +942,13 @@@ pub fn get_supertraits(cdata: cmd, id: 
      let mut results = ~[];
      let item_doc = lookup_item(id, cdata.data);
      do reader::tagged_docs(item_doc, tag_item_super_trait_ref) |trait_doc| {
 -        results.push(@doc_trait_ref(trait_doc, tcx, cdata));
 +        // NB. Only reads the ones that *aren't* builtin-bounds. See also
 +        // get_trait_def() for collecting the builtin bounds.
 +        // FIXME(#8559): The builtin bounds shouldn't be encoded in the first place.
 +        let trait_ref = doc_trait_ref(trait_doc, tcx, cdata);
 +        if tcx.lang_items.to_builtin_kind(trait_ref.def_id).is_none() {
 +            results.push(@trait_ref);
 +        }
          true
      };
      return results;
@@@ -1206,8 -1189,14 +1207,14 @@@ fn list_crate_attributes(intr: @ident_i
      out.write_str("\n\n");
  }
  
- pub fn get_crate_attributes(data: @~[u8]) -> ~[ast::Attribute] {
-     return get_attributes(reader::Doc(data));
+ pub fn get_crate_attributes(data: MetadataSection) -> ~[ast::Attribute] {
+     return get_attributes(section_to_ebml_doc(data));
+ }
+ 
+ pub fn section_to_ebml_doc(data: MetadataSection) -> ebml::Doc {
+     match data {
+         UnsafeSection(_, buf, len) => reader::unsafe_Doc(buf, len)
+     }
  }
  
  #[deriving(Clone)]
@@@ -1218,9 -1207,9 +1225,9 @@@ pub struct crate_dep 
      hash: @str
  }
  
- pub fn get_crate_deps(data: @~[u8]) -> ~[crate_dep] {
+ pub fn get_crate_deps(data: MetadataSection) -> ~[crate_dep] {
      let mut deps: ~[crate_dep] = ~[];
-     let cratedoc = reader::Doc(data);
+     let cratedoc = section_to_ebml_doc(data);
      let depsdoc = reader::get_doc(cratedoc, tag_crate_deps);
      let mut crate_num = 1;
      fn docstr(doc: ebml::Doc, tag_: uint) -> @str {
@@@ -1238,7 -1227,7 +1245,7 @@@
      return deps;
  }
  
- fn list_crate_deps(data: @~[u8], out: @io::Writer) {
+ fn list_crate_deps(data: MetadataSection, out: @io::Writer) {
      out.write_str("=External Dependencies=\n");
  
      let r = get_crate_deps(data);
@@@ -1251,13 -1240,13 +1258,13 @@@
      out.write_str("\n");
  }
  
- pub fn get_crate_hash(data: @~[u8]) -> @str {
-     let cratedoc = reader::Doc(data);
+ pub fn get_crate_hash(data: MetadataSection) -> @str {
+     let cratedoc = section_to_ebml_doc(data);
      let hashdoc = reader::get_doc(cratedoc, tag_crate_hash);
      hashdoc.as_str_slice().to_managed()
  }
  
- pub fn get_crate_vers(data: @~[u8]) -> @str {
+ pub fn get_crate_vers(data: MetadataSection) -> @str {
      let attrs = decoder::get_crate_attributes(data);
      let linkage_attrs = attr::find_linkage_metas(attrs);
  
@@@ -1282,10 -1271,10 +1289,10 @@@ fn iter_crate_items(intr: @ident_intern
      };
  }
  
- pub fn list_crate_metadata(intr: @ident_interner, bytes: @~[u8],
+ pub fn list_crate_metadata(intr: @ident_interner, bytes: MetadataSection,
                             out: @io::Writer) {
      let hash = get_crate_hash(bytes);
-     let md = reader::Doc(bytes);
+     let md = section_to_ebml_doc(bytes);
      list_crate_attributes(intr, md, hash, out);
      list_crate_deps(bytes, out);
  }
@@@ -1307,7 -1296,7 +1314,7 @@@ pub fn translate_def_id(cdata: cmd, did
  }
  
  pub fn get_link_args_for_crate(cdata: cmd) -> ~[~str] {
-     let link_args = reader::get_doc(reader::Doc(cdata.data), tag_link_args);
+     let link_args = reader::get_doc(section_to_ebml_doc(cdata.data), tag_link_args);
      let mut result = ~[];
      do reader::tagged_docs(link_args, tag_link_args_arg) |arg_doc| {
          result.push(arg_doc.as_str());

diff --combined src/librustc/metadata/encoder.rs
index dbed508348d03297de14364ec2e9316a6f854fda,38d2d3567bb7fc0c79c39199064fa175c4fb700d..bd7442472fe40db3d0cf68f77d5a21103902594b
--- 1/src/librustc/metadata/encoder.rs
--- 2/src/librustc/metadata/encoder.rs
+++ b/src/librustc/metadata/encoder.rs
@@@ -26,7 -26,6 +26,6 @@@ use std::hashmap::{HashMap, HashSet}
  use std::io;
  use std::str;
  use std::vec;
- use extra::flate;
  use extra::serialize::Encodable;
  use extra;
  use syntax::abi::AbiSet;
@@@ -1066,9 -1065,6 +1065,9 @@@ fn encode_info_for_item(ecx: &EncodeCon
              ebml_w.end_tag();
          }
          encode_path(ecx, ebml_w, path, ast_map::path_name(item.ident));
 +        // FIXME(#8559): This should use the tcx's supertrait cache instead of
 +        // reading the AST's list, because the former has already filtered out
 +        // the builtin-kinds-as-supertraits. See corresponding fixme in decoder.
          for ast_trait_ref in super_traits.iter() {
              let trait_ref = ty::node_id_to_trait_ref(ecx.tcx, ast_trait_ref.ref_id);
              encode_trait_ref(ebml_w, ecx, trait_ref, tag_item_super_trait_ref);
@@@ -1570,7 -1566,7 +1569,7 @@@ pub static metadata_encoding_version : 
        0x75, //'u' as u8,
        0x73, //'s' as u8,
        0x74, //'t' as u8,
-       0, 0, 0, 1 ];
+       0, 0, 0, 2 ];
  
  pub fn encode_metadata(parms: EncodeParams, crate: &Crate) -> ~[u8] {
      let wr = @io::BytesWriter::new();
@@@ -1683,8 -1679,7 +1682,7 @@@
  
      let writer_bytes: &mut ~[u8] = wr.bytes;
  
-     metadata_encoding_version.to_owned() +
-         flate::deflate_bytes(*writer_bytes)
+     return metadata_encoding_version.to_owned() + *writer_bytes;
  }
  
  // Get the encoded string for a type