From: bjorn3 Date: Sat, 19 Oct 2019 10:54:13 +0000 (+0200) Subject: Merge pull request #758 from bjorn3/object_write X-Git-Url: https://git.lizzy.rs/?a=commitdiff_plain;h=75c24b9c9677600422ec86fa9f4c78fe3678d2ce;hp=8243a370d21a6d7b69c8d7fc11d2f2fc3e939297;p=rust.git Merge pull request #758 from bjorn3/object_write Add object::write as alternative object file writer --- diff --git a/.travis.yml b/.travis.yml index e73e9933976..76573978556 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,3 +21,6 @@ env: - RUST_BACKTRACE=1 # Reduce amount of benchmark runs as they are slow. - COMPILE_RUNS=2 RUN_RUNS=2 + jobs: + - "CG_CLIF_COMPILE_FLAGS=" + - "CG_CLIF_COMPILE_FLAGS='--features backend_object'" diff --git a/Cargo.lock b/Cargo.lock index dab1e3f3860..d67181bd562 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -155,6 +155,17 @@ dependencies = [ "target-lexicon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cranelift-object" +version = "0.46.1" +source = "git+https://github.com/CraneStation/cranelift.git#387593d6c94d291e614c08d7a03f77b40efa451d" +dependencies = [ + "cranelift-codegen 0.46.1 (git+https://github.com/CraneStation/cranelift.git)", + "cranelift-module 0.46.1 (git+https://github.com/CraneStation/cranelift.git)", + "object 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "target-lexicon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "cranelift-simplejit" version = "0.46.1" @@ -332,8 +343,10 @@ name = "object" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", "goblin 0.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "scroll 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "target-lexicon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -411,6 +424,7 @@ dependencies = [ "cranelift 0.46.1 (git+https://github.com/CraneStation/cranelift.git)", "cranelift-faerie 0.46.1 (git+https://github.com/CraneStation/cranelift.git)", "cranelift-module 0.46.1 (git+https://github.com/CraneStation/cranelift.git)", + "cranelift-object 0.46.1 (git+https://github.com/CraneStation/cranelift.git)", "cranelift-simplejit 0.46.1 (git+https://github.com/CraneStation/cranelift.git)", "faerie 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "gimli 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -594,6 +608,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum cranelift-frontend 0.46.1 (git+https://github.com/CraneStation/cranelift.git)" = "" "checksum cranelift-module 0.46.1 (git+https://github.com/CraneStation/cranelift.git)" = "" "checksum cranelift-native 0.46.1 (git+https://github.com/CraneStation/cranelift.git)" = "" +"checksum cranelift-object 0.46.1 (git+https://github.com/CraneStation/cranelift.git)" = "" "checksum cranelift-simplejit 0.46.1 (git+https://github.com/CraneStation/cranelift.git)" = "" "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" "checksum errno 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2a071601ed01b988f896ab14b95e67335d1eeb50190932a1320f7fe3cadc84e" diff --git a/Cargo.toml b/Cargo.toml index 66256cd68e1..86347a006d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,11 +9,15 @@ edition = "2018" [lib] crate-type = ["dylib"] +[features] +backend_object = ["object/write", "cranelift-object"] + [dependencies] # These have to be in sync with each other cranelift = { git = "https://github.com/CraneStation/cranelift.git" } cranelift-module = { git = "https://github.com/CraneStation/cranelift.git" } cranelift-faerie = { git = "https://github.com/CraneStation/cranelift.git" } +cranelift-object = { git = "https://github.com/CraneStation/cranelift.git", optional = true } target-lexicon = "0.8.1" faerie = "0.11.0" @@ -36,6 +40,7 @@ features = ["compression", "read", "std"] # We don't need WASM support #cranelift-module = { path = "../cranelift/cranelift-module" } #cranelift-simplejit = { path = "../cranelift/cranelift-simplejit" } #cranelift-faerie = { path = "../cranelift/cranelift-faerie" } +#cranelift-object = { path = "../cranelift/cranelift-object" } #[patch.crates-io] #gimli = { path = "../" } diff --git a/src/backend.rs b/src/backend.rs new file mode 100644 index 00000000000..ccb95624bad --- /dev/null +++ b/src/backend.rs @@ -0,0 +1,222 @@ +use std::collections::HashMap; + +use rustc::session::Session; + +use cranelift_module::{FuncId, Module}; + +use faerie::*; +#[cfg(feature = "backend_object")] +use object::{SectionKind, RelocationKind, RelocationEncoding}; +#[cfg(feature = "backend_object")] +use object::write::*; +use cranelift_faerie::{FaerieBackend, FaerieBuilder, FaerieProduct, FaerieTrapCollection}; + +#[cfg(feature = "backend_object")] +use cranelift_object::*; + +use gimli::SectionId; + +use crate::debuginfo::{DebugReloc, DebugRelocName}; + +pub trait WriteMetadata { + fn add_rustc_section(&mut self, symbol_name: String, data: Vec, is_like_osx: bool); +} + +impl WriteMetadata for faerie::Artifact { + fn add_rustc_section(&mut self, symbol_name: String, data: Vec, is_like_osx: bool) { + self + .declare(".rustc", faerie::Decl::section(faerie::SectionKind::Data)) + .unwrap(); + self + .define_with_symbols(".rustc", data, { + let mut map = std::collections::BTreeMap::new(); + // FIXME implement faerie elf backend section custom symbols + // For MachO this is necessary to prevent the linker from throwing away the .rustc section, + // but for ELF it isn't. + if is_like_osx { + map.insert( + symbol_name, + 0, + ); + } + map + }) + .unwrap(); + } +} + +#[cfg(feature = "backend_object")] +impl WriteMetadata for object::write::Object { + fn add_rustc_section(&mut self, symbol_name: String, data: Vec, _is_like_osx: bool) { + let segment = self.segment_name(object::write::StandardSegment::Data).to_vec(); + let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data); + let offset = self.append_section_data(section_id, &data, 1); + // For MachO and probably PE this is necessary to prevent the linker from throwing away the + // .rustc section. For ELF this isn't necessary, but it also doesn't harm. + self.add_symbol(object::write::Symbol { + name: symbol_name.into_bytes(), + value: offset, + size: data.len() as u64, + kind: object::SymbolKind::Data, + scope: object::SymbolScope::Dynamic, + weak: false, + section: Some(section_id), + }); + } +} + +pub trait WriteDebugInfo { + type SectionId; + + fn add_debug_section(&mut self, name: SectionId, data: Vec) -> Self::SectionId; + fn add_debug_reloc( + &mut self, + section_map: &HashMap, + symbol_map: &indexmap::IndexMap, + from: &Self::SectionId, + reloc: &DebugReloc, + ); +} + +impl WriteDebugInfo for FaerieProduct { + type SectionId = SectionId; + + fn add_debug_section(&mut self, id: SectionId, data: Vec) -> SectionId { + self.artifact.declare_with(id.name(), Decl::section(faerie::SectionKind::Debug), data).unwrap(); + id + } + + fn add_debug_reloc( + &mut self, + _section_map: &HashMap, + symbol_map: &indexmap::IndexMap, + from: &Self::SectionId, + reloc: &DebugReloc, + ) { + self + .artifact + .link_with( + faerie::Link { + from: from.name(), + to: match reloc.name { + DebugRelocName::Section(id) => id.name(), + DebugRelocName::Symbol(index) => &symbol_map.get_index(index).unwrap().1, + }, + at: u64::from(reloc.offset), + }, + faerie::Reloc::Debug { + size: reloc.size, + addend: reloc.addend as i32, + }, + ) + .expect("faerie relocation error"); + } +} + +#[cfg(feature = "backend_object")] +impl WriteDebugInfo for ObjectProduct { + type SectionId = (object::write::SectionId, object::write::SymbolId); + + fn add_debug_section( + &mut self, + id: SectionId, + data: Vec, + ) -> (object::write::SectionId, object::write::SymbolId) { + let segment = self.object.segment_name(StandardSegment::Debug).to_vec(); + let name = id.name().as_bytes().to_vec(); + let section_id = self.object.add_section(segment, name, SectionKind::Debug); + self.object.section_mut(section_id).set_data(data, 1); + let symbol_id = self.object.section_symbol(section_id); + (section_id, symbol_id) + } + + fn add_debug_reloc( + &mut self, + section_map: &HashMap, + symbol_map: &indexmap::IndexMap, + from: &Self::SectionId, + reloc: &DebugReloc, + ) { + let symbol = match reloc.name { + DebugRelocName::Section(id) => section_map.get(&id).unwrap().1, + DebugRelocName::Symbol(id) => { + self.function_symbol(*symbol_map.get_index(id).unwrap().0) + } + }; + self.object.add_relocation(from.0, Relocation { + offset: u64::from(reloc.offset), + symbol, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + size: reloc.size * 8, + addend: reloc.addend, + }).unwrap(); + } +} + +pub trait Emit { + fn emit(self) -> Vec; +} + +impl Emit for FaerieProduct { + fn emit(self) -> Vec { + self.artifact.emit().unwrap() + } +} + +#[cfg(feature = "backend_object")] +impl Emit for ObjectProduct { + fn emit(self) -> Vec { + self.object.write().unwrap() + } +} + +#[cfg(not(feature = "backend_object"))] +pub fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Artifact)) -> Vec { + let mut metadata_artifact = faerie::Artifact::new( + crate::build_isa(sess, true).triple().clone(), + name.to_string(), + ); + f(&mut metadata_artifact); + metadata_artifact.emit().unwrap() +} + +#[cfg(feature = "backend_object")] +pub fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec { + let triple = crate::build_isa(sess, true).triple().clone(); + let mut metadata_object = + object::write::Object::new(triple.binary_format, triple.architecture); + metadata_object.add_file_symbol(name.as_bytes().to_vec()); + f(&mut metadata_object); + metadata_object.write().unwrap() +} + +pub type Backend = impl cranelift_module::Backend; + +#[cfg(not(feature = "backend_object"))] +pub fn make_module(sess: &Session, name: String) -> Module { + let module: Module = Module::new( + FaerieBuilder::new( + crate::build_isa(sess, true), + name + ".o", + FaerieTrapCollection::Disabled, + cranelift_module::default_libcall_names(), + ) + .unwrap(), + ); + module +} + +#[cfg(feature = "backend_object")] +pub fn make_module(sess: &Session, name: String) -> Module { + let module: Module = Module::new( + ObjectBuilder::new( + crate::build_isa(sess, true), + name + ".o", + ObjectTrapCollection::Disabled, + cranelift_module::default_libcall_names(), + ) + .unwrap(), + ); + module +} diff --git a/src/base.rs b/src/base.rs index 81750efd169..ef32dafb2ef 100644 --- a/src/base.rs +++ b/src/base.rs @@ -17,7 +17,7 @@ pub fn trans_fn<'clif, 'tcx, B: Backend + 'static>( let mut debug_context = cx .debug_context .as_mut() - .map(|debug_context| FunctionDebugContext::new(tcx, debug_context, mir, &name, &sig)); + .map(|debug_context| FunctionDebugContext::new(tcx, debug_context, mir, func_id, &name, &sig)); // Make FunctionBuilder let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); diff --git a/src/debuginfo.rs b/src/debuginfo.rs index 0ded96cec8a..762fa8668bf 100644 --- a/src/debuginfo.rs +++ b/src/debuginfo.rs @@ -1,5 +1,7 @@ use crate::prelude::*; +use crate::backend::WriteDebugInfo; + use std::marker::PhantomData; use syntax::source_map::FileName; @@ -10,8 +12,6 @@ }; use gimli::{Encoding, Format, LineEncoding, RunTimeEndian, SectionId}; -use faerie::*; - fn target_endian(tcx: TyCtxt) -> RunTimeEndian { use rustc::ty::layout::Endian; @@ -56,31 +56,22 @@ fn line_program_add_file( } #[derive(Clone)] -struct DebugReloc { - offset: u32, - size: u8, - name: DebugRelocName, - addend: i64, +pub struct DebugReloc { + pub offset: u32, + pub size: u8, + pub name: DebugRelocName, + pub addend: i64, } #[derive(Clone)] -enum DebugRelocName { +pub enum DebugRelocName { Section(SectionId), Symbol(usize), } -impl DebugReloc { - fn name<'a>(&self, ctx: &'a DebugContext) -> &'a str { - match self.name { - DebugRelocName::Section(id) => id.name(), - DebugRelocName::Symbol(index) => ctx.symbols.get_index(index).unwrap(), - } - } -} - pub struct DebugContext<'tcx> { endian: RunTimeEndian, - symbols: indexmap::IndexSet, + symbols: indexmap::IndexMap, dwarf: DwarfUnit, unit_range_list: RangeList, @@ -142,7 +133,7 @@ pub fn new(tcx: TyCtxt<'tcx>, address_size: u8) -> Self { DebugContext { endian: target_endian(tcx), - symbols: indexmap::IndexSet::new(), + symbols: indexmap::IndexMap::new(), dwarf, unit_range_list: RangeList(Vec::new()), @@ -177,7 +168,7 @@ fn emit_location(&mut self, tcx: TyCtxt<'tcx>, entry_id: UnitEntryId, span: Span ); } - pub fn emit(&mut self, artifact: &mut Artifact) { + pub fn emit(&mut self, product: &mut P) { let unit_range_list_id = self.dwarf.unit.ranges.add(self.unit_range_list.clone()); let root = self.dwarf.unit.root(); let root = self.dwarf.unit.get_mut(root); @@ -189,34 +180,20 @@ pub fn emit(&mut self, artifact: &mut Artifact) { let mut sections = Sections::new(WriterRelocate::new(self)); self.dwarf.write(&mut sections).unwrap(); + let mut section_map = HashMap::new(); let _: Result<()> = sections.for_each_mut(|id, section| { if !section.writer.slice().is_empty() { - artifact - .declare_with( - id.name(), - Decl::section(SectionKind::Debug), - section.writer.take(), - ) - .unwrap(); + let section_id = product.add_debug_section(id, section.writer.take()); + section_map.insert(id, section_id); } Ok(()) }); let _: Result<()> = sections.for_each(|id, section| { - for reloc in §ion.relocs { - artifact - .link_with( - faerie::Link { - from: id.name(), - to: reloc.name(self), - at: u64::from(reloc.offset), - }, - faerie::Reloc::Debug { - size: reloc.size, - addend: reloc.addend as i32, - }, - ) - .expect("faerie relocation error"); + if let Some(section_id) = section_map.get(&id) { + for reloc in §ion.relocs { + product.add_debug_reloc(§ion_map, &self.symbols, section_id, reloc); + } } Ok(()) }); @@ -235,10 +212,11 @@ pub fn new( tcx: TyCtxt<'tcx>, debug_context: &'a mut DebugContext<'tcx>, mir: &Body, + func_id: FuncId, name: &str, _sig: &Signature, ) -> Self { - let (symbol, _) = debug_context.symbols.insert_full(name.to_string()); + let (symbol, _) = debug_context.symbols.insert_full(func_id, name.to_string()); // FIXME: add to appropriate scope intead of root let scope = debug_context.dwarf.unit.root(); diff --git a/src/driver.rs b/src/driver.rs index 19f53ebda13..215ae071c3f 100644 --- a/src/driver.rs +++ b/src/driver.rs @@ -8,10 +8,10 @@ use rustc_codegen_ssa::back::linker::LinkerInfo; use rustc_codegen_ssa::CrateInfo; -use cranelift_faerie::*; - use crate::prelude::*; +use crate::backend::{Emit, WriteDebugInfo}; + pub fn codegen_crate( tcx: TyCtxt<'_>, metadata: EncodedMetadata, @@ -147,36 +147,34 @@ fn run_aot( need_metadata_module: bool, ) -> Box { let new_module = |name: String| { - let module: Module = Module::new( - FaerieBuilder::new( - crate::build_isa(tcx.sess, true), - name + ".o", - FaerieTrapCollection::Disabled, - cranelift_module::default_libcall_names(), - ) - .unwrap(), - ); + let module = crate::backend::make_module(tcx.sess, name); assert_eq!(pointer_ty(tcx), module.target_config().pointer_type()); module }; - let emit_module = |kind: ModuleKind, - mut module: Module, - debug: Option| { + fn emit_module( + tcx: TyCtxt<'_>, + name: String, + kind: ModuleKind, + mut module: Module, + debug: Option, + ) -> CompiledModule + where B::Product: Emit + WriteDebugInfo, + { module.finalize_definitions(); - let mut artifact = module.finish().artifact; + let mut product = module.finish(); if let Some(mut debug) = debug { - debug.emit(&mut artifact); + debug.emit(&mut product); } let tmp_file = tcx .output_filenames(LOCAL_CRATE) - .temp_path(OutputType::Object, Some(&artifact.name)); - let obj = artifact.emit().unwrap(); + .temp_path(OutputType::Object, Some(&name)); + let obj = product.emit(); std::fs::write(&tmp_file, obj).unwrap(); CompiledModule { - name: artifact.name, + name, kind, object: Some(tmp_file), bytecode: None, @@ -184,7 +182,7 @@ fn run_aot( } }; - let mut faerie_module = new_module("some_file".to_string()); + let mut module = new_module("some_file".to_string()); let mut debug = if tcx.sess.opts.debuginfo != DebugInfo::None // macOS debuginfo doesn't work yet (see #303) @@ -192,14 +190,14 @@ fn run_aot( { let debug = DebugContext::new( tcx, - faerie_module.target_config().pointer_type().bytes() as u8, + module.target_config().pointer_type().bytes() as u8, ); Some(debug) } else { None }; - codegen_cgus(tcx, &mut faerie_module, &mut debug); + codegen_cgus(tcx, &mut module, &mut debug); tcx.sess.abort_if_errors(); @@ -221,17 +219,14 @@ fn run_aot( .as_str() .to_string(); - let mut metadata_artifact = faerie::Artifact::new( - crate::build_isa(tcx.sess, true).triple().clone(), - metadata_cgu_name.clone(), - ); - crate::metadata::write_metadata(tcx, &mut metadata_artifact); - let tmp_file = tcx .output_filenames(LOCAL_CRATE) .temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); - let obj = metadata_artifact.emit().unwrap(); + let obj = crate::backend::with_object(tcx.sess, &metadata_cgu_name, |object| { + crate::metadata::write_metadata(tcx, object); + }); + std::fs::write(&tmp_file, obj).unwrap(); (metadata_cgu_name, tmp_file) @@ -251,12 +246,16 @@ fn run_aot( Box::new(CodegenResults { crate_name: tcx.crate_name(LOCAL_CRATE), modules: vec![emit_module( + tcx, + "some_file".to_string(), ModuleKind::Regular, - faerie_module, + module, debug, )], allocator_module: if created_alloc_shim { Some(emit_module( + tcx, + "allocator_shim".to_string(), ModuleKind::Allocator, allocator_module, None, diff --git a/src/lib.rs b/src/lib.rs index 866a3d31146..169397f14a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(rustc_private, never_type, decl_macro)] +#![feature(rustc_private, never_type, decl_macro, type_alias_impl_trait, associated_type_bounds)] #![allow(intra_doc_link_resolution_failure)] extern crate flate2; @@ -35,6 +35,7 @@ mod analyze; mod archive; mod base; +mod backend; mod cast; mod codegen_i128; mod common; diff --git a/src/metadata.rs b/src/metadata.rs index 8d17922725a..f15e1682d13 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -9,6 +9,8 @@ use rustc_data_structures::rustc_erase_owner; use rustc_target::spec::Target; +use crate::backend::WriteMetadata; + pub struct CraneliftMetadataLoader; impl MetadataLoader for CraneliftMetadataLoader { @@ -51,7 +53,7 @@ fn get_dylib_metadata( } // Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112 -pub fn write_metadata(tcx: TyCtxt<'_>, artifact: &mut faerie::Artifact) -> EncodedMetadata { +pub fn write_metadata(tcx: TyCtxt<'_>, product: &mut P) -> EncodedMetadata { use flate2::write::DeflateEncoder; use flate2::Compression; use std::io::Write; @@ -95,24 +97,11 @@ enum MetadataKind { .write_all(&metadata.raw_data) .unwrap(); - artifact - .declare(".rustc", faerie::Decl::section(faerie::SectionKind::Data)) - .unwrap(); - artifact - .define_with_symbols(".rustc", compressed, { - let mut map = std::collections::BTreeMap::new(); - // FIXME implement faerie elf backend section custom symbols - // For MachO this is necessary to prevent the linker from throwing away the .rustc section, - // but for ELF it isn't. - if tcx.sess.target.target.options.is_like_osx { - map.insert( - rustc::middle::exported_symbols::metadata_symbol_name(tcx), - 0, - ); - } - map - }) - .unwrap(); + product.add_rustc_section( + rustc::middle::exported_symbols::metadata_symbol_name(tcx), + compressed, + tcx.sess.target.target.options.is_like_osx, + ); metadata } diff --git a/test.sh b/test.sh index f32ec10268e..773b7c296d4 100755 --- a/test.sh +++ b/test.sh @@ -4,10 +4,10 @@ set -e if [[ "$1" == "--release" ]]; then export CHANNEL='release' - cargo build --release + cargo build --release $CG_CLIF_COMPILE_FLAGS else export CHANNEL='debug' - cargo build + cargo build $CG_CLIF_COMPILE_FLAGS fi source config.sh