compiler/rustc_codegen_cranelift/src/driver/jit.rs

   1 //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object
   2 //! files.
   3
   4 use std::cell::RefCell;
   5 use std::ffi::CString;
   6 use std::os::raw::{c_char, c_int};
   7
   8 use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
   9 use rustc_codegen_ssa::CrateInfo;
  10 use rustc_middle::mir::mono::MonoItem;
  11 use rustc_session::config::EntryFnType;
  12
  13 use cranelift_jit::{JITBuilder, JITModule};
  14
  15 use crate::{prelude::*, BackendConfig};
  16 use crate::{CodegenCx, CodegenMode};
  17
  18 thread_local! {
  19     pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None);
  20     pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
  21 }
  22
  23 pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
  24     if !tcx.sess.opts.output_types.should_codegen() {
  25         tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
  26     }
  27
  28     let imported_symbols = load_imported_symbols_for_jit(tcx);
  29
  30     let mut jit_builder =
  31         JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
  32     jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
  33     crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
  34     jit_builder.symbols(imported_symbols);
  35     let mut jit_module = JITModule::new(jit_builder);
  36     assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
  37
  38     let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
  39     let mono_items = cgus
  40         .iter()
  41         .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter())
  42         .flatten()
  43         .collect::<FxHashMap<_, (_, _)>>()
  44         .into_iter()
  45         .collect::<Vec<(_, (_, _))>>();
  46
  47     let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false);
  48
  49     super::time(tcx, "codegen mono items", || {
  50         super::predefine_mono_items(&mut cx, &mono_items);
  51         for (mono_item, _) in mono_items {
  52             match mono_item {
  53                 MonoItem::Fn(inst) => match backend_config.codegen_mode {
  54                     CodegenMode::Aot => unreachable!(),
  55                     CodegenMode::Jit => {
  56                         cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
  57                     }
  58                     CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
  59                 },
  60                 MonoItem::Static(def_id) => {
  61                     crate::constant::codegen_static(&mut cx.constants_cx, def_id);
  62                 }
  63                 MonoItem::GlobalAsm(item_id) => {
  64                     let item = cx.tcx.hir().item(item_id);
  65                     tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode");
  66                 }
  67             }
  68         }
  69     });
  70
  71     let (global_asm, _debug, mut unwind_context) =
  72         tcx.sess.time("finalize CodegenCx", || cx.finalize());
  73     jit_module.finalize_definitions();
  74
  75     if !global_asm.is_empty() {
  76         tcx.sess.fatal("Inline asm is not supported in JIT mode");
  77     }
  78
  79     crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
  80
  81     tcx.sess.abort_if_errors();
  82
  83     jit_module.finalize_definitions();
  84     let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };
  85
  86     println!(
  87         "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
  88     );
  89
  90     let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
  91     let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
  92         .chain(args.split(' '))
  93         .map(|arg| CString::new(arg).unwrap())
  94         .collect::<Vec<_>>();
  95     let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>();
  96
  97     // Push a null pointer as a terminating argument. This is required by POSIX and
  98     // useful as some dynamic linkers use it as a marker to jump over.
  99     argv.push(std::ptr::null());
 100
 101     BACKEND_CONFIG.with(|tls_backend_config| {
 102         assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
 103     });
 104
 105     let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap();
 106     let instance = Instance::mono(tcx, main_def_id.to_def_id()).polymorphize(tcx);
 107
 108     match entry_ty {
 109         EntryFnType::Main => {
 110             // FIXME set program arguments somehow
 111
 112             let main_sig = Signature {
 113                 params: vec![],
 114                 returns: vec![],
 115                 call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
 116             };
 117             let main_func_id = jit_module
 118                 .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig)
 119                 .unwrap();
 120             let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
 121
 122             CURRENT_MODULE.with(|current_module| {
 123                 assert!(current_module.borrow_mut().replace(jit_module).is_none())
 124             });
 125
 126             let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) };
 127             f();
 128             std::process::exit(0);
 129         }
 130         EntryFnType::Start => {
 131             let start_sig = Signature {
 132                 params: vec![
 133                     AbiParam::new(jit_module.target_config().pointer_type()),
 134                     AbiParam::new(jit_module.target_config().pointer_type()),
 135                 ],
 136                 returns: vec![AbiParam::new(
 137                     jit_module.target_config().pointer_type(), /*isize*/
 138                 )],
 139                 call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
 140             };
 141             let start_func_id = jit_module
 142                 .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig)
 143                 .unwrap();
 144             let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
 145
 146             CURRENT_MODULE.with(|current_module| {
 147                 assert!(current_module.borrow_mut().replace(jit_module).is_none())
 148             });
 149
 150             let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
 151                 unsafe { ::std::mem::transmute(finalized_start) };
 152             let ret = f(args.len() as c_int, argv.as_ptr());
 153             std::process::exit(ret);
 154         }
 155     }
 156 }
 157
 158 #[no_mangle]
 159 extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 {
 160     rustc_middle::ty::tls::with(|tcx| {
 161         // lift is used to ensure the correct lifetime for instance.
 162         let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
 163
 164         CURRENT_MODULE.with(|jit_module| {
 165             let mut jit_module = jit_module.borrow_mut();
 166             let jit_module = jit_module.as_mut().unwrap();
 167             let backend_config =
 168                 BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap());
 169
 170             let name = tcx.symbol_name(instance).name.to_string();
 171             let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance);
 172             let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap();
 173             jit_module.prepare_for_function_redefine(func_id).unwrap();
 174
 175             let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
 176             tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance));
 177
 178             let (global_asm, _debug_context, unwind_context) = cx.finalize();
 179             assert!(global_asm.is_empty());
 180             jit_module.finalize_definitions();
 181             std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
 182             jit_module.get_finalized_function(func_id)
 183         })
 184     })
 185 }
 186
 187 fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
 188     use rustc_middle::middle::dependency_format::Linkage;
 189
 190     let mut dylib_paths = Vec::new();
 191
 192     let crate_info = CrateInfo::new(tcx);
 193     let formats = tcx.dependency_formats(LOCAL_CRATE);
 194     let data = &formats
 195         .iter()
 196         .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable)
 197         .unwrap()
 198         .1;
 199     for &(cnum, _) in &crate_info.used_crates_dynamic {
 200         let src = &crate_info.used_crate_source[&cnum];
 201         match data[cnum.as_usize() - 1] {
 202             Linkage::NotLinked | Linkage::IncludedFromDylib => {}
 203             Linkage::Static => {
 204                 let name = tcx.crate_name(cnum);
 205                 let mut err =
 206                     tcx.sess.struct_err(&format!("Can't load static lib {}", name.as_str()));
 207                 err.note("rustc_codegen_cranelift can only load dylibs in JIT mode.");
 208                 err.emit();
 209             }
 210             Linkage::Dynamic => {
 211                 dylib_paths.push(src.dylib.as_ref().unwrap().0.clone());
 212             }
 213         }
 214     }
 215
 216     let mut imported_symbols = Vec::new();
 217     for path in dylib_paths {
 218         use object::{Object, ObjectSymbol};
 219         let lib = libloading::Library::new(&path).unwrap();
 220         let obj = std::fs::read(path).unwrap();
 221         let obj = object::File::parse(&obj).unwrap();
 222         imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| {
 223             let name = symbol.name().unwrap().to_string();
 224             if name.is_empty() || !symbol.is_global() || symbol.is_undefined() {
 225                 return None;
 226             }
 227             if name.starts_with("rust_metadata_") {
 228                 // The metadata is part of a section that is not loaded by the dynamic linker in
 229                 // case of cg_llvm.
 230                 return None;
 231             }
 232             let dlsym_name = if cfg!(target_os = "macos") {
 233                 // On macOS `dlsym` expects the name without leading `_`.
 234                 assert!(name.starts_with('_'), "{:?}", name);
 235                 &name[1..]
 236             } else {
 237                 &name
 238             };
 239             let symbol: libloading::Symbol<'_, *const u8> =
 240                 unsafe { lib.get(dlsym_name.as_bytes()) }.unwrap();
 241             Some((name, *symbol))
 242         }));
 243         std::mem::forget(lib)
 244     }
 245
 246     tcx.sess.abort_if_errors();
 247
 248     imported_symbols
 249 }
 250
 251 fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
 252     let tcx = cx.tcx;
 253
 254     let pointer_type = cx.module.target_config().pointer_type();
 255
 256     let name = tcx.symbol_name(inst).name.to_string();
 257     let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst);
 258     let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap();
 259
 260     let instance_ptr = Box::into_raw(Box::new(inst));
 261
 262     let jit_fn = cx
 263         .module
 264         .declare_function(
 265             "__clif_jit_fn",
 266             Linkage::Import,
 267             &Signature {
 268                 call_conv: cx.module.target_config().default_call_conv,
 269                 params: vec![AbiParam::new(pointer_type)],
 270                 returns: vec![AbiParam::new(pointer_type)],
 271             },
 272         )
 273         .unwrap();
 274
 275     let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone());
 276     let mut builder_ctx = FunctionBuilderContext::new();
 277     let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
 278
 279     let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func);
 280     let sig_ref = trampoline_builder.func.import_signature(sig);
 281
 282     let entry_block = trampoline_builder.create_block();
 283     trampoline_builder.append_block_params_for_function_params(entry_block);
 284     let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec();
 285
 286     trampoline_builder.switch_to_block(entry_block);
 287     let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64);
 288     let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]);
 289     let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
 290     let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args);
 291     let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
 292     trampoline_builder.ins().return_(&ret_vals);
 293
 294     cx.module
 295         .define_function(
 296             func_id,
 297             &mut Context::for_function(trampoline),
 298             &mut NullTrapSink {},
 299             &mut NullStackMapSink {},
 300         )
 301         .unwrap();
 302 }