1 //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object
4 use std::cell::RefCell;
6 use std::os::raw::{c_char, c_int};
8 use rustc_codegen_ssa::CrateInfo;
9 use rustc_middle::mir::mono::MonoItem;
11 use cranelift_jit::{JITBuilder, JITModule};
13 use crate::prelude::*;
14 use crate::{CodegenCx, CodegenMode};
17 pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
20 pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
21 if !tcx.sess.opts.output_types.should_codegen() {
22 tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
27 // When not using our custom driver rustc will open us without the RTLD_GLOBAL flag, so
28 // __cg_clif_global_atomic_mutex will not be exported. We fix this by opening ourself again
30 // FIXME remove once atomic_shim is gone
32 let mut dl_info: libc::Dl_info = std::mem::zeroed();
34 libc::dladdr(run_jit as *const libc::c_void, &mut dl_info),
38 libc::dlopen(dl_info.dli_fname, libc::RTLD_NOW | libc::RTLD_GLOBAL),
43 let imported_symbols = load_imported_symbols_for_jit(tcx);
45 let mut jit_builder = JITBuilder::with_isa(
46 crate::build_isa(tcx.sess),
47 cranelift_module::default_libcall_names(),
49 jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy));
50 jit_builder.symbols(imported_symbols);
51 let mut jit_module = JITModule::new(jit_builder);
52 assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
56 AbiParam::new(jit_module.target_config().pointer_type()),
57 AbiParam::new(jit_module.target_config().pointer_type()),
59 returns: vec![AbiParam::new(
60 jit_module.target_config().pointer_type(), /*isize*/
62 call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
64 let main_func_id = jit_module
65 .declare_function("main", Linkage::Import, &sig)
68 let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
71 .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter())
73 .collect::<FxHashMap<_, (_, _)>>()
75 .collect::<Vec<(_, (_, _))>>();
77 let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
79 super::time(tcx, "codegen mono items", || {
80 super::predefine_mono_items(&mut cx, &mono_items);
81 for (mono_item, (linkage, visibility)) in mono_items {
82 let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
84 MonoItem::Fn(inst) => match codegen_mode {
85 CodegenMode::Aot => unreachable!(),
87 cx.tcx.sess.time("codegen fn", || {
88 crate::base::codegen_fn(&mut cx, inst, linkage)
91 CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
93 MonoItem::Static(def_id) => {
94 crate::constant::codegen_static(&mut cx.constants_cx, def_id);
96 MonoItem::GlobalAsm(hir_id) => {
97 let item = cx.tcx.hir().expect_item(hir_id);
99 .span_fatal(item.span, "Global asm is not supported in JIT mode");
105 let (mut jit_module, global_asm, _debug, mut unwind_context) =
106 tcx.sess.time("finalize CodegenCx", || cx.finalize());
107 jit_module.finalize_definitions();
109 if !global_asm.is_empty() {
110 tcx.sess.fatal("Inline asm is not supported in JIT mode");
113 crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true);
114 crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
116 tcx.sess.abort_if_errors();
118 jit_module.finalize_definitions();
120 let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };
122 let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
124 println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed");
126 let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
127 unsafe { ::std::mem::transmute(finalized_main) };
129 let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
130 let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
131 .chain(args.split(' '))
132 .map(|arg| CString::new(arg).unwrap())
133 .collect::<Vec<_>>();
134 let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>();
136 // Push a null pointer as a terminating argument. This is required by POSIX and
137 // useful as some dynamic linkers use it as a marker to jump over.
138 argv.push(std::ptr::null());
141 .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
143 let ret = f(args.len() as c_int, argv.as_ptr());
145 std::process::exit(ret);
149 extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 {
150 rustc_middle::ty::tls::with(|tcx| {
151 // lift is used to ensure the correct lifetime for instance.
152 let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
154 CURRENT_MODULE.with(|jit_module| {
155 let mut jit_module = jit_module.borrow_mut();
156 let jit_module = jit_module.as_mut().unwrap();
157 let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
159 let name = tcx.symbol_name(instance).name.to_string();
160 let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance);
163 .declare_function(&name, Linkage::Export, &sig)
165 cx.module.prepare_for_function_redefine(func_id).unwrap();
167 tcx.sess.time("codegen fn", || {
168 crate::base::codegen_fn(&mut cx, instance, Linkage::Export)
171 let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize();
172 assert!(global_asm.is_empty());
173 jit_module.finalize_definitions();
174 std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
175 jit_module.get_finalized_function(func_id)
180 fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
181 use rustc_middle::middle::dependency_format::Linkage;
183 let mut dylib_paths = Vec::new();
185 let crate_info = CrateInfo::new(tcx);
186 let formats = tcx.dependency_formats(LOCAL_CRATE);
189 .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable)
192 for &(cnum, _) in &crate_info.used_crates_dynamic {
193 let src = &crate_info.used_crate_source[&cnum];
194 match data[cnum.as_usize() - 1] {
195 Linkage::NotLinked | Linkage::IncludedFromDylib => {}
197 let name = tcx.crate_name(cnum);
200 .struct_err(&format!("Can't load static lib {}", name.as_str()));
201 err.note("rustc_codegen_cranelift can only load dylibs in JIT mode.");
204 Linkage::Dynamic => {
205 dylib_paths.push(src.dylib.as_ref().unwrap().0.clone());
210 let mut imported_symbols = Vec::new();
211 for path in dylib_paths {
212 use object::{Object, ObjectSymbol};
213 let lib = libloading::Library::new(&path).unwrap();
214 let obj = std::fs::read(path).unwrap();
215 let obj = object::File::parse(&obj).unwrap();
216 imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| {
217 let name = symbol.name().unwrap().to_string();
218 if name.is_empty() || !symbol.is_global() || symbol.is_undefined() {
221 let dlsym_name = if cfg!(target_os = "macos") {
222 // On macOS `dlsym` expects the name without leading `_`.
223 assert!(name.starts_with('_'), "{:?}", name);
228 let symbol: libloading::Symbol<'_, *const u8> =
229 unsafe { lib.get(dlsym_name.as_bytes()) }.unwrap();
230 Some((name, *symbol))
232 std::mem::forget(lib)
235 tcx.sess.abort_if_errors();
240 pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) {
243 let pointer_type = cx.module.target_config().pointer_type();
245 let name = tcx.symbol_name(inst).name.to_string();
246 let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst);
249 .declare_function(&name, Linkage::Export, &sig)
252 let instance_ptr = Box::into_raw(Box::new(inst));
260 call_conv: cx.module.target_config().default_call_conv,
261 params: vec![AbiParam::new(pointer_type)],
262 returns: vec![AbiParam::new(pointer_type)],
267 let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone());
268 let mut builder_ctx = FunctionBuilderContext::new();
269 let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
273 .declare_func_in_func(jit_fn, trampoline_builder.func);
274 let sig_ref = trampoline_builder.func.import_signature(sig);
276 let entry_block = trampoline_builder.create_block();
277 trampoline_builder.append_block_params_for_function_params(entry_block);
278 let fn_args = trampoline_builder
281 .block_params(entry_block)
284 trampoline_builder.switch_to_block(entry_block);
285 let instance_ptr = trampoline_builder
287 .iconst(pointer_type, instance_ptr as u64 as i64);
288 let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]);
289 let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
290 let call_inst = trampoline_builder
292 .call_indirect(sig_ref, jitted_fn, &fn_args);
293 let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
294 trampoline_builder.ins().return_(&ret_vals);
299 &mut Context::for_function(trampoline),
300 &mut cranelift_codegen::binemit::NullTrapSink {},