1 //! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object
4 use std::cell::RefCell;
6 use std::os::raw::{c_char, c_int};
8 use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
9 use rustc_codegen_ssa::CrateInfo;
10 use rustc_middle::mir::mono::MonoItem;
12 use cranelift_jit::{JITBuilder, JITModule};
14 use crate::{prelude::*, BackendConfig};
15 use crate::{CodegenCx, CodegenMode};
18 backend_config: BackendConfig,
19 jit_module: JITModule,
23 static LAZY_JIT_STATE: RefCell<Option<JitState>> = RefCell::new(None);
26 fn create_jit_module<'tcx>(
28 backend_config: &BackendConfig,
30 ) -> (JITModule, CodegenCx<'tcx>) {
31 let imported_symbols = load_imported_symbols_for_jit(tcx);
33 let isa = crate::build_isa(tcx.sess, backend_config);
34 let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
35 jit_builder.hotswap(hotswap);
36 crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
37 jit_builder.symbols(imported_symbols);
38 let mut jit_module = JITModule::new(jit_builder);
40 let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false);
42 crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context);
43 crate::main_shim::maybe_create_entry_wrapper(
46 &mut cx.unwind_context,
53 pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
54 if !tcx.sess.opts.output_types.should_codegen() {
55 tcx.sess.fatal("JIT mode doesn't work with `cargo check`");
58 if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) {
59 tcx.sess.fatal("can't jit non-executable crate");
62 let (mut jit_module, mut cx) = create_jit_module(
65 matches!(backend_config.codegen_mode, CodegenMode::JitLazy),
68 let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
71 .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter())
73 .collect::<FxHashMap<_, (_, _)>>()
75 .collect::<Vec<(_, (_, _))>>();
77 super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
78 super::predefine_mono_items(tcx, &mut jit_module, &mono_items);
79 for (mono_item, _) in mono_items {
81 MonoItem::Fn(inst) => match backend_config.codegen_mode {
82 CodegenMode::Aot => unreachable!(),
84 cx.tcx.sess.time("codegen fn", || {
85 crate::base::codegen_fn(&mut cx, &mut jit_module, inst)
88 CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst),
90 MonoItem::Static(def_id) => {
91 crate::constant::codegen_static(tcx, &mut jit_module, def_id);
93 MonoItem::GlobalAsm(item_id) => {
94 let item = tcx.hir().item(item_id);
95 tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode");
101 if !cx.global_asm.is_empty() {
102 tcx.sess.fatal("Inline asm is not supported in JIT mode");
105 tcx.sess.abort_if_errors();
107 jit_module.finalize_definitions();
108 unsafe { cx.unwind_context.register_jit(&jit_module) };
111 "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
114 let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
115 .chain(backend_config.jit_args.iter().map(|arg| &**arg))
116 .map(|arg| CString::new(arg).unwrap())
117 .collect::<Vec<_>>();
118 let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>();
120 // Push a null pointer as a terminating argument. This is required by POSIX and
121 // useful as some dynamic linkers use it as a marker to jump over.
122 argv.push(std::ptr::null());
124 let start_sig = Signature {
126 AbiParam::new(jit_module.target_config().pointer_type()),
127 AbiParam::new(jit_module.target_config().pointer_type()),
129 returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
130 call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
132 let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap();
133 let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
135 LAZY_JIT_STATE.with(|lazy_jit_state| {
136 let mut lazy_jit_state = lazy_jit_state.borrow_mut();
137 assert!(lazy_jit_state.is_none());
138 *lazy_jit_state = Some(JitState { backend_config, jit_module });
141 let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
142 unsafe { ::std::mem::transmute(finalized_start) };
143 let ret = f(args.len() as c_int, argv.as_ptr());
144 std::process::exit(ret);
148 extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 {
149 rustc_middle::ty::tls::with(|tcx| {
150 // lift is used to ensure the correct lifetime for instance.
151 let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
153 LAZY_JIT_STATE.with(|lazy_jit_state| {
154 let mut lazy_jit_state = lazy_jit_state.borrow_mut();
155 let lazy_jit_state = lazy_jit_state.as_mut().unwrap();
156 let jit_module = &mut lazy_jit_state.jit_module;
157 let backend_config = lazy_jit_state.backend_config.clone();
159 let name = tcx.symbol_name(instance).name;
160 let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance);
161 let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap();
162 jit_module.prepare_for_function_redefine(func_id).unwrap();
164 let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false);
165 tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance));
167 assert!(cx.global_asm.is_empty());
168 jit_module.finalize_definitions();
169 unsafe { cx.unwind_context.register_jit(&jit_module) };
170 jit_module.get_finalized_function(func_id)
175 fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
176 use rustc_middle::middle::dependency_format::Linkage;
178 let mut dylib_paths = Vec::new();
180 let crate_info = CrateInfo::new(tcx);
181 let formats = tcx.dependency_formats(LOCAL_CRATE);
184 .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable)
187 for &(cnum, _) in &crate_info.used_crates_dynamic {
188 let src = &crate_info.used_crate_source[&cnum];
189 match data[cnum.as_usize() - 1] {
190 Linkage::NotLinked | Linkage::IncludedFromDylib => {}
192 let name = tcx.crate_name(cnum);
194 tcx.sess.struct_err(&format!("Can't load static lib {}", name.as_str()));
195 err.note("rustc_codegen_cranelift can only load dylibs in JIT mode.");
198 Linkage::Dynamic => {
199 dylib_paths.push(src.dylib.as_ref().unwrap().0.clone());
204 let mut imported_symbols = Vec::new();
205 for path in dylib_paths {
206 use object::{Object, ObjectSymbol};
207 let lib = libloading::Library::new(&path).unwrap();
208 let obj = std::fs::read(path).unwrap();
209 let obj = object::File::parse(&obj).unwrap();
210 imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| {
211 let name = symbol.name().unwrap().to_string();
212 if name.is_empty() || !symbol.is_global() || symbol.is_undefined() {
215 if name.starts_with("rust_metadata_") {
216 // The metadata is part of a section that is not loaded by the dynamic linker in
220 let dlsym_name = if cfg!(target_os = "macos") {
221 // On macOS `dlsym` expects the name without leading `_`.
222 assert!(name.starts_with('_'), "{:?}", name);
227 let symbol: libloading::Symbol<'_, *const u8> =
228 unsafe { lib.get(dlsym_name.as_bytes()) }.unwrap();
229 Some((name, *symbol))
231 std::mem::forget(lib)
234 tcx.sess.abort_if_errors();
239 fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) {
242 let pointer_type = module.target_config().pointer_type();
244 let name = tcx.symbol_name(inst).name;
245 let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst);
246 let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap();
248 let instance_ptr = Box::into_raw(Box::new(inst));
255 call_conv: module.target_config().default_call_conv,
256 params: vec![AbiParam::new(pointer_type)],
257 returns: vec![AbiParam::new(pointer_type)],
262 cx.cached_context.clear();
263 let trampoline = &mut cx.cached_context.func;
264 trampoline.signature = sig.clone();
266 let mut builder_ctx = FunctionBuilderContext::new();
267 let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx);
269 let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func);
270 let sig_ref = trampoline_builder.func.import_signature(sig);
272 let entry_block = trampoline_builder.create_block();
273 trampoline_builder.append_block_params_for_function_params(entry_block);
274 let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec();
276 trampoline_builder.switch_to_block(entry_block);
277 let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64);
278 let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]);
279 let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
280 let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args);
281 let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
282 trampoline_builder.ins().return_(&ret_vals);
287 &mut cx.cached_context,
288 &mut NullTrapSink {},
289 &mut NullStackMapSink {},