3 use crate::rustc_target::spec::{HasTargetSpec, Target};
5 use cranelift_module::Module;
9 pub fn mir_var(loc: Local) -> Variable {
10 Variable::with_u32(loc.index() as u32)
13 pub fn pointer_ty(tcx: TyCtxt) -> types::Type {
14 match tcx.data_layout.pointer_size.bits() {
18 bits => bug!("ptr_sized_integer: unknown pointer bit size {}", bits),
22 fn scalar_to_cton_type(tcx: TyCtxt, scalar: &Scalar) -> Type {
23 match scalar.value.size(tcx).bits() {
28 size => bug!("Unsupported scalar size {}", size),
32 pub fn cton_type_from_ty<'a, 'tcx: 'a>(
33 tcx: TyCtxt<'a, 'tcx, 'tcx>,
35 ) -> Option<types::Type> {
37 ty::Bool => types::I8,
38 ty::Uint(size) => match size {
39 UintTy::U8 => types::I8,
40 UintTy::U16 => types::I16,
41 UintTy::U32 => types::I32,
42 UintTy::U64 => types::I64,
43 UintTy::U128 => unimpl!("u128"),
44 UintTy::Usize => pointer_ty(tcx),
46 ty::Int(size) => match size {
47 IntTy::I8 => types::I8,
48 IntTy::I16 => types::I16,
49 IntTy::I32 => types::I32,
50 IntTy::I64 => types::I64,
51 IntTy::I128 => unimpl!("i128"),
52 IntTy::Isize => pointer_ty(tcx),
54 ty::Char => types::I32,
55 ty::Float(size) => match size {
56 FloatTy::F32 => types::F32,
57 FloatTy::F64 => types::F64,
59 ty::FnPtr(_) => pointer_ty(tcx),
60 ty::RawPtr(TypeAndMut { ty, mutbl: _ }) | ty::Ref(_, ty, _) => {
61 if ty.is_sized(tcx.at(DUMMY_SP), ParamEnv::reveal_all()) {
67 ty::Param(_) => bug!("{:?}: {:?}", ty, ty.sty),
72 fn codegen_field<'a, 'tcx: 'a>(
73 fx: &mut FunctionCx<'a, 'tcx, impl Backend>,
75 layout: TyLayout<'tcx>,
77 ) -> (Value, TyLayout<'tcx>) {
78 let field_offset = layout.fields.offset(field.index());
79 let field_ty = layout.field(&*fx, field.index());
80 if field_offset.bytes() > 0 {
82 fx.bcx.ins().iadd_imm(base, field_offset.bytes() as i64),
91 #[derive(Debug, Copy, Clone)]
92 pub enum CValue<'tcx> {
93 ByRef(Value, TyLayout<'tcx>),
94 ByVal(Value, TyLayout<'tcx>),
95 ByValPair(Value, Value, TyLayout<'tcx>),
98 impl<'tcx> CValue<'tcx> {
99 pub fn layout(&self) -> TyLayout<'tcx> {
101 CValue::ByRef(_, layout)
102 | CValue::ByVal(_, layout)
103 | CValue::ByValPair(_, _, layout) => layout,
107 pub fn force_stack<'a>(self, fx: &mut FunctionCx<'a, 'tcx, impl Backend>) -> Value
112 CValue::ByRef(value, _layout) => value,
113 CValue::ByVal(value, layout) => {
114 let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
115 kind: StackSlotKind::ExplicitSlot,
116 size: layout.size.bytes() as u32,
119 fx.bcx.ins().stack_store(value, stack_slot, 0);
122 .stack_addr(fx.module.pointer_type(), stack_slot, 0)
124 CValue::ByValPair(value, extra, layout) => {
125 let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
126 kind: StackSlotKind::ExplicitSlot,
127 size: layout.size.bytes() as u32,
130 let base = fx.bcx.ins().stack_addr(types::I64, stack_slot, 0);
131 let a_addr = codegen_field(fx, base, layout, mir::Field::new(0)).0;
132 let b_addr = codegen_field(fx, base, layout, mir::Field::new(1)).0;
133 fx.bcx.ins().store(MemFlags::new(), value, a_addr, 0);
134 fx.bcx.ins().store(MemFlags::new(), extra, b_addr, 0);
140 pub fn load_value<'a>(self, fx: &mut FunctionCx<'a, 'tcx, impl Backend>) -> Value
145 CValue::ByRef(addr, layout) => {
146 let cton_ty = fx.cton_type(layout.ty).unwrap_or_else(|| {
147 if layout.ty.is_box() && !fx
148 .layout_of(layout.ty.builtin_deref(true).unwrap().ty)
151 // Consider sized box to be a ptr
154 panic!("load_value of type {:?}", layout.ty);
157 fx.bcx.ins().load(cton_ty, MemFlags::new(), addr, 0)
159 CValue::ByVal(value, _layout) => value,
160 CValue::ByValPair(_, _, _layout) => bug!("Please use load_value_pair for ByValPair"),
164 pub fn load_value_pair<'a>(self, fx: &mut FunctionCx<'a, 'tcx, impl Backend>) -> (Value, Value)
169 CValue::ByRef(addr, layout) => {
172 fx.tcx.data_layout.pointer_size.bytes() * 2
174 let val1_offset = layout.fields.offset(0).bytes() as i32;
175 let val2_offset = layout.fields.offset(1).bytes() as i32;
179 .load(fx.module.pointer_type(), MemFlags::new(), addr, val1_offset);
183 .load(fx.module.pointer_type(), MemFlags::new(), addr, val2_offset);
186 CValue::ByVal(_, _layout) => bug!("Please use load_value for ByVal"),
187 CValue::ByValPair(val1, val2, _layout) => (val1, val2),
191 pub fn expect_byref(self) -> (Value, TyLayout<'tcx>) {
193 CValue::ByRef(value, layout) => (value, layout),
194 CValue::ByVal(_, _) => bug!("Expected CValue::ByRef, found CValue::ByVal: {:?}", self),
195 CValue::ByValPair(_, _, _) => bug!(
196 "Expected CValue::ByRef, found CValue::ByValPair: {:?}",
202 pub fn value_field<'a>(
204 fx: &mut FunctionCx<'a, 'tcx, impl Backend>,
210 let (base, layout) = match self {
211 CValue::ByRef(addr, layout) => (addr, layout),
212 _ => bug!("place_field for {:?}", self),
215 let (field_ptr, field_layout) = codegen_field(fx, base, layout, field);
216 CValue::ByRef(field_ptr, field_layout)
219 pub fn unsize_value<'a>(self, fx: &mut FunctionCx<'a, 'tcx, impl Backend>, dest: CPlace<'tcx>) {
220 if self.layout().ty == dest.layout().ty {
221 dest.write_cvalue(fx, self); // FIXME this shouldn't happen (rust-lang/rust#53602)
224 match &self.layout().ty.sty {
225 ty::Ref(_, ty, _) | ty::RawPtr(TypeAndMut { ty, mutbl: _ }) => {
226 let (ptr, extra) = match dest.layout().ty.builtin_deref(true).unwrap().ty.sty {
227 ty::Slice(slice_elem_ty) => match ty.sty {
228 ty::Array(array_elem_ty, size) => {
229 assert_eq!(slice_elem_ty, array_elem_ty);
230 let ptr = self.load_value(fx);
234 .iconst(fx.module.pointer_type(), size.unwrap_usize(fx.tcx) as i64);
237 _ => bug!("unsize non array {:?} to slice", ty),
239 ty::Dynamic(data, _) => match ty.sty {
240 ty::Dynamic(_, _) => self.load_value_pair(fx),
242 let ptr = self.load_value(fx);
243 let vtable = crate::vtable::get_vtable(fx, ty, data.principal());
248 "unsize of type {:?} to {:?}",
253 dest.write_cvalue(fx, CValue::ByValPair(ptr, extra, dest.layout()));
256 assert!(!self.layout().ty.is_enum(), "Tried to unsize enum");
257 let field_count = self.layout().fields.count();
258 let mut found_unsize_field = false;
259 for idx in 0..field_count {
260 let field_dest = dest.place_field(fx, mir::Field::new(idx));
261 let field_src = self.value_field(fx, mir::Field::new(idx));
262 if field_src.layout().ty.is_phantom_data() {
263 // Ignore PhantomData so for example `Unique<()>` can coerce to `Unique<Debug>`
266 // struct Unique<T: ?Sized> {
267 // pointer: NonZero<*const T>,
268 // _marker: PhantomData<T>,
273 if field_src.layout().ty != field_dest.layout().ty {
274 assert!(!found_unsize_field);
275 found_unsize_field = true;
276 field_src.unsize_value(fx, field_dest);
278 field_dest.write_cvalue(fx, field_src);
285 pub fn const_val<'a>(
286 fx: &mut FunctionCx<'a, 'tcx, impl Backend>,
293 let cton_ty = fx.cton_type(ty).unwrap();
294 let layout = fx.layout_of(ty);
295 CValue::ByVal(fx.bcx.ins().iconst(cton_ty, const_val), layout)
298 pub fn unchecked_cast_to(self, layout: TyLayout<'tcx>) -> Self {
300 CValue::ByRef(addr, _) => CValue::ByRef(addr, layout),
301 CValue::ByVal(val, _) => CValue::ByVal(val, layout),
302 CValue::ByValPair(val, extra, _) => CValue::ByValPair(val, extra, layout),
307 /// A place where you can write a value to or read a value from
308 #[derive(Debug, Copy, Clone)]
309 pub enum CPlace<'tcx> {
310 Var(Local, TyLayout<'tcx>),
311 Addr(Value, Option<Value>, TyLayout<'tcx>),
314 impl<'a, 'tcx: 'a> CPlace<'tcx> {
315 pub fn layout(&self) -> TyLayout<'tcx> {
317 CPlace::Var(_, layout) | CPlace::Addr(_, _, layout) => layout,
321 pub fn temp(fx: &mut FunctionCx<'a, 'tcx, impl Backend>, ty: Ty<'tcx>) -> CPlace<'tcx> {
322 let layout = fx.layout_of(ty);
323 assert!(!layout.is_unsized());
324 let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
325 kind: StackSlotKind::ExplicitSlot,
326 size: layout.size.bytes() as u32,
332 .stack_addr(fx.module.pointer_type(), stack_slot, 0),
338 pub fn from_stack_slot(
339 fx: &mut FunctionCx<'a, 'tcx, impl Backend>,
340 stack_slot: StackSlot,
343 let layout = fx.layout_of(ty);
344 assert!(!layout.is_unsized());
348 .stack_addr(fx.module.pointer_type(), stack_slot, 0),
354 pub fn to_cvalue(self, fx: &mut FunctionCx<'a, 'tcx, impl Backend>) -> CValue<'tcx> {
356 CPlace::Var(var, layout) => CValue::ByVal(fx.bcx.use_var(mir_var(var)), layout),
357 CPlace::Addr(addr, extra, layout) => {
358 assert!(extra.is_none(), "unsized values are not yet supported");
359 CValue::ByRef(addr, layout)
364 pub fn expect_addr(self) -> Value {
366 CPlace::Addr(addr, None, _layout) => addr,
367 CPlace::Addr(_, _, _) => bug!("Expected sized CPlace::Addr, found {:?}", self),
368 CPlace::Var(_, _) => bug!("Expected CPlace::Addr, found CPlace::Var"),
372 pub fn write_cvalue(self, fx: &mut FunctionCx<'a, 'tcx, impl Backend>, from: CValue<'tcx>) {
373 match (&self.layout().ty.sty, &from.layout().ty.sty) {
374 (ty::Ref(_, t, dest_mut), ty::Ref(_, u, src_mut))
375 if (if *dest_mut != crate::rustc::hir::Mutability::MutImmutable && src_mut != dest_mut {
383 // &mut T -> &T is allowed
384 // &'a T -> &'b T is allowed
390 "Can't write value of incompatible type to place {:?} {:?}\n\n{:#?}",
391 self.layout().ty.sty,
392 from.layout().ty.sty,
399 CPlace::Var(var, _) => {
400 let data = from.load_value(fx);
401 fx.bcx.def_var(mir_var(var), data)
403 CPlace::Addr(addr, None, layout) => {
404 let size = layout.size.bytes() as i32;
407 CValue::ByVal(val, _layout) => {
408 fx.bcx.ins().store(MemFlags::new(), val, addr, 0);
410 CValue::ByValPair(val1, val2, _layout) => {
411 let val1_offset = layout.fields.offset(0).bytes() as i32;
412 let val2_offset = layout.fields.offset(1).bytes() as i32;
413 fx.bcx.ins().store(MemFlags::new(), val1, addr, val1_offset);
414 fx.bcx.ins().store(MemFlags::new(), val2, addr, val2_offset);
416 CValue::ByRef(from, _layout) => {
418 while size - offset >= 8 {
419 let byte = fx.bcx.ins().load(
420 fx.module.pointer_type(),
425 fx.bcx.ins().store(MemFlags::new(), byte, addr, offset);
428 while size - offset >= 4 {
429 let byte = fx.bcx.ins().load(types::I32, MemFlags::new(), from, offset);
430 fx.bcx.ins().store(MemFlags::new(), byte, addr, offset);
433 while offset < size {
434 let byte = fx.bcx.ins().load(types::I8, MemFlags::new(), from, offset);
435 fx.bcx.ins().store(MemFlags::new(), byte, addr, offset);
441 CPlace::Addr(_, _, _) => bug!("Can't write value to unsized place {:?}", self),
447 fx: &mut FunctionCx<'a, 'tcx, impl Backend>,
451 CPlace::Var(var, layout) => {
452 bug!("Tried to project {:?}, which is put in SSA var {:?}", layout.ty, var);
454 CPlace::Addr(base, extra, layout) => {
455 let (field_ptr, field_layout) = codegen_field(fx, base, layout, field);
456 CPlace::Addr(field_ptr, extra, field_layout)
463 fx: &mut FunctionCx<'a, 'tcx, impl Backend>,
466 let (elem_layout, addr) = match self.layout().ty.sty {
467 ty::Array(elem_ty, _) => (fx.layout_of(elem_ty), self.expect_addr()),
468 ty::Slice(elem_ty) => (
469 fx.layout_of(elem_ty),
471 CPlace::Addr(addr, _, _) => addr,
472 CPlace::Var(_, _) => bug!("Expected CPlace::Addr found CPlace::Var"),
475 _ => bug!("place_index({:?})", self.layout().ty),
481 .imul_imm(index, elem_layout.size.bytes() as i64);
483 CPlace::Addr(fx.bcx.ins().iadd(addr, offset), None, elem_layout)
486 pub fn place_deref(self, fx: &mut FunctionCx<'a, 'tcx, impl Backend>) -> CPlace<'tcx> {
487 let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap().ty);
488 if !inner_layout.is_unsized() {
489 CPlace::Addr(self.to_cvalue(fx).load_value(fx), None, inner_layout)
491 match self.layout().abi {
492 Abi::ScalarPair(ref a, ref b) => {
493 let addr = self.expect_addr();
497 .load(scalar_to_cton_type(fx.tcx, a), MemFlags::new(), addr, 0);
498 let extra = fx.bcx.ins().load(
499 scalar_to_cton_type(fx.tcx, b),
502 a.value.size(fx.tcx).bytes() as u32 as i32,
504 CPlace::Addr(ptr, Some(extra), inner_layout)
507 "Fat ptr doesn't have abi ScalarPair, but it has {:?}",
514 pub fn write_place_ref(self, fx: &mut FunctionCx<'a, 'tcx, impl Backend>, dest: CPlace<'tcx>) {
515 if !self.layout().is_unsized() {
516 let ptr = CValue::ByVal(self.expect_addr(), dest.layout());
517 dest.write_cvalue(fx, ptr);
520 CPlace::Var(_, _) => bug!("expected CPlace::Addr found CPlace::Var"),
521 CPlace::Addr(value, extra, _) => match dest.layout().abi {
522 Abi::ScalarPair(ref a, _) => {
525 .store(MemFlags::new(), value, dest.expect_addr(), 0);
528 extra.expect("unsized type without metadata"),
530 a.value.size(fx.tcx).bytes() as u32 as i32,
534 "Non ScalarPair abi {:?} in write_place_ref dest",
542 pub fn unchecked_cast_to(self, layout: TyLayout<'tcx>) -> Self {
544 CPlace::Var(var, _) => CPlace::Var(var, layout),
545 CPlace::Addr(addr, extra, _) => {
546 assert!(!layout.is_unsized());
547 CPlace::Addr(addr, extra, layout)
552 pub fn downcast_variant(self, fx: &FunctionCx<'a, 'tcx, impl Backend>, variant: usize) -> Self {
553 let layout = self.layout().for_variant(fx, variant);
554 self.unchecked_cast_to(layout)
558 pub fn cton_intcast<'a, 'tcx: 'a>(
559 fx: &mut FunctionCx<'a, 'tcx, impl Backend>,
564 let from = fx.bcx.func.dfg.value_type(val);
568 if to.wider_or_equal(from) {
570 fx.bcx.ins().sextend(to, val)
572 fx.bcx.ins().uextend(to, val)
575 fx.bcx.ins().ireduce(to, val)
579 pub struct FunctionCx<'a, 'tcx: 'a, B: Backend + 'a> {
580 pub tcx: TyCtxt<'a, 'tcx, 'tcx>,
581 pub module: &'a mut Module<B>,
582 pub instance: Instance<'tcx>,
583 pub mir: &'tcx Mir<'tcx>,
584 pub param_substs: &'tcx Substs<'tcx>,
585 pub bcx: FunctionBuilder<'a>,
586 pub ebb_map: HashMap<BasicBlock, Ebb>,
587 pub local_map: HashMap<Local, CPlace<'tcx>>,
588 pub comments: HashMap<Inst, String>,
589 pub constants: &'a mut crate::constant::ConstantCx,
590 pub caches: &'a mut Caches<'tcx>,
592 /// add_global_comment inserts a comment here
593 pub top_nop: Option<Inst>,
596 impl<'a, 'tcx: 'a, B: Backend + 'a> fmt::Debug for FunctionCx<'a, 'tcx, B> {
597 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
598 writeln!(f, "{:?}", self.param_substs)?;
599 writeln!(f, "{:?}", self.local_map)?;
601 let mut clif = String::new();
602 let mut writer = crate::pretty_clif::CommentWriter(self.comments.clone());
603 ::cranelift::codegen::write::decorate_function(
609 writeln!(f, "\n{}", clif)
613 impl<'a, 'tcx: 'a, B: Backend> LayoutOf for &'a FunctionCx<'a, 'tcx, B> {
615 type TyLayout = TyLayout<'tcx>;
617 fn layout_of(self, ty: Ty<'tcx>) -> TyLayout<'tcx> {
618 let ty = self.monomorphize(&ty);
619 self.tcx.layout_of(ParamEnv::reveal_all().and(&ty)).unwrap()
623 impl<'a, 'tcx, B: Backend + 'a> layout::HasTyCtxt<'tcx> for &'a FunctionCx<'a, 'tcx, B> {
624 fn tcx<'b>(&'b self) -> TyCtxt<'b, 'tcx, 'tcx> {
629 impl<'a, 'tcx, B: Backend + 'a> layout::HasDataLayout for &'a FunctionCx<'a, 'tcx, B> {
630 fn data_layout(&self) -> &layout::TargetDataLayout {
631 &self.tcx.data_layout
635 impl<'a, 'tcx, B: Backend + 'a> HasTargetSpec for &'a FunctionCx<'a, 'tcx, B> {
636 fn target_spec(&self) -> &Target {
637 &self.tcx.sess.target.target
641 impl<'a, 'tcx: 'a, B: Backend + 'a> FunctionCx<'a, 'tcx, B> {
642 pub fn monomorphize<T>(&self, value: &T) -> T
644 T: TypeFoldable<'tcx>,
646 self.tcx.subst_and_normalize_erasing_regions(
648 ty::ParamEnv::reveal_all(),
653 pub fn cton_type(&self, ty: Ty<'tcx>) -> Option<Type> {
654 cton_type_from_ty(self.tcx, self.monomorphize(&ty))
657 pub fn get_ebb(&self, bb: BasicBlock) -> Ebb {
658 *self.ebb_map.get(&bb).unwrap()
661 pub fn get_local_place(&mut self, local: Local) -> CPlace<'tcx> {
662 *self.local_map.get(&local).unwrap()