src/intrinsics.rs

   1
   2 use crate::prelude::*;
   3
   4 pub fn codegen_intrinsic_call<'a, 'tcx: 'a>(
   5     fx: &mut FunctionCx<'a, 'tcx, impl Backend>,
   6     def_id: DefId,
   7     substs: &'tcx Substs,
   8     args: &[CValue<'tcx>],
   9     destination: Option<(CPlace<'tcx>, BasicBlock)>,
  10 ) {
  11     let intrinsic = fx.tcx.item_name(def_id).as_str();
  12     let intrinsic = &intrinsic[..];
  13
  14     let ret = match destination {
  15         Some((place, _)) => place,
  16         None => {
  17             // Insert non returning intrinsics here
  18             match intrinsic {
  19                 "abort" => {
  20                     fx.bcx.ins().trap(TrapCode::User(!0 - 1));
  21                 }
  22                 "unreachable" => {
  23                     fx.bcx.ins().trap(TrapCode::User(!0 - 1));
  24                 }
  25                 _ => unimplemented!("unsupported instrinsic {}", intrinsic),
  26             }
  27             return;
  28         }
  29     };
  30
  31     let u64_layout = fx.layout_of(fx.tcx.types.u64);
  32     let usize_layout = fx.layout_of(fx.tcx.types.usize);
  33
  34     match intrinsic {
  35         "assume" => {
  36             assert_eq!(args.len(), 1);
  37         }
  38         "arith_offset" => {
  39             assert_eq!(args.len(), 2);
  40             let base = args[0].load_value(fx);
  41             let offset = args[1].load_value(fx);
  42             let res = fx.bcx.ins().iadd(base, offset);
  43             let res = CValue::ByVal(res, ret.layout());
  44             ret.write_cvalue(fx, res);
  45         }
  46         "likely" | "unlikely" => {
  47             assert_eq!(args.len(), 1);
  48             ret.write_cvalue(fx, args[0]);
  49         }
  50         "copy" | "copy_nonoverlapping" => {
  51             let elem_ty = substs.type_at(0);
  52             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
  53             let elem_size = fx
  54                 .bcx
  55                 .ins()
  56                 .iconst(fx.module.pointer_type(), elem_size as i64);
  57             assert_eq!(args.len(), 3);
  58             let src = args[0].load_value(fx);
  59             let dst = args[1].load_value(fx);
  60             let count = args[2].load_value(fx);
  61             let byte_amount = fx.bcx.ins().imul(count, elem_size);
  62
  63             if intrinsic.ends_with("_nonoverlapping") {
  64                 fx.bcx.call_memcpy(fx.isa, dst, src, byte_amount);
  65             } else {
  66                 fx.bcx.call_memmove(fx.isa, dst, src, byte_amount);
  67             }
  68         }
  69         "discriminant_value" => {
  70             assert_eq!(args.len(), 1);
  71             let discr = crate::base::trans_get_discriminant(fx, args[0], ret.layout());
  72             ret.write_cvalue(fx, discr);
  73         }
  74         "size_of" => {
  75             assert_eq!(args.len(), 0);
  76             let size_of = fx.layout_of(substs.type_at(0)).size.bytes();
  77             let size_of = CValue::const_val(fx, usize_layout.ty, size_of as i64);
  78             ret.write_cvalue(fx, size_of);
  79         }
  80         "size_of_val" => {
  81             assert_eq!(args.len(), 1);
  82             let layout = fx.layout_of(substs.type_at(0));
  83             let size = match &layout.ty.sty {
  84                 _ if !layout.is_unsized() => fx
  85                     .bcx
  86                     .ins()
  87                     .iconst(fx.module.pointer_type(), layout.size.bytes() as i64),
  88                 ty::Slice(elem) => {
  89                     let len = args[0].load_value_pair(fx).1;
  90                     let elem_size = fx.layout_of(elem).size.bytes();
  91                     fx.bcx.ins().imul_imm(len, elem_size as i64)
  92                 }
  93                 ty::Dynamic(..) => crate::vtable::size_of_obj(fx, args[0]),
  94                 ty => bug!("size_of_val for unknown unsized type {:?}", ty),
  95             };
  96             ret.write_cvalue(fx, CValue::ByVal(size, usize_layout));
  97         }
  98         "min_align_of" => {
  99             assert_eq!(args.len(), 0);
 100             let min_align = fx.layout_of(substs.type_at(0)).align.abi();
 101             let min_align = CValue::const_val(fx, usize_layout.ty, min_align as i64);
 102             ret.write_cvalue(fx, min_align);
 103         }
 104         "min_align_of_val" => {
 105             assert_eq!(args.len(), 1);
 106             let layout = fx.layout_of(substs.type_at(0));
 107             let align = match &layout.ty.sty {
 108                 _ if !layout.is_unsized() => fx
 109                     .bcx
 110                     .ins()
 111                     .iconst(fx.module.pointer_type(), layout.align.abi() as i64),
 112                 ty::Slice(elem) => {
 113                     let align = fx.layout_of(elem).align.abi() as i64;
 114                     fx.bcx.ins().iconst(fx.module.pointer_type(), align)
 115                 }
 116                 ty::Dynamic(..) => crate::vtable::min_align_of_obj(fx, args[0]),
 117                 ty => unimplemented!("min_align_of_val for {:?}", ty),
 118             };
 119             ret.write_cvalue(fx, CValue::ByVal(align, usize_layout));
 120         }
 121         "type_id" => {
 122             assert_eq!(args.len(), 0);
 123             let type_id = fx.tcx.type_id_hash(substs.type_at(0));
 124             let type_id = CValue::const_val(fx, u64_layout.ty, type_id as i64);
 125             ret.write_cvalue(fx, type_id);
 126         }
 127         _ if intrinsic.starts_with("unchecked_") => {
 128             assert_eq!(args.len(), 2);
 129             let bin_op = match intrinsic {
 130                 "unchecked_div" => BinOp::Div,
 131                 "unchecked_rem" => BinOp::Rem,
 132                 "unchecked_shl" => BinOp::Shl,
 133                 "unchecked_shr" => BinOp::Shr,
 134                 _ => unimplemented!("intrinsic {}", intrinsic),
 135             };
 136             let res = match ret.layout().ty.sty {
 137                 ty::Uint(_) => crate::base::trans_int_binop(
 138                     fx,
 139                     bin_op,
 140                     args[0],
 141                     args[1],
 142                     ret.layout().ty,
 143                     false,
 144                 ),
 145                 ty::Int(_) => crate::base::trans_int_binop(
 146                     fx,
 147                     bin_op,
 148                     args[0],
 149                     args[1],
 150                     ret.layout().ty,
 151                     true,
 152                 ),
 153                 _ => panic!(),
 154             };
 155             ret.write_cvalue(fx, res);
 156         }
 157         _ if intrinsic.ends_with("_with_overflow") => {
 158             assert_eq!(args.len(), 2);
 159             assert_eq!(args[0].layout().ty, args[1].layout().ty);
 160             let bin_op = match intrinsic {
 161                 "add_with_overflow" => BinOp::Add,
 162                 "sub_with_overflow" => BinOp::Sub,
 163                 "mul_with_overflow" => BinOp::Mul,
 164                 _ => unimplemented!("intrinsic {}", intrinsic),
 165             };
 166             let res = match args[0].layout().ty.sty {
 167                 ty::Uint(_) => crate::base::trans_checked_int_binop(
 168                     fx,
 169                     bin_op,
 170                     args[0],
 171                     args[1],
 172                     ret.layout().ty,
 173                     false,
 174                 ),
 175                 ty::Int(_) => crate::base::trans_checked_int_binop(
 176                     fx,
 177                     bin_op,
 178                     args[0],
 179                     args[1],
 180                     ret.layout().ty,
 181                     true,
 182                 ),
 183                 _ => panic!(),
 184             };
 185             ret.write_cvalue(fx, res);
 186         }
 187         _ if intrinsic.starts_with("overflowing_") => {
 188             assert_eq!(args.len(), 2);
 189             assert_eq!(args[0].layout().ty, args[1].layout().ty);
 190             let bin_op = match intrinsic {
 191                 "overflowing_add" => BinOp::Add,
 192                 "overflowing_sub" => BinOp::Sub,
 193                 "overflowing_mul" => BinOp::Mul,
 194                 _ => unimplemented!("intrinsic {}", intrinsic),
 195             };
 196             let res = match args[0].layout().ty.sty {
 197                 ty::Uint(_) => crate::base::trans_int_binop(
 198                     fx,
 199                     bin_op,
 200                     args[0],
 201                     args[1],
 202                     ret.layout().ty,
 203                     false,
 204                 ),
 205                 ty::Int(_) => crate::base::trans_int_binop(
 206                     fx,
 207                     bin_op,
 208                     args[0],
 209                     args[1],
 210                     ret.layout().ty,
 211                     true,
 212                 ),
 213                 _ => panic!(),
 214             };
 215             ret.write_cvalue(fx, res);
 216         }
 217         "offset" => {
 218             assert_eq!(args.len(), 2);
 219             let base = args[0].load_value(fx);
 220             let offset = args[1].load_value(fx);
 221             let res = fx.bcx.ins().iadd(base, offset);
 222             ret.write_cvalue(fx, CValue::ByVal(res, args[0].layout()));
 223         }
 224         "transmute" => {
 225             assert_eq!(args.len(), 1);
 226             let src_ty = substs.type_at(0);
 227             let dst_ty = substs.type_at(1);
 228             assert_eq!(args[0].layout().ty, src_ty);
 229             let addr = args[0].force_stack(fx);
 230             let dst_layout = fx.layout_of(dst_ty);
 231             ret.write_cvalue(fx, CValue::ByRef(addr, dst_layout))
 232         }
 233         "init" => {
 234             assert_eq!(args.len(), 0);
 235             let ty = substs.type_at(0);
 236             let layout = fx.layout_of(ty);
 237             let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
 238                 kind: StackSlotKind::ExplicitSlot,
 239                 size: layout.size.bytes() as u32,
 240                 offset: None,
 241             });
 242             let addr = fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0);
 243             let zero_val = fx.bcx.ins().iconst(types::I8, 0);
 244             let len_val = fx.bcx.ins().iconst(pointer_ty(fx.tcx), layout.size.bytes() as i64);
 245             fx.bcx.call_memset(fx.isa, addr, zero_val, len_val);
 246
 247             let uninit_place = CPlace::from_stack_slot(fx, stack_slot, ty);
 248             let uninit_val = uninit_place.to_cvalue(fx);
 249             ret.write_cvalue(fx, uninit_val);
 250         }
 251         "uninit" => {
 252             assert_eq!(args.len(), 0);
 253             let ty = substs.type_at(0);
 254             let layout = fx.layout_of(ty);
 255             let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
 256                 kind: StackSlotKind::ExplicitSlot,
 257                 size: layout.size.bytes() as u32,
 258                 offset: None,
 259             });
 260
 261             let uninit_place = CPlace::from_stack_slot(fx, stack_slot, ty);
 262             let uninit_val = uninit_place.to_cvalue(fx);
 263             ret.write_cvalue(fx, uninit_val);
 264         }
 265         "ctlz" | "ctlz_nonzero" => {
 266             assert_eq!(args.len(), 1);
 267             let arg = args[0].load_value(fx);
 268             let res = CValue::ByVal(fx.bcx.ins().clz(arg), args[0].layout());
 269             ret.write_cvalue(fx, res);
 270         }
 271         "cttz" | "cttz_nonzero" => {
 272             assert_eq!(args.len(), 1);
 273             let arg = args[0].load_value(fx);
 274             let res = CValue::ByVal(fx.bcx.ins().clz(arg), args[0].layout());
 275             ret.write_cvalue(fx, res);
 276         }
 277         "ctpop" => {
 278             assert_eq!(args.len(), 1);
 279             let arg = args[0].load_value(fx);
 280             let res = CValue::ByVal(fx.bcx.ins().popcnt(arg), args[0].layout());
 281             ret.write_cvalue(fx, res);
 282         }
 283         "bitreverse" => {
 284             assert_eq!(args.len(), 1);
 285             let arg = args[0].load_value(fx);
 286             let res = CValue::ByVal(fx.bcx.ins().bitrev(arg), args[0].layout());
 287             ret.write_cvalue(fx, res);
 288         }
 289         "needs_drop" => {
 290             assert_eq!(args.len(), 0);
 291             let ty = substs.type_at(0);
 292             let needs_drop = if ty.needs_drop(fx.tcx, ParamEnv::reveal_all()) {
 293                 1
 294             } else {
 295                 0
 296             };
 297             let needs_drop = CValue::const_val(fx, fx.tcx.types.bool, needs_drop);
 298             ret.write_cvalue(fx, needs_drop);
 299         }
 300         _ if intrinsic.starts_with("atomic_fence") => {}
 301         _ if intrinsic.starts_with("atomic_singlethreadfence") => {}
 302         _ if intrinsic.starts_with("atomic_load") => {
 303             assert_eq!(args.len(), 1);
 304             let inner_layout =
 305                 fx.layout_of(args[0].layout().ty.builtin_deref(true).unwrap().ty);
 306             let val = CValue::ByRef(args[0].load_value(fx), inner_layout);
 307             ret.write_cvalue(fx, val);
 308         }
 309         _ if intrinsic.starts_with("atomic_store") => {
 310             assert_eq!(args.len(), 2);
 311             let dest = CPlace::Addr(args[0].load_value(fx), None, args[1].layout());
 312             dest.write_cvalue(fx, args[1]);
 313         }
 314         _ if intrinsic.starts_with("atomic_xadd") => {
 315             assert_eq!(args.len(), 2);
 316             let clif_ty = fx.cton_type(substs.type_at(0)).unwrap();
 317             let ptr = args[0].load_value(fx);
 318             let amount = args[1].load_value(fx);
 319             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 320             let new = fx.bcx.ins().iadd(old, amount);
 321             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 322             ret.write_cvalue(fx, CValue::ByVal(old, fx.layout_of(substs.type_at(0))));
 323         }
 324         _ if intrinsic.starts_with("atomic_xsub") => {
 325             assert_eq!(args.len(), 2);
 326             let clif_ty = fx.cton_type(substs.type_at(0)).unwrap();
 327             let ptr = args[0].load_value(fx);
 328             let amount = args[1].load_value(fx);
 329             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 330             let new = fx.bcx.ins().isub(old, amount);
 331             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 332             ret.write_cvalue(fx, CValue::ByVal(old, fx.layout_of(substs.type_at(0))));
 333         }
 334         _ => unimpl!("unsupported intrinsic {}", intrinsic),
 335     }
 336
 337     if let Some((_, dest)) = destination {
 338         let ret_ebb = fx.get_ebb(dest);
 339         fx.bcx.ins().jump(ret_ebb, &[]);
 340     } else {
 341         fx.bcx.ins().trap(TrapCode::User(!0));
 342     }
 343 }