src/intrinsics.rs

   1 use crate::prelude::*;
   2
   3 use rustc::ty::subst::SubstsRef;
   4
   5 macro intrinsic_pat {
   6     (_) => {
   7         _
   8     },
   9     ($name:ident) => {
  10         stringify!($name)
  11     },
  12     ($name:literal) => {
  13         stringify!($name)
  14     },
  15     ($x:ident . $($xs:tt).*) => {
  16         concat!(stringify!($x), ".", intrinsic_pat!($($xs).*))
  17     }
  18 }
  19
  20 macro intrinsic_arg {
  21     (o $fx:expr, $arg:ident) => {
  22         $arg
  23     },
  24     (c $fx:expr, $arg:ident) => {
  25         trans_operand($fx, $arg)
  26     },
  27     (v $fx:expr, $arg:ident) => {
  28         trans_operand($fx, $arg).load_scalar($fx)
  29     }
  30 }
  31
  32 macro intrinsic_substs {
  33     ($substs:expr, $index:expr,) => {},
  34     ($substs:expr, $index:expr, $first:ident $(,$rest:ident)*) => {
  35         let $first = $substs.type_at($index);
  36         intrinsic_substs!($substs, $index+1, $($rest),*);
  37     }
  38 }
  39
  40 pub macro intrinsic_match {
  41     ($fx:expr, $intrinsic:expr, $substs:expr, $args:expr,
  42     _ => $unknown:block;
  43     $(
  44         $($($name:tt).*)|+ $(if $cond:expr)?, $(<$($subst:ident),*>)? ($($a:ident $arg:ident),*) $content:block;
  45     )*) => {
  46         match $intrinsic {
  47             $(
  48                 $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
  49                     #[allow(unused_parens, non_snake_case)]
  50                     {
  51                         $(
  52                             intrinsic_substs!($substs, 0, $($subst),*);
  53                         )?
  54                         if let [$($arg),*] = $args {
  55                             let ($($arg,)*) = (
  56                                 $(intrinsic_arg!($a $fx, $arg),)*
  57                             );
  58                             #[warn(unused_parens, non_snake_case)]
  59                             {
  60                                 $content
  61                             }
  62                         } else {
  63                             bug!("wrong number of args for intrinsic {:?}", $intrinsic);
  64                         }
  65                     }
  66                 }
  67             )*
  68             _ => $unknown,
  69         }
  70     }
  71 }
  72
  73 macro_rules! call_intrinsic_match {
  74     ($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $destination:expr, $args:expr, $(
  75         $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
  76     )*) => {
  77         match $intrinsic {
  78             $(
  79                 stringify!($name) => {
  80                     assert!($substs.is_noop());
  81                     if let [$(ref $arg),*] = *$args {
  82                         let ($($arg,)*) = (
  83                             $(trans_operand($fx, $arg),)*
  84                         );
  85                         let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty);
  86                         $ret.write_cvalue($fx, res);
  87
  88                         if let Some((_, dest)) = $destination {
  89                             let ret_ebb = $fx.get_ebb(dest);
  90                             $fx.bcx.ins().jump(ret_ebb, &[]);
  91                             return;
  92                         } else {
  93                             unreachable!();
  94                         }
  95                     } else {
  96                         bug!("wrong number of args for intrinsic {:?}", $intrinsic);
  97                     }
  98                 }
  99             )*
 100             _ => {}
 101         }
 102     }
 103 }
 104
 105 macro_rules! atomic_binop_return_old {
 106     ($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) => {
 107         let clif_ty = $fx.clif_type($T).unwrap();
 108         let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 109         let new = $fx.bcx.ins().$op(old, $src);
 110         $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 111         $ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
 112     };
 113 }
 114
 115 macro_rules! atomic_minmax {
 116     ($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) => {
 117         // Read old
 118         let clif_ty = $fx.clif_type($T).unwrap();
 119         let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 120
 121         // Compare
 122         let is_eq = $fx.bcx.ins().icmp(IntCC::SignedGreaterThan, old, $src);
 123         let new = crate::common::codegen_select(&mut $fx.bcx, is_eq, old, $src);
 124
 125         // Write new
 126         $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 127
 128         let ret_val = CValue::by_val(old, $ret.layout());
 129         $ret.write_cvalue($fx, ret_val);
 130     };
 131 }
 132
 133 pub fn lane_type_and_count<'tcx>(
 134     fx: &FunctionCx<'_, 'tcx, impl Backend>,
 135     layout: TyLayout<'tcx>,
 136     intrinsic: &str,
 137 ) -> (TyLayout<'tcx>, u32) {
 138     assert!(layout.ty.is_simd());
 139     let lane_count = match layout.fields {
 140         layout::FieldPlacement::Array { stride: _, count } => u32::try_from(count).unwrap(),
 141         _ => panic!("Non vector type {:?} passed to or returned from simd_* intrinsic {}", layout.ty, intrinsic),
 142     };
 143     let lane_layout = layout.field(fx, 0);
 144     (lane_layout, lane_count)
 145 }
 146
 147 pub fn simd_for_each_lane<'tcx, B: Backend>(
 148     fx: &mut FunctionCx<'_, 'tcx, B>,
 149     intrinsic: &str,
 150     x: CValue<'tcx>,
 151     y: CValue<'tcx>,
 152     ret: CPlace<'tcx>,
 153     f: impl Fn(&mut FunctionCx<'_, 'tcx, B>, TyLayout<'tcx>, TyLayout<'tcx>, Value, Value) -> CValue<'tcx>,
 154 ) {
 155     assert_eq!(x.layout(), y.layout());
 156     let layout = x.layout();
 157
 158     let (lane_layout, lane_count) = lane_type_and_count(fx, layout, intrinsic);
 159     let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx, ret.layout(), intrinsic);
 160     assert_eq!(lane_count, ret_lane_count);
 161
 162     for lane in 0..lane_count {
 163         let lane = mir::Field::new(lane.try_into().unwrap());
 164         let x_lane = x.value_field(fx, lane).load_scalar(fx);
 165         let y_lane = y.value_field(fx, lane).load_scalar(fx);
 166
 167         let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane);
 168
 169         ret.place_field(fx, lane).write_cvalue(fx, res_lane);
 170     }
 171 }
 172
 173 pub fn bool_to_zero_or_max_uint<'tcx>(
 174     fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
 175     layout: TyLayout<'tcx>,
 176     val: Value,
 177 ) -> CValue<'tcx> {
 178     let ty = fx.clif_type(layout.ty).unwrap();
 179
 180     let int_ty = match ty {
 181         types::F32 => types::I32,
 182         types::F64 => types::I64,
 183         ty => ty,
 184     };
 185
 186     let zero = fx.bcx.ins().iconst(int_ty, 0);
 187     let max = fx.bcx.ins().iconst(int_ty, (u64::max_value() >> (64 - int_ty.bits())) as i64);
 188     let mut res = crate::common::codegen_select(&mut fx.bcx, val, max, zero);
 189
 190     if ty.is_float() {
 191         res = fx.bcx.ins().bitcast(ty, res);
 192     }
 193
 194     CValue::by_val(res, layout)
 195 }
 196
 197 macro_rules! simd_cmp {
 198     ($fx:expr, $intrinsic:expr, $cc:ident($x:ident, $y:ident) -> $ret:ident) => {
 199         simd_for_each_lane($fx, $intrinsic, $x, $y, $ret, |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 200             let res_lane = match lane_layout.ty.sty {
 201                 ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane),
 202                 _ => unreachable!("{:?}", lane_layout.ty),
 203             };
 204             bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 205         });
 206     };
 207     ($fx:expr, $intrinsic:expr, $cc_u:ident|$cc_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 208         simd_for_each_lane($fx, $intrinsic, $x, $y, $ret, |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 209             let res_lane = match lane_layout.ty.sty {
 210                 ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
 211                 ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
 212                 _ => unreachable!("{:?}", lane_layout.ty),
 213             };
 214             bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 215         });
 216     };
 217
 218 }
 219
 220 macro_rules! simd_int_binop {
 221     ($fx:expr, $intrinsic:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
 222         simd_for_each_lane($fx, $intrinsic, $x, $y, $ret, |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 223             let res_lane = match lane_layout.ty.sty {
 224                 ty::Uint(_) | ty::Int(_) => fx.bcx.ins().$op(x_lane, y_lane),
 225                 _ => unreachable!("{:?}", lane_layout.ty),
 226             };
 227             CValue::by_val(res_lane, ret_lane_layout)
 228         });
 229     };
 230     ($fx:expr, $intrinsic:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 231         simd_for_each_lane($fx, $intrinsic, $x, $y, $ret, |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 232             let res_lane = match lane_layout.ty.sty {
 233                 ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 234                 ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 235                 _ => unreachable!("{:?}", lane_layout.ty),
 236             };
 237             CValue::by_val(res_lane, ret_lane_layout)
 238         });
 239     };
 240 }
 241
 242 macro_rules! simd_int_flt_binop {
 243     ($fx:expr, $intrinsic:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 244         simd_for_each_lane($fx, $intrinsic, $x, $y, $ret, |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 245             let res_lane = match lane_layout.ty.sty {
 246                 ty::Uint(_) | ty::Int(_) => fx.bcx.ins().$op(x_lane, y_lane),
 247                 ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
 248                 _ => unreachable!("{:?}", lane_layout.ty),
 249             };
 250             CValue::by_val(res_lane, ret_lane_layout)
 251         });
 252     };
 253     ($fx:expr, $intrinsic:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 254         simd_for_each_lane($fx, $intrinsic, $x, $y, $ret, |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 255             let res_lane = match lane_layout.ty.sty {
 256                 ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 257                 ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 258                 ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
 259                 _ => unreachable!("{:?}", lane_layout.ty),
 260             };
 261             CValue::by_val(res_lane, ret_lane_layout)
 262         });
 263     };
 264 }
 265
 266 macro_rules! simd_flt_binop {
 267     ($fx:expr, $intrinsic:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
 268         simd_for_each_lane($fx, $intrinsic, $x, $y, $ret, |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 269             let res_lane = match lane_layout.ty.sty {
 270                 ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
 271                 _ => unreachable!("{:?}", lane_layout.ty),
 272             };
 273             CValue::by_val(res_lane, ret_lane_layout)
 274         });
 275     }
 276 }
 277
 278 pub fn codegen_intrinsic_call<'a, 'tcx: 'a>(
 279     fx: &mut FunctionCx<'a, 'tcx, impl Backend>,
 280     def_id: DefId,
 281     substs: SubstsRef<'tcx>,
 282     args: &[mir::Operand<'tcx>],
 283     destination: Option<(CPlace<'tcx>, BasicBlock)>,
 284 ) {
 285     let intrinsic = fx.tcx.item_name(def_id).as_str();
 286     let intrinsic = &intrinsic[..];
 287
 288     let ret = match destination {
 289         Some((place, _)) => place,
 290         None => {
 291             // Insert non returning intrinsics here
 292             match intrinsic {
 293                 "abort" => {
 294                     trap_panic(fx, "Called intrinsic::abort.");
 295                 }
 296                 "unreachable" => {
 297                     trap_unreachable(fx, "[corruption] Called intrinsic::unreachable.");
 298                 }
 299                 _ => unimplemented!("unsupported instrinsic {}", intrinsic),
 300             }
 301             return;
 302         }
 303     };
 304
 305     let u64_layout = fx.layout_of(fx.tcx.types.u64);
 306     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 307
 308     call_intrinsic_match! {
 309         fx, intrinsic, substs, ret, destination, args,
 310         expf32(flt) -> f32 => expf,
 311         expf64(flt) -> f64 => exp,
 312         exp2f32(flt) -> f32 => exp2f,
 313         exp2f64(flt) -> f64 => exp2,
 314         sqrtf32(flt) -> f32 => sqrtf,
 315         sqrtf64(flt) -> f64 => sqrt,
 316         powif32(a, x) -> f32 => __powisf2, // compiler-builtins
 317         powif64(a, x) -> f64 => __powidf2, // compiler-builtins
 318         logf32(flt) -> f32 => logf,
 319         logf64(flt) -> f64 => log,
 320         fabsf32(flt) -> f32 => fabsf,
 321         fabsf64(flt) -> f64 => fabs,
 322         fmaf32(x, y, z) -> f32 => fmaf,
 323         fmaf64(x, y, z) -> f64 => fma,
 324
 325         // rounding variants
 326         floorf32(flt) -> f32 => floorf,
 327         floorf64(flt) -> f64 => floor,
 328         ceilf32(flt) -> f32 => ceilf,
 329         ceilf64(flt) -> f64 => ceil,
 330         truncf32(flt) -> f32 => truncf,
 331         truncf64(flt) -> f64 => trunc,
 332         roundf32(flt) -> f32 => roundf,
 333         roundf64(flt) -> f64 => round,
 334
 335         // trigonometry
 336         sinf32(flt) -> f32 => sinf,
 337         sinf64(flt) -> f64 => sin,
 338         cosf32(flt) -> f32 => cosf,
 339         cosf64(flt) -> f64 => cos,
 340         tanf32(flt) -> f32 => tanf,
 341         tanf64(flt) -> f64 => tan,
 342     }
 343
 344     intrinsic_match! {
 345         fx, intrinsic, substs, args,
 346         _ => {
 347             unimpl!("unsupported intrinsic {}", intrinsic)
 348         };
 349
 350         assume, (c _a) {};
 351         likely | unlikely, (c a) {
 352             ret.write_cvalue(fx, a);
 353         };
 354         breakpoint, () {
 355             fx.bcx.ins().debugtrap();
 356         };
 357         copy | copy_nonoverlapping, <elem_ty> (v src, v dst, v count) {
 358             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 359             let elem_size = fx
 360                 .bcx
 361                 .ins()
 362                 .iconst(fx.pointer_type, elem_size as i64);
 363             assert_eq!(args.len(), 3);
 364             let byte_amount = fx.bcx.ins().imul(count, elem_size);
 365
 366             if intrinsic.ends_with("_nonoverlapping") {
 367                 fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
 368             } else {
 369                 fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
 370             }
 371         };
 372         discriminant_value, (c val) {
 373             let pointee_layout = fx.layout_of(val.layout().ty.builtin_deref(true).unwrap().ty);
 374             let place = CPlace::for_addr(val.load_scalar(fx), pointee_layout);
 375             let discr = crate::base::trans_get_discriminant(fx, place, ret.layout());
 376             ret.write_cvalue(fx, discr);
 377         };
 378         size_of, <T> () {
 379             let size_of = fx.layout_of(T).size.bytes();
 380             let size_of = CValue::const_val(fx, usize_layout.ty, size_of.into());
 381             ret.write_cvalue(fx, size_of);
 382         };
 383         size_of_val, <T> (c ptr) {
 384             let layout = fx.layout_of(T);
 385             let size = if layout.is_unsized() {
 386                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 387                 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout.ty, info);
 388                 size
 389             } else {
 390                 fx
 391                     .bcx
 392                     .ins()
 393                     .iconst(fx.pointer_type, layout.size.bytes() as i64)
 394             };
 395             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 396         };
 397         min_align_of, <T> () {
 398             let min_align = fx.layout_of(T).align.abi.bytes();
 399             let min_align = CValue::const_val(fx, usize_layout.ty, min_align.into());
 400             ret.write_cvalue(fx, min_align);
 401         };
 402         min_align_of_val, <T> (c ptr) {
 403             let layout = fx.layout_of(T);
 404             let align = if layout.is_unsized() {
 405                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 406                 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout.ty, info);
 407                 align
 408             } else {
 409                 fx
 410                     .bcx
 411                     .ins()
 412                     .iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
 413             };
 414             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 415         };
 416         pref_align_of, <T> () {
 417             let pref_align = fx.layout_of(T).align.pref.bytes();
 418             let pref_align = CValue::const_val(fx, usize_layout.ty, pref_align.into());
 419             ret.write_cvalue(fx, pref_align);
 420         };
 421
 422
 423         type_id, <T> () {
 424             let type_id = fx.tcx.type_id_hash(T);
 425             let type_id = CValue::const_val(fx, u64_layout.ty, type_id.into());
 426             ret.write_cvalue(fx, type_id);
 427         };
 428         type_name, <T> () {
 429             let type_name = fx.tcx.type_name(T);
 430             let type_name = crate::constant::trans_const_value(fx, type_name);
 431             ret.write_cvalue(fx, type_name);
 432         };
 433
 434         _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) {
 435             // FIXME trap on overflow
 436             let bin_op = match intrinsic {
 437                 "unchecked_sub" => BinOp::Sub,
 438                 "unchecked_div" | "exact_div" => BinOp::Div,
 439                 "unchecked_rem" => BinOp::Rem,
 440                 "unchecked_shl" => BinOp::Shl,
 441                 "unchecked_shr" => BinOp::Shr,
 442                 _ => unimplemented!("intrinsic {}", intrinsic),
 443             };
 444             let res = match ret.layout().ty.sty {
 445                 ty::Uint(_) => crate::base::trans_int_binop(
 446                     fx,
 447                     bin_op,
 448                     x,
 449                     y,
 450                     ret.layout().ty,
 451                     false,
 452                 ),
 453                 ty::Int(_) => crate::base::trans_int_binop(
 454                     fx,
 455                     bin_op,
 456                     x,
 457                     y,
 458                     ret.layout().ty,
 459                     true,
 460                 ),
 461                 _ => panic!(),
 462             };
 463             ret.write_cvalue(fx, res);
 464         };
 465         _ if intrinsic.ends_with("_with_overflow"), <T> (c x, c y) {
 466             assert_eq!(x.layout().ty, y.layout().ty);
 467             let bin_op = match intrinsic {
 468                 "add_with_overflow" => BinOp::Add,
 469                 "sub_with_overflow" => BinOp::Sub,
 470                 "mul_with_overflow" => BinOp::Mul,
 471                 _ => unimplemented!("intrinsic {}", intrinsic),
 472             };
 473             let res = match T.sty {
 474                 ty::Uint(_) => crate::base::trans_checked_int_binop(
 475                     fx,
 476                     bin_op,
 477                     x,
 478                     y,
 479                     ret.layout().ty,
 480                     false,
 481                 ),
 482                 ty::Int(_) => crate::base::trans_checked_int_binop(
 483                     fx,
 484                     bin_op,
 485                     x,
 486                     y,
 487                     ret.layout().ty,
 488                     true,
 489                 ),
 490                 _ => panic!(),
 491             };
 492             ret.write_cvalue(fx, res);
 493         };
 494         _ if intrinsic.starts_with("overflowing_"), <T> (c x, c y) {
 495             assert_eq!(x.layout().ty, y.layout().ty);
 496             let bin_op = match intrinsic {
 497                 "overflowing_add" => BinOp::Add,
 498                 "overflowing_sub" => BinOp::Sub,
 499                 "overflowing_mul" => BinOp::Mul,
 500                 _ => unimplemented!("intrinsic {}", intrinsic),
 501             };
 502             let res = match T.sty {
 503                 ty::Uint(_) => crate::base::trans_int_binop(
 504                     fx,
 505                     bin_op,
 506                     x,
 507                     y,
 508                     ret.layout().ty,
 509                     false,
 510                 ),
 511                 ty::Int(_) => crate::base::trans_int_binop(
 512                     fx,
 513                     bin_op,
 514                     x,
 515                     y,
 516                     ret.layout().ty,
 517                     true,
 518                 ),
 519                 _ => panic!(),
 520             };
 521             ret.write_cvalue(fx, res);
 522         };
 523         _ if intrinsic.starts_with("saturating_"), <T> (c x, c y) {
 524             // FIXME implement saturating behavior
 525             assert_eq!(x.layout().ty, y.layout().ty);
 526             let bin_op = match intrinsic {
 527                 "saturating_add" => BinOp::Add,
 528                 "saturating_sub" => BinOp::Sub,
 529                 "saturating_mul" => BinOp::Mul,
 530                 _ => unimplemented!("intrinsic {}", intrinsic),
 531             };
 532             let res = match T.sty {
 533                 ty::Uint(_) => crate::base::trans_int_binop(
 534                     fx,
 535                     bin_op,
 536                     x,
 537                     y,
 538                     ret.layout().ty,
 539                     false,
 540                 ),
 541                 ty::Int(_) => crate::base::trans_int_binop(
 542                     fx,
 543                     bin_op,
 544                     x,
 545                     y,
 546                     ret.layout().ty,
 547                     true,
 548                 ),
 549                 _ => panic!(),
 550             };
 551             ret.write_cvalue(fx, res);
 552         };
 553         rotate_left, <T>(v x, v y) {
 554             let layout = fx.layout_of(T);
 555             let res = fx.bcx.ins().rotl(x, y);
 556             ret.write_cvalue(fx, CValue::by_val(res, layout));
 557         };
 558         rotate_right, <T>(v x, v y) {
 559             let layout = fx.layout_of(T);
 560             let res = fx.bcx.ins().rotr(x, y);
 561             ret.write_cvalue(fx, CValue::by_val(res, layout));
 562         };
 563
 564         // The only difference between offset and arith_offset is regarding UB. Because Cranelift
 565         // doesn't have UB both are codegen'ed the same way
 566         offset | arith_offset, (c base, v offset) {
 567             let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
 568             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 569             let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
 570             let base_val = base.load_scalar(fx);
 571             let res = fx.bcx.ins().iadd(base_val, ptr_diff);
 572             ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
 573         };
 574
 575         transmute, <src_ty, dst_ty> (c from) {
 576             assert_eq!(from.layout().ty, src_ty);
 577             let addr = from.force_stack(fx);
 578             let dst_layout = fx.layout_of(dst_ty);
 579             ret.write_cvalue(fx, CValue::by_ref(addr, dst_layout))
 580         };
 581         init, () {
 582             if ret.layout().abi == Abi::Uninhabited {
 583                 crate::trap::trap_panic(fx, "[panic] Called intrinsic::init for uninhabited type.");
 584                 return;
 585             }
 586
 587             match ret {
 588                 CPlace::NoPlace(_layout) => {}
 589                 CPlace::Var(var, layout) => {
 590                     let clif_ty = fx.clif_type(layout.ty).unwrap();
 591                     let val = match clif_ty {
 592                         types::I8 | types::I16 | types::I32 | types::I64 => fx.bcx.ins().iconst(clif_ty, 0),
 593                         types::F32 => {
 594                             let zero = fx.bcx.ins().iconst(types::I32, 0);
 595                             fx.bcx.ins().bitcast(types::F32, zero)
 596                         }
 597                         types::F64 => {
 598                             let zero = fx.bcx.ins().iconst(types::I64, 0);
 599                             fx.bcx.ins().bitcast(types::F64, zero)
 600                         }
 601                         _ => panic!("clif_type returned {}", clif_ty),
 602                     };
 603                     fx.bcx.def_var(mir_var(var), val);
 604                 }
 605                 _ => {
 606                     let addr = ret.to_addr(fx);
 607                     let layout = ret.layout();
 608                     fx.bcx.emit_small_memset(fx.module.target_config(), addr, 0, layout.size.bytes(), 1);
 609                 }
 610             }
 611         };
 612         write_bytes, (c dst, v val, v count) {
 613             let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
 614             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 615             let count = fx.bcx.ins().imul_imm(count, pointee_size as i64);
 616             let dst_ptr = dst.load_scalar(fx);
 617             fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count);
 618         };
 619         ctlz | ctlz_nonzero, <T> (v arg) {
 620             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 621                 // FIXME verify this algorithm is correct
 622                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 623                 let lsb_lz = fx.bcx.ins().clz(lsb);
 624                 let msb_lz = fx.bcx.ins().clz(msb);
 625                 let msb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, msb, 0);
 626                 let lsb_lz_plus_64 = fx.bcx.ins().iadd_imm(lsb_lz, 64);
 627                 fx.bcx.ins().select(msb_is_zero, lsb_lz_plus_64, msb_lz)
 628             } else {
 629                 fx.bcx.ins().clz(arg)
 630             };
 631             let res = CValue::by_val(res, fx.layout_of(T));
 632             ret.write_cvalue(fx, res);
 633         };
 634         cttz | cttz_nonzero, <T> (v arg) {
 635             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 636                 // FIXME verify this algorithm is correct
 637                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 638                 let lsb_tz = fx.bcx.ins().ctz(lsb);
 639                 let msb_tz = fx.bcx.ins().ctz(msb);
 640                 let lsb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, lsb, 0);
 641                 let msb_tz_plus_64 = fx.bcx.ins().iadd_imm(msb_tz, 64);
 642                 fx.bcx.ins().select(lsb_is_zero, msb_tz_plus_64, lsb_tz)
 643             } else {
 644                 fx.bcx.ins().ctz(arg)
 645             };
 646             let res = CValue::by_val(res, fx.layout_of(T));
 647             ret.write_cvalue(fx, res);
 648         };
 649         ctpop, <T> (v arg) {
 650             let res = CValue::by_val(fx.bcx.ins().popcnt(arg), fx.layout_of(T));
 651             ret.write_cvalue(fx, res);
 652         };
 653         bitreverse, <T> (v arg) {
 654             let res = CValue::by_val(fx.bcx.ins().bitrev(arg), fx.layout_of(T));
 655             ret.write_cvalue(fx, res);
 656         };
 657         bswap, <T> (v arg) {
 658             // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
 659             fn swap(bcx: &mut FunctionBuilder, v: Value) -> Value {
 660                 match bcx.func.dfg.value_type(v) {
 661                     types::I8 => v,
 662
 663                     // https://code.woboq.org/gcc/include/bits/byteswap.h.html
 664                     types::I16 => {
 665                         let tmp1 = bcx.ins().ishl_imm(v, 8);
 666                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00);
 667
 668                         let tmp2 = bcx.ins().ushr_imm(v, 8);
 669                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF);
 670
 671                         bcx.ins().bor(n1, n2)
 672                     }
 673                     types::I32 => {
 674                         let tmp1 = bcx.ins().ishl_imm(v, 24);
 675                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000);
 676
 677                         let tmp2 = bcx.ins().ishl_imm(v, 8);
 678                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000);
 679
 680                         let tmp3 = bcx.ins().ushr_imm(v, 8);
 681                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00);
 682
 683                         let tmp4 = bcx.ins().ushr_imm(v, 24);
 684                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF);
 685
 686                         let or_tmp1 = bcx.ins().bor(n1, n2);
 687                         let or_tmp2 = bcx.ins().bor(n3, n4);
 688                         bcx.ins().bor(or_tmp1, or_tmp2)
 689                     }
 690                     types::I64 => {
 691                         let tmp1 = bcx.ins().ishl_imm(v, 56);
 692                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64);
 693
 694                         let tmp2 = bcx.ins().ishl_imm(v, 40);
 695                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64);
 696
 697                         let tmp3 = bcx.ins().ishl_imm(v, 24);
 698                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64);
 699
 700                         let tmp4 = bcx.ins().ishl_imm(v, 8);
 701                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64);
 702
 703                         let tmp5 = bcx.ins().ushr_imm(v, 8);
 704                         let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64);
 705
 706                         let tmp6 = bcx.ins().ushr_imm(v, 24);
 707                         let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64);
 708
 709                         let tmp7 = bcx.ins().ushr_imm(v, 40);
 710                         let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64);
 711
 712                         let tmp8 = bcx.ins().ushr_imm(v, 56);
 713                         let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64);
 714
 715                         let or_tmp1 = bcx.ins().bor(n1, n2);
 716                         let or_tmp2 = bcx.ins().bor(n3, n4);
 717                         let or_tmp3 = bcx.ins().bor(n5, n6);
 718                         let or_tmp4 = bcx.ins().bor(n7, n8);
 719
 720                         let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2);
 721                         let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4);
 722                         bcx.ins().bor(or_tmp5, or_tmp6)
 723                     }
 724                     types::I128 => {
 725                         let (lo, hi) = bcx.ins().isplit(v);
 726                         let lo = swap(bcx, lo);
 727                         let hi = swap(bcx, hi);
 728                         bcx.ins().iconcat(hi, lo)
 729                     }
 730                     ty => unimplemented!("bswap {}", ty),
 731                 }
 732             };
 733             let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T));
 734             ret.write_cvalue(fx, res);
 735         };
 736         needs_drop, <T> () {
 737             let needs_drop = if T.needs_drop(fx.tcx, ParamEnv::reveal_all()) {
 738                 1
 739             } else {
 740                 0
 741             };
 742             let needs_drop = CValue::const_val(fx, fx.tcx.types.bool, needs_drop);
 743             ret.write_cvalue(fx, needs_drop);
 744         };
 745         panic_if_uninhabited, <T> () {
 746             if fx.layout_of(T).abi.is_uninhabited() {
 747                 crate::trap::trap_panic(fx, "[panic] Called intrinsic::panic_if_uninhabited for uninhabited type.");
 748                 return;
 749             }
 750         };
 751
 752         volatile_load, (c ptr) {
 753             // Cranelift treats loads as volatile by default
 754             let inner_layout =
 755                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 756             let val = CValue::by_ref(ptr.load_scalar(fx), inner_layout);
 757             ret.write_cvalue(fx, val);
 758         };
 759         volatile_store, (v ptr, c val) {
 760             // Cranelift treats stores as volatile by default
 761             let dest = CPlace::for_addr(ptr, val.layout());
 762             dest.write_cvalue(fx, val);
 763         };
 764
 765         _ if intrinsic.starts_with("atomic_fence"), () {};
 766         _ if intrinsic.starts_with("atomic_singlethreadfence"), () {};
 767         _ if intrinsic.starts_with("atomic_load"), (c ptr) {
 768             let inner_layout =
 769                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 770             let val = CValue::by_ref(ptr.load_scalar(fx), inner_layout);
 771             ret.write_cvalue(fx, val);
 772         };
 773         _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
 774             let dest = CPlace::for_addr(ptr, val.layout());
 775             dest.write_cvalue(fx, val);
 776         };
 777         _ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
 778             // Read old
 779             let clif_ty = fx.clif_type(T).unwrap();
 780             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 781             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
 782
 783             // Write new
 784             let dest = CPlace::for_addr(ptr, src.layout());
 785             dest.write_cvalue(fx, src);
 786         };
 787         _ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, v test_old, v new) { // both atomic_cxchg_* and atomic_cxchgweak_*
 788             // Read old
 789             let clif_ty = fx.clif_type(T).unwrap();
 790             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 791
 792             // Compare
 793             let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
 794             let new = crate::common::codegen_select(&mut fx.bcx, is_eq, new, old); // Keep old if not equal to test_old
 795
 796             // Write new
 797             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 798
 799             let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
 800             ret.write_cvalue(fx, ret_val);
 801         };
 802
 803         _ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, v amount) {
 804             atomic_binop_return_old! (fx, iadd<T>(ptr, amount) -> ret);
 805         };
 806         _ if intrinsic.starts_with("atomic_xsub"), <T> (v ptr, v amount) {
 807             atomic_binop_return_old! (fx, isub<T>(ptr, amount) -> ret);
 808         };
 809         _ if intrinsic.starts_with("atomic_and"), <T> (v ptr, v src) {
 810             atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
 811         };
 812         _ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, v src) {
 813             atomic_binop_return_old! (fx, band_not<T>(ptr, src) -> ret);
 814         };
 815         _ if intrinsic.starts_with("atomic_or"), <T> (v ptr, v src) {
 816             atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);
 817         };
 818         _ if intrinsic.starts_with("atomic_xor"), <T> (v ptr, v src) {
 819             atomic_binop_return_old! (fx, bxor<T>(ptr, src) -> ret);
 820         };
 821
 822         _ if intrinsic.starts_with("atomic_max"), <T> (v ptr, v src) {
 823             atomic_minmax!(fx, IntCC::SignedGreaterThan, <T> (ptr, src) -> ret);
 824         };
 825         _ if intrinsic.starts_with("atomic_umax"), <T> (v ptr, v src) {
 826             atomic_minmax!(fx, IntCC::UnsignedGreaterThan, <T> (ptr, src) -> ret);
 827         };
 828         _ if intrinsic.starts_with("atomic_min"), <T> (v ptr, v src) {
 829             atomic_minmax!(fx, IntCC::SignedLessThan, <T> (ptr, src) -> ret);
 830         };
 831         _ if intrinsic.starts_with("atomic_umin"), <T> (v ptr, v src) {
 832             atomic_minmax!(fx, IntCC::UnsignedLessThan, <T> (ptr, src) -> ret);
 833         };
 834
 835         minnumf32, (v a, v b) {
 836             let val = fx.bcx.ins().fmin(a, b);
 837             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
 838             ret.write_cvalue(fx, val);
 839         };
 840         minnumf64, (v a, v b) {
 841             let val = fx.bcx.ins().fmin(a, b);
 842             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
 843             ret.write_cvalue(fx, val);
 844         };
 845         maxnumf32, (v a, v b) {
 846             let val = fx.bcx.ins().fmax(a, b);
 847             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
 848             ret.write_cvalue(fx, val);
 849         };
 850         maxnumf64, (v a, v b) {
 851             let val = fx.bcx.ins().fmax(a, b);
 852             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
 853             ret.write_cvalue(fx, val);
 854         };
 855
 856         simd_cast, (c x) {
 857             ret.write_cvalue(fx, x.unchecked_cast_to(ret.layout()));
 858         };
 859
 860         simd_eq, (c x, c y) {
 861             simd_cmp!(fx, intrinsic, Equal(x, y) -> ret);
 862         };
 863         simd_ne, (c x, c y) {
 864             simd_cmp!(fx, intrinsic, NotEqual(x, y) -> ret);
 865         };
 866         simd_lt, (c x, c y) {
 867             simd_cmp!(fx, intrinsic, UnsignedLessThan|SignedLessThan(x, y) -> ret);
 868         };
 869         simd_le, (c x, c y) {
 870             simd_cmp!(fx, intrinsic, UnsignedLessThanOrEqual|SignedLessThanOrEqual(x, y) -> ret);
 871         };
 872         simd_gt, (c x, c y) {
 873             simd_cmp!(fx, intrinsic, UnsignedGreaterThan|SignedGreaterThan(x, y) -> ret);
 874         };
 875         simd_ge, (c x, c y) {
 876             simd_cmp!(fx, intrinsic, UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual(x, y) -> ret);
 877         };
 878
 879         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
 880         _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) {
 881             let n: u32 = intrinsic["simd_shuffle".len()..].parse().unwrap();
 882
 883             assert_eq!(x.layout(), y.layout());
 884             let layout = x.layout();
 885
 886             let (lane_type, lane_count) = lane_type_and_count(fx, layout, intrinsic);
 887             let (ret_lane_type, ret_lane_count) = lane_type_and_count(fx, ret.layout(), intrinsic);
 888
 889             assert_eq!(lane_type, ret_lane_type);
 890             assert_eq!(n, ret_lane_count);
 891
 892             let total_len = lane_count * 2;
 893
 894             let indexes = {
 895                 use rustc::mir::interpret::*;
 896                 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
 897
 898                 let idx_bytes = match idx_const.val {
 899                     ConstValue::ByRef { align: _, offset, alloc } => {
 900                         let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
 901                         let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
 902                         alloc.get_bytes(fx, ptr, size).unwrap()
 903                     }
 904                     _ => unreachable!("{:?}", idx_const),
 905                 };
 906
 907                 (0..ret_lane_count).map(|i| {
 908                     let i = usize::try_from(i).unwrap();
 909                     let idx = rustc::mir::interpret::read_target_uint(
 910                         fx.tcx.data_layout.endian,
 911                         &idx_bytes[4*i.. 4*i + 4],
 912                     ).expect("read_target_uint");
 913                     u32::try_from(idx).expect("try_from u32")
 914                 }).collect::<Vec<u32>>()
 915             };
 916
 917             for &idx in &indexes {
 918                 assert!(idx < total_len, "idx {} out of range 0..{}", idx, total_len);
 919             }
 920
 921             for (out_idx, in_idx) in indexes.into_iter().enumerate() {
 922                 let in_lane = if in_idx < lane_count {
 923                     x.value_field(fx, mir::Field::new(in_idx.try_into().unwrap()))
 924                 } else {
 925                     y.value_field(fx, mir::Field::new((in_idx - lane_count).try_into().unwrap()))
 926                 };
 927                 let out_lane = ret.place_field(fx, mir::Field::new(out_idx));
 928                 out_lane.write_cvalue(fx, in_lane);
 929             }
 930         };
 931
 932         simd_add, (c x, c y) {
 933             simd_int_flt_binop!(fx, intrinsic, iadd|fadd(x, y) -> ret);
 934         };
 935         simd_sub, (c x, c y) {
 936             simd_int_flt_binop!(fx, intrinsic, isub|fsub(x, y) -> ret);
 937         };
 938         simd_mul, (c x, c y) {
 939             simd_int_flt_binop!(fx, intrinsic, imul|fmul(x, y) -> ret);
 940         };
 941         simd_div, (c x, c y) {
 942             simd_int_flt_binop!(fx, intrinsic, udiv|sdiv|fdiv(x, y) -> ret);
 943         };
 944         simd_shl, (c x, c y) {
 945             simd_int_binop!(fx, intrinsic, ishl(x, y) -> ret);
 946         };
 947         simd_shr, (c x, c y) {
 948             simd_int_binop!(fx, intrinsic, ushr|sshr(x, y) -> ret);
 949         };
 950         simd_and, (c x, c y) {
 951             simd_int_binop!(fx, intrinsic, band(x, y) -> ret);
 952         };
 953         simd_or, (c x, c y) {
 954             simd_int_binop!(fx, intrinsic, bor(x, y) -> ret);
 955         };
 956         simd_xor, (c x, c y) {
 957             simd_int_binop!(fx, intrinsic, bxor(x, y) -> ret);
 958         };
 959
 960         simd_fmin, (c x, c y) {
 961             simd_flt_binop!(fx, intrinsic, fmin(x, y) -> ret);
 962         };
 963         simd_fmax, (c x, c y) {
 964             simd_flt_binop!(fx, intrinsic, fmax(x, y) -> ret);
 965         };
 966     }
 967
 968     if let Some((_, dest)) = destination {
 969         let ret_ebb = fx.get_ebb(dest);
 970         fx.bcx.ins().jump(ret_ebb, &[]);
 971     } else {
 972         trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
 973     }
 974 }