src/intrinsics/mod.rs

   1 pub mod llvm;
   2
   3 use crate::prelude::*;
   4
   5 macro intrinsic_pat {
   6     (_) => {
   7         _
   8     },
   9     ($name:ident) => {
  10         stringify!($name)
  11     },
  12     ($name:literal) => {
  13         stringify!($name)
  14     },
  15     ($x:ident . $($xs:tt).*) => {
  16         concat!(stringify!($x), ".", intrinsic_pat!($($xs).*))
  17     }
  18 }
  19
  20 macro intrinsic_arg {
  21     (o $fx:expr, $arg:ident) => {
  22         $arg
  23     },
  24     (c $fx:expr, $arg:ident) => {
  25         trans_operand($fx, $arg)
  26     },
  27     (v $fx:expr, $arg:ident) => {
  28         trans_operand($fx, $arg).load_scalar($fx)
  29     }
  30 }
  31
  32 macro intrinsic_substs {
  33     ($substs:expr, $index:expr,) => {},
  34     ($substs:expr, $index:expr, $first:ident $(,$rest:ident)*) => {
  35         let $first = $substs.type_at($index);
  36         intrinsic_substs!($substs, $index+1, $($rest),*);
  37     }
  38 }
  39
  40 macro intrinsic_match {
  41     ($fx:expr, $intrinsic:expr, $substs:expr, $args:expr,
  42     _ => $unknown:block;
  43     $(
  44         $($($name:tt).*)|+ $(if $cond:expr)?, $(<$($subst:ident),*>)? ($($a:ident $arg:ident),*) $content:block;
  45     )*) => {
  46         match $intrinsic {
  47             $(
  48                 $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
  49                     #[allow(unused_parens, non_snake_case)]
  50                     {
  51                         $(
  52                             intrinsic_substs!($substs, 0, $($subst),*);
  53                         )?
  54                         if let [$($arg),*] = $args {
  55                             let ($($arg,)*) = (
  56                                 $(intrinsic_arg!($a $fx, $arg),)*
  57                             );
  58                             #[warn(unused_parens, non_snake_case)]
  59                             {
  60                                 $content
  61                             }
  62                         } else {
  63                             bug!("wrong number of args for intrinsic {:?}", $intrinsic);
  64                         }
  65                     }
  66                 }
  67             )*
  68             _ => $unknown,
  69         }
  70     }
  71 }
  72
  73 macro_rules! call_intrinsic_match {
  74     ($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $destination:expr, $args:expr, $(
  75         $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
  76     )*) => {
  77         match $intrinsic {
  78             $(
  79                 stringify!($name) => {
  80                     assert!($substs.is_noop());
  81                     if let [$(ref $arg),*] = *$args {
  82                         let ($($arg,)*) = (
  83                             $(trans_operand($fx, $arg),)*
  84                         );
  85                         let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty);
  86                         $ret.write_cvalue($fx, res);
  87
  88                         if let Some((_, dest)) = $destination {
  89                             let ret_ebb = $fx.get_ebb(dest);
  90                             $fx.bcx.ins().jump(ret_ebb, &[]);
  91                             return;
  92                         } else {
  93                             unreachable!();
  94                         }
  95                     } else {
  96                         bug!("wrong number of args for intrinsic {:?}", $intrinsic);
  97                     }
  98                 }
  99             )*
 100             _ => {}
 101         }
 102     }
 103 }
 104
 105 macro_rules! atomic_binop_return_old {
 106     ($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) => {
 107         let clif_ty = $fx.clif_type($T).unwrap();
 108         let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 109         let new = $fx.bcx.ins().$op(old, $src);
 110         $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 111         $ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
 112     };
 113 }
 114
 115 macro_rules! atomic_minmax {
 116     ($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) => {
 117         // Read old
 118         let clif_ty = $fx.clif_type($T).unwrap();
 119         let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 120
 121         // Compare
 122         let is_eq = codegen_icmp($fx, IntCC::SignedGreaterThan, old, $src);
 123         let new = $fx.bcx.ins().select(is_eq, old, $src);
 124
 125         // Write new
 126         $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 127
 128         let ret_val = CValue::by_val(old, $ret.layout());
 129         $ret.write_cvalue($fx, ret_val);
 130     };
 131 }
 132
 133 fn lane_type_and_count<'tcx>(
 134     fx: &FunctionCx<'_, 'tcx, impl Backend>,
 135     layout: TyLayout<'tcx>,
 136     intrinsic: &str,
 137 ) -> (TyLayout<'tcx>, u32) {
 138     assert!(layout.ty.is_simd());
 139     let lane_count = match layout.fields {
 140         layout::FieldPlacement::Array { stride: _, count } => u32::try_from(count).unwrap(),
 141         _ => panic!(
 142             "Non vector type {:?} passed to or returned from simd_* intrinsic {}",
 143             layout.ty, intrinsic
 144         ),
 145     };
 146     let lane_layout = layout.field(fx, 0);
 147     (lane_layout, lane_count)
 148 }
 149
 150 fn simd_for_each_lane<'tcx, B: Backend>(
 151     fx: &mut FunctionCx<'_, 'tcx, B>,
 152     intrinsic: &str,
 153     x: CValue<'tcx>,
 154     y: CValue<'tcx>,
 155     ret: CPlace<'tcx>,
 156     f: impl Fn(
 157         &mut FunctionCx<'_, 'tcx, B>,
 158         TyLayout<'tcx>,
 159         TyLayout<'tcx>,
 160         Value,
 161         Value,
 162     ) -> CValue<'tcx>,
 163 ) {
 164     assert_eq!(x.layout(), y.layout());
 165     let layout = x.layout();
 166
 167     let (lane_layout, lane_count) = lane_type_and_count(fx, layout, intrinsic);
 168     let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx, ret.layout(), intrinsic);
 169     assert_eq!(lane_count, ret_lane_count);
 170
 171     for lane in 0..lane_count {
 172         let lane = mir::Field::new(lane.try_into().unwrap());
 173         let x_lane = x.value_field(fx, lane).load_scalar(fx);
 174         let y_lane = y.value_field(fx, lane).load_scalar(fx);
 175
 176         let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane);
 177
 178         ret.place_field(fx, lane).write_cvalue(fx, res_lane);
 179     }
 180 }
 181
 182 fn bool_to_zero_or_max_uint<'tcx>(
 183     fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
 184     layout: TyLayout<'tcx>,
 185     val: Value,
 186 ) -> CValue<'tcx> {
 187     let ty = fx.clif_type(layout.ty).unwrap();
 188
 189     let int_ty = match ty {
 190         types::F32 => types::I32,
 191         types::F64 => types::I64,
 192         ty => ty,
 193     };
 194
 195     let zero = fx.bcx.ins().iconst(int_ty, 0);
 196     let max = fx
 197         .bcx
 198         .ins()
 199         .iconst(int_ty, (u64::max_value() >> (64 - int_ty.bits())) as i64);
 200     let mut res = fx.bcx.ins().select(val, max, zero);
 201
 202     if ty.is_float() {
 203         res = fx.bcx.ins().bitcast(ty, res);
 204     }
 205
 206     CValue::by_val(res, layout)
 207 }
 208
 209 macro_rules! simd_cmp {
 210     ($fx:expr, $intrinsic:expr, $cc:ident($x:ident, $y:ident) -> $ret:ident) => {
 211         simd_for_each_lane(
 212             $fx,
 213             $intrinsic,
 214             $x,
 215             $y,
 216             $ret,
 217             |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 218                 let res_lane = match lane_layout.ty.kind {
 219                     ty::Uint(_) | ty::Int(_) => codegen_icmp(fx, IntCC::$cc, x_lane, y_lane),
 220                     _ => unreachable!("{:?}", lane_layout.ty),
 221                 };
 222                 bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 223             },
 224         );
 225     };
 226     ($fx:expr, $intrinsic:expr, $cc_u:ident|$cc_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 227         simd_for_each_lane(
 228             $fx,
 229             $intrinsic,
 230             $x,
 231             $y,
 232             $ret,
 233             |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 234                 let res_lane = match lane_layout.ty.kind {
 235                     ty::Uint(_) => codegen_icmp(fx, IntCC::$cc_u, x_lane, y_lane),
 236                     ty::Int(_) => codegen_icmp(fx, IntCC::$cc_s, x_lane, y_lane),
 237                     _ => unreachable!("{:?}", lane_layout.ty),
 238                 };
 239                 bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 240             },
 241         );
 242     };
 243 }
 244
 245 macro_rules! simd_int_binop {
 246     ($fx:expr, $intrinsic:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
 247         simd_for_each_lane(
 248             $fx,
 249             $intrinsic,
 250             $x,
 251             $y,
 252             $ret,
 253             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 254                 let res_lane = match lane_layout.ty.kind {
 255                     ty::Uint(_) | ty::Int(_) => fx.bcx.ins().$op(x_lane, y_lane),
 256                     _ => unreachable!("{:?}", lane_layout.ty),
 257                 };
 258                 CValue::by_val(res_lane, ret_lane_layout)
 259             },
 260         );
 261     };
 262     ($fx:expr, $intrinsic:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 263         simd_for_each_lane(
 264             $fx,
 265             $intrinsic,
 266             $x,
 267             $y,
 268             $ret,
 269             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 270                 let res_lane = match lane_layout.ty.kind {
 271                     ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 272                     ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 273                     _ => unreachable!("{:?}", lane_layout.ty),
 274                 };
 275                 CValue::by_val(res_lane, ret_lane_layout)
 276             },
 277         );
 278     };
 279 }
 280
 281 macro_rules! simd_int_flt_binop {
 282     ($fx:expr, $intrinsic:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 283         simd_for_each_lane(
 284             $fx,
 285             $intrinsic,
 286             $x,
 287             $y,
 288             $ret,
 289             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 290                 let res_lane = match lane_layout.ty.kind {
 291                     ty::Uint(_) | ty::Int(_) => fx.bcx.ins().$op(x_lane, y_lane),
 292                     ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
 293                     _ => unreachable!("{:?}", lane_layout.ty),
 294                 };
 295                 CValue::by_val(res_lane, ret_lane_layout)
 296             },
 297         );
 298     };
 299     ($fx:expr, $intrinsic:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 300         simd_for_each_lane(
 301             $fx,
 302             $intrinsic,
 303             $x,
 304             $y,
 305             $ret,
 306             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 307                 let res_lane = match lane_layout.ty.kind {
 308                     ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 309                     ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 310                     ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
 311                     _ => unreachable!("{:?}", lane_layout.ty),
 312                 };
 313                 CValue::by_val(res_lane, ret_lane_layout)
 314             },
 315         );
 316     };
 317 }
 318
 319 macro_rules! simd_flt_binop {
 320     ($fx:expr, $intrinsic:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
 321         simd_for_each_lane(
 322             $fx,
 323             $intrinsic,
 324             $x,
 325             $y,
 326             $ret,
 327             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 328                 let res_lane = match lane_layout.ty.kind {
 329                     ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
 330                     _ => unreachable!("{:?}", lane_layout.ty),
 331                 };
 332                 CValue::by_val(res_lane, ret_lane_layout)
 333             },
 334         );
 335     };
 336 }
 337
 338 pub fn codegen_intrinsic_call<'tcx>(
 339     fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
 340     instance: Instance<'tcx>,
 341     args: &[mir::Operand<'tcx>],
 342     destination: Option<(CPlace<'tcx>, BasicBlock)>,
 343     span: Span,
 344 ) {
 345     let def_id = instance.def_id();
 346     let substs = instance.substs;
 347
 348     let intrinsic = fx.tcx.item_name(def_id).as_str();
 349     let intrinsic = &intrinsic[..];
 350
 351     let ret = match destination {
 352         Some((place, _)) => place,
 353         None => {
 354             // Insert non returning intrinsics here
 355             match intrinsic {
 356                 "abort" => {
 357                     trap_panic(fx, "Called intrinsic::abort.");
 358                 }
 359                 "unreachable" => {
 360                     trap_unreachable(fx, "[corruption] Called intrinsic::unreachable.");
 361                 }
 362                 "transmute" => {
 363                     trap_unreachable(
 364                         fx,
 365                         "[corruption] Called intrinsic::transmute with uninhabited argument.",
 366                     );
 367                 }
 368                 _ => unimplemented!("unsupported instrinsic {}", intrinsic),
 369             }
 370             return;
 371         }
 372     };
 373
 374     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 375
 376     call_intrinsic_match! {
 377         fx, intrinsic, substs, ret, destination, args,
 378         expf32(flt) -> f32 => expf,
 379         expf64(flt) -> f64 => exp,
 380         exp2f32(flt) -> f32 => exp2f,
 381         exp2f64(flt) -> f64 => exp2,
 382         sqrtf32(flt) -> f32 => sqrtf,
 383         sqrtf64(flt) -> f64 => sqrt,
 384         powif32(a, x) -> f32 => __powisf2, // compiler-builtins
 385         powif64(a, x) -> f64 => __powidf2, // compiler-builtins
 386         powf32(a, x) -> f32 => powf,
 387         powf64(a, x) -> f64 => pow,
 388         logf32(flt) -> f32 => logf,
 389         logf64(flt) -> f64 => log,
 390         log2f32(flt) -> f32 => log2f,
 391         log2f64(flt) -> f64 => log2,
 392         log10f32(flt) -> f32 => log10f,
 393         log10f64(flt) -> f64 => log10,
 394         fabsf32(flt) -> f32 => fabsf,
 395         fabsf64(flt) -> f64 => fabs,
 396         fmaf32(x, y, z) -> f32 => fmaf,
 397         fmaf64(x, y, z) -> f64 => fma,
 398         copysignf32(x, y) -> f32 => copysignf,
 399         copysignf64(x, y) -> f64 => copysign,
 400
 401         // rounding variants
 402         // FIXME use clif insts
 403         floorf32(flt) -> f32 => floorf,
 404         floorf64(flt) -> f64 => floor,
 405         ceilf32(flt) -> f32 => ceilf,
 406         ceilf64(flt) -> f64 => ceil,
 407         truncf32(flt) -> f32 => truncf,
 408         truncf64(flt) -> f64 => trunc,
 409         roundf32(flt) -> f32 => roundf,
 410         roundf64(flt) -> f64 => round,
 411
 412         // trigonometry
 413         sinf32(flt) -> f32 => sinf,
 414         sinf64(flt) -> f64 => sin,
 415         cosf32(flt) -> f32 => cosf,
 416         cosf64(flt) -> f64 => cos,
 417         tanf32(flt) -> f32 => tanf,
 418         tanf64(flt) -> f64 => tan,
 419     }
 420
 421     intrinsic_match! {
 422         fx, intrinsic, substs, args,
 423         _ => {
 424             unimpl!("unsupported intrinsic {}", intrinsic)
 425         };
 426
 427         assume, (c _a) {};
 428         likely | unlikely, (c a) {
 429             ret.write_cvalue(fx, a);
 430         };
 431         breakpoint, () {
 432             fx.bcx.ins().debugtrap();
 433         };
 434         copy | copy_nonoverlapping, <elem_ty> (v src, v dst, v count) {
 435             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 436             let elem_size = fx
 437                 .bcx
 438                 .ins()
 439                 .iconst(fx.pointer_type, elem_size as i64);
 440             assert_eq!(args.len(), 3);
 441             let byte_amount = fx.bcx.ins().imul(count, elem_size);
 442
 443             if intrinsic.ends_with("_nonoverlapping") {
 444                 fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
 445             } else {
 446                 fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
 447             }
 448         };
 449         discriminant_value, (c ptr) {
 450             let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 451             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout);
 452             let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout());
 453             ret.write_cvalue(fx, discr);
 454         };
 455         size_of_val, <T> (c ptr) {
 456             let layout = fx.layout_of(T);
 457             let size = if layout.is_unsized() {
 458                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 459                 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout.ty, info);
 460                 size
 461             } else {
 462                 fx
 463                     .bcx
 464                     .ins()
 465                     .iconst(fx.pointer_type, layout.size.bytes() as i64)
 466             };
 467             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 468         };
 469         min_align_of_val, <T> (c ptr) {
 470             let layout = fx.layout_of(T);
 471             let align = if layout.is_unsized() {
 472                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 473                 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout.ty, info);
 474                 align
 475             } else {
 476                 fx
 477                     .bcx
 478                     .ins()
 479                     .iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
 480             };
 481             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 482         };
 483
 484         _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) {
 485             // FIXME trap on overflow
 486             let bin_op = match intrinsic {
 487                 "unchecked_sub" => BinOp::Sub,
 488                 "unchecked_div" | "exact_div" => BinOp::Div,
 489                 "unchecked_rem" => BinOp::Rem,
 490                 "unchecked_shl" => BinOp::Shl,
 491                 "unchecked_shr" => BinOp::Shr,
 492                 _ => unimplemented!("intrinsic {}", intrinsic),
 493             };
 494             let res = crate::num::trans_int_binop(fx, bin_op, x, y);
 495             ret.write_cvalue(fx, res);
 496         };
 497         _ if intrinsic.ends_with("_with_overflow"), (c x, c y) {
 498             assert_eq!(x.layout().ty, y.layout().ty);
 499             let bin_op = match intrinsic {
 500                 "add_with_overflow" => BinOp::Add,
 501                 "sub_with_overflow" => BinOp::Sub,
 502                 "mul_with_overflow" => BinOp::Mul,
 503                 _ => unimplemented!("intrinsic {}", intrinsic),
 504             };
 505
 506             let res = crate::num::trans_checked_int_binop(
 507                 fx,
 508                 bin_op,
 509                 x,
 510                 y,
 511             );
 512             ret.write_cvalue(fx, res);
 513         };
 514         _ if intrinsic.starts_with("wrapping_"), (c x, c y) {
 515             assert_eq!(x.layout().ty, y.layout().ty);
 516             let bin_op = match intrinsic {
 517                 "wrapping_add" => BinOp::Add,
 518                 "wrapping_sub" => BinOp::Sub,
 519                 "wrapping_mul" => BinOp::Mul,
 520                 _ => unimplemented!("intrinsic {}", intrinsic),
 521             };
 522             let res = crate::num::trans_int_binop(
 523                 fx,
 524                 bin_op,
 525                 x,
 526                 y,
 527             );
 528             ret.write_cvalue(fx, res);
 529         };
 530         _ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) {
 531             assert_eq!(lhs.layout().ty, rhs.layout().ty);
 532             let bin_op = match intrinsic {
 533                 "saturating_add" => BinOp::Add,
 534                 "saturating_sub" => BinOp::Sub,
 535                 _ => unimplemented!("intrinsic {}", intrinsic),
 536             };
 537
 538             let signed = type_sign(T);
 539
 540             let checked_res = crate::num::trans_checked_int_binop(
 541                 fx,
 542                 bin_op,
 543                 lhs,
 544                 rhs,
 545             );
 546
 547             let (val, has_overflow) = checked_res.load_scalar_pair(fx);
 548             let clif_ty = fx.clif_type(T).unwrap();
 549
 550             // `select.i8` is not implemented by Cranelift.
 551             let has_overflow = fx.bcx.ins().uextend(types::I32, has_overflow);
 552
 553             let (min, max) = type_min_max_value(clif_ty, signed);
 554             let min = fx.bcx.ins().iconst(clif_ty, min);
 555             let max = fx.bcx.ins().iconst(clif_ty, max);
 556
 557             let val = match (intrinsic, signed) {
 558                 ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val),
 559                 ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val),
 560                 ("saturating_add", true) => {
 561                     let rhs = rhs.load_scalar(fx);
 562                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 563                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
 564                     fx.bcx.ins().select(has_overflow, sat_val, val)
 565                 }
 566                 ("saturating_sub", true) => {
 567                     let rhs = rhs.load_scalar(fx);
 568                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 569                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
 570                     fx.bcx.ins().select(has_overflow, sat_val, val)
 571                 }
 572                 _ => unreachable!(),
 573             };
 574
 575             let res = CValue::by_val(val, fx.layout_of(T));
 576
 577             ret.write_cvalue(fx, res);
 578         };
 579         rotate_left, <T>(v x, v y) {
 580             let layout = fx.layout_of(T);
 581             let res = fx.bcx.ins().rotl(x, y);
 582             ret.write_cvalue(fx, CValue::by_val(res, layout));
 583         };
 584         rotate_right, <T>(v x, v y) {
 585             let layout = fx.layout_of(T);
 586             let res = fx.bcx.ins().rotr(x, y);
 587             ret.write_cvalue(fx, CValue::by_val(res, layout));
 588         };
 589
 590         // The only difference between offset and arith_offset is regarding UB. Because Cranelift
 591         // doesn't have UB both are codegen'ed the same way
 592         offset | arith_offset, (c base, v offset) {
 593             let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
 594             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 595             let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
 596             let base_val = base.load_scalar(fx);
 597             let res = fx.bcx.ins().iadd(base_val, ptr_diff);
 598             ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
 599         };
 600
 601         transmute, <src_ty, dst_ty> (c from) {
 602             assert_eq!(from.layout().ty, src_ty);
 603             let addr = from.force_stack(fx);
 604             let dst_layout = fx.layout_of(dst_ty);
 605             ret.write_cvalue(fx, CValue::by_ref(addr, dst_layout))
 606         };
 607         init, () {
 608             let layout = ret.layout();
 609             if layout.abi == Abi::Uninhabited {
 610                 crate::trap::trap_panic(fx, "[panic] Called intrinsic::init for uninhabited type.");
 611                 return;
 612             }
 613
 614             match *ret.inner() {
 615                 CPlaceInner::NoPlace => {}
 616                 CPlaceInner::Var(var) => {
 617                     let clif_ty = fx.clif_type(layout.ty).unwrap();
 618                     let val = match clif_ty {
 619                         types::I8 | types::I16 | types::I32 | types::I64 => fx.bcx.ins().iconst(clif_ty, 0),
 620                         types::I128 => {
 621                             let zero = fx.bcx.ins().iconst(types::I64, 0);
 622                             fx.bcx.ins().iconcat(zero, zero)
 623                         }
 624                         types::F32 => {
 625                             let zero = fx.bcx.ins().iconst(types::I32, 0);
 626                             fx.bcx.ins().bitcast(types::F32, zero)
 627                         }
 628                         types::F64 => {
 629                             let zero = fx.bcx.ins().iconst(types::I64, 0);
 630                             fx.bcx.ins().bitcast(types::F64, zero)
 631                         }
 632                         _ => panic!("clif_type returned {}", clif_ty),
 633                     };
 634                     fx.bcx.set_val_label(val, cranelift::codegen::ir::ValueLabel::from_u32(var.as_u32()));
 635                     fx.bcx.def_var(mir_var(var), val);
 636                 }
 637                 _ => {
 638                     let addr = ret.to_ptr(fx).get_addr(fx);
 639                     let layout = ret.layout();
 640                     fx.bcx.emit_small_memset(fx.module.target_config(), addr, 0, layout.size.bytes(), 1);
 641                 }
 642             }
 643         };
 644         uninit, () {
 645             let layout = ret.layout();
 646             if layout.abi == Abi::Uninhabited {
 647                 crate::trap::trap_panic(fx, "[panic] Called intrinsic::uninit for uninhabited type.");
 648                 return;
 649             }
 650             match *ret.inner() {
 651                 CPlaceInner::NoPlace => {},
 652                 CPlaceInner::Var(var) => {
 653                     let clif_ty = fx.clif_type(layout.ty).unwrap();
 654                     let val = match clif_ty {
 655                         types::I8 | types::I16 | types::I32 | types::I64 => fx.bcx.ins().iconst(clif_ty, 42),
 656                         types::I128 => {
 657                             let zero = fx.bcx.ins().iconst(types::I64, 0);
 658                             let fourty_two = fx.bcx.ins().iconst(types::I64, 42);
 659                             fx.bcx.ins().iconcat(fourty_two, zero)
 660                         }
 661                         types::F32 => {
 662                             let zero = fx.bcx.ins().iconst(types::I32, 0xdeadbeef);
 663                             fx.bcx.ins().bitcast(types::F32, zero)
 664                         }
 665                         types::F64 => {
 666                             let zero = fx.bcx.ins().iconst(types::I64, 0xcafebabedeadbeefu64 as i64);
 667                             fx.bcx.ins().bitcast(types::F64, zero)
 668                         }
 669                         _ => panic!("clif_type returned {}", clif_ty),
 670                     };
 671                     fx.bcx.set_val_label(val, cranelift::codegen::ir::ValueLabel::from_u32(var.as_u32()));
 672                     fx.bcx.def_var(mir_var(var), val);
 673                 }
 674                 CPlaceInner::Addr(_, _) => {
 675                     // Don't write to `ret`, as the destination memory is already uninitialized.
 676                 }
 677             }
 678         };
 679         write_bytes, (c dst, v val, v count) {
 680             let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
 681             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 682             let count = fx.bcx.ins().imul_imm(count, pointee_size as i64);
 683             let dst_ptr = dst.load_scalar(fx);
 684             fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count);
 685         };
 686         ctlz | ctlz_nonzero, <T> (v arg) {
 687             // FIXME trap on `ctlz_nonzero` with zero arg.
 688             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 689                 // FIXME verify this algorithm is correct
 690                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 691                 let lsb_lz = fx.bcx.ins().clz(lsb);
 692                 let msb_lz = fx.bcx.ins().clz(msb);
 693                 let msb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, msb, 0);
 694                 let lsb_lz_plus_64 = fx.bcx.ins().iadd_imm(lsb_lz, 64);
 695                 let res = fx.bcx.ins().select(msb_is_zero, lsb_lz_plus_64, msb_lz);
 696                 fx.bcx.ins().uextend(types::I128, res)
 697             } else {
 698                 fx.bcx.ins().clz(arg)
 699             };
 700             let res = CValue::by_val(res, fx.layout_of(T));
 701             ret.write_cvalue(fx, res);
 702         };
 703         cttz | cttz_nonzero, <T> (v arg) {
 704             // FIXME trap on `cttz_nonzero` with zero arg.
 705             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 706                 // FIXME verify this algorithm is correct
 707                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 708                 let lsb_tz = fx.bcx.ins().ctz(lsb);
 709                 let msb_tz = fx.bcx.ins().ctz(msb);
 710                 let lsb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, lsb, 0);
 711                 let msb_tz_plus_64 = fx.bcx.ins().iadd_imm(msb_tz, 64);
 712                 let res = fx.bcx.ins().select(lsb_is_zero, msb_tz_plus_64, lsb_tz);
 713                 fx.bcx.ins().uextend(types::I128, res)
 714             } else {
 715                 fx.bcx.ins().ctz(arg)
 716             };
 717             let res = CValue::by_val(res, fx.layout_of(T));
 718             ret.write_cvalue(fx, res);
 719         };
 720         ctpop, <T> (v arg) {
 721             let res = fx.bcx.ins().popcnt(arg);
 722             let res = CValue::by_val(res, fx.layout_of(T));
 723             ret.write_cvalue(fx, res);
 724         };
 725         bitreverse, <T> (v arg) {
 726             let res = fx.bcx.ins().bitrev(arg);
 727             let res = CValue::by_val(res, fx.layout_of(T));
 728             ret.write_cvalue(fx, res);
 729         };
 730         bswap, <T> (v arg) {
 731             // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
 732             fn swap(bcx: &mut FunctionBuilder, v: Value) -> Value {
 733                 match bcx.func.dfg.value_type(v) {
 734                     types::I8 => v,
 735
 736                     // https://code.woboq.org/gcc/include/bits/byteswap.h.html
 737                     types::I16 => {
 738                         let tmp1 = bcx.ins().ishl_imm(v, 8);
 739                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00);
 740
 741                         let tmp2 = bcx.ins().ushr_imm(v, 8);
 742                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF);
 743
 744                         bcx.ins().bor(n1, n2)
 745                     }
 746                     types::I32 => {
 747                         let tmp1 = bcx.ins().ishl_imm(v, 24);
 748                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000);
 749
 750                         let tmp2 = bcx.ins().ishl_imm(v, 8);
 751                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000);
 752
 753                         let tmp3 = bcx.ins().ushr_imm(v, 8);
 754                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00);
 755
 756                         let tmp4 = bcx.ins().ushr_imm(v, 24);
 757                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF);
 758
 759                         let or_tmp1 = bcx.ins().bor(n1, n2);
 760                         let or_tmp2 = bcx.ins().bor(n3, n4);
 761                         bcx.ins().bor(or_tmp1, or_tmp2)
 762                     }
 763                     types::I64 => {
 764                         let tmp1 = bcx.ins().ishl_imm(v, 56);
 765                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64);
 766
 767                         let tmp2 = bcx.ins().ishl_imm(v, 40);
 768                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64);
 769
 770                         let tmp3 = bcx.ins().ishl_imm(v, 24);
 771                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64);
 772
 773                         let tmp4 = bcx.ins().ishl_imm(v, 8);
 774                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64);
 775
 776                         let tmp5 = bcx.ins().ushr_imm(v, 8);
 777                         let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64);
 778
 779                         let tmp6 = bcx.ins().ushr_imm(v, 24);
 780                         let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64);
 781
 782                         let tmp7 = bcx.ins().ushr_imm(v, 40);
 783                         let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64);
 784
 785                         let tmp8 = bcx.ins().ushr_imm(v, 56);
 786                         let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64);
 787
 788                         let or_tmp1 = bcx.ins().bor(n1, n2);
 789                         let or_tmp2 = bcx.ins().bor(n3, n4);
 790                         let or_tmp3 = bcx.ins().bor(n5, n6);
 791                         let or_tmp4 = bcx.ins().bor(n7, n8);
 792
 793                         let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2);
 794                         let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4);
 795                         bcx.ins().bor(or_tmp5, or_tmp6)
 796                     }
 797                     types::I128 => {
 798                         let (lo, hi) = bcx.ins().isplit(v);
 799                         let lo = swap(bcx, lo);
 800                         let hi = swap(bcx, hi);
 801                         bcx.ins().iconcat(hi, lo)
 802                     }
 803                     ty => unimplemented!("bswap {}", ty),
 804                 }
 805             };
 806             let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T));
 807             ret.write_cvalue(fx, res);
 808         };
 809         panic_if_uninhabited, <T> () {
 810             if fx.layout_of(T).abi.is_uninhabited() {
 811                 crate::trap::trap_panic(fx, "[panic] Called intrinsic::panic_if_uninhabited for uninhabited type.");
 812                 return;
 813             }
 814         };
 815
 816         volatile_load, (c ptr) {
 817             // Cranelift treats loads as volatile by default
 818             let inner_layout =
 819                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 820             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 821             ret.write_cvalue(fx, val);
 822         };
 823         volatile_store, (v ptr, c val) {
 824             // Cranelift treats stores as volatile by default
 825             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 826             dest.write_cvalue(fx, val);
 827         };
 828
 829         size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name, () {
 830             let const_val =
 831                 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
 832             let val = crate::constant::trans_const_value(fx, const_val);
 833             ret.write_cvalue(fx, val);
 834         };
 835
 836         ptr_offset_from, <T> (v ptr, v base) {
 837             let isize_layout = fx.layout_of(fx.tcx.types.isize);
 838
 839             let pointee_size: u64 = fx.layout_of(T).size.bytes();
 840             let diff = fx.bcx.ins().isub(ptr, base);
 841             // FIXME this can be an exact division.
 842             let val = CValue::by_val(fx.bcx.ins().udiv_imm(diff, pointee_size as i64), isize_layout);
 843             ret.write_cvalue(fx, val);
 844         };
 845
 846         caller_location, () {
 847             let caller_location = fx.get_caller_location(span);
 848             ret.write_cvalue(fx, caller_location);
 849         };
 850
 851         _ if intrinsic.starts_with("atomic_fence"), () {};
 852         _ if intrinsic.starts_with("atomic_singlethreadfence"), () {};
 853         _ if intrinsic.starts_with("atomic_load"), (c ptr) {
 854             let inner_layout =
 855                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 856             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 857             ret.write_cvalue(fx, val);
 858         };
 859         _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
 860             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 861             dest.write_cvalue(fx, val);
 862         };
 863         _ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
 864             // Read old
 865             let clif_ty = fx.clif_type(T).unwrap();
 866             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 867             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
 868
 869             // Write new
 870             let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout());
 871             dest.write_cvalue(fx, src);
 872         };
 873         _ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, v test_old, v new) { // both atomic_cxchg_* and atomic_cxchgweak_*
 874             // Read old
 875             let clif_ty = fx.clif_type(T).unwrap();
 876             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 877
 878             // Compare
 879             let is_eq = codegen_icmp(fx, IntCC::Equal, old, test_old);
 880             let new = fx.bcx.ins().select(is_eq, new, old); // Keep old if not equal to test_old
 881
 882             // Write new
 883             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 884
 885             let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
 886             ret.write_cvalue(fx, ret_val);
 887         };
 888
 889         _ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, v amount) {
 890             atomic_binop_return_old! (fx, iadd<T>(ptr, amount) -> ret);
 891         };
 892         _ if intrinsic.starts_with("atomic_xsub"), <T> (v ptr, v amount) {
 893             atomic_binop_return_old! (fx, isub<T>(ptr, amount) -> ret);
 894         };
 895         _ if intrinsic.starts_with("atomic_and"), <T> (v ptr, v src) {
 896             atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
 897         };
 898         _ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, v src) {
 899             let clif_ty = fx.clif_type(T).unwrap();
 900             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 901             let and = fx.bcx.ins().band(old, src);
 902             let new = fx.bcx.ins().bnot(and);
 903             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 904             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
 905         };
 906         _ if intrinsic.starts_with("atomic_or"), <T> (v ptr, v src) {
 907             atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);
 908         };
 909         _ if intrinsic.starts_with("atomic_xor"), <T> (v ptr, v src) {
 910             atomic_binop_return_old! (fx, bxor<T>(ptr, src) -> ret);
 911         };
 912
 913         _ if intrinsic.starts_with("atomic_max"), <T> (v ptr, v src) {
 914             atomic_minmax!(fx, IntCC::SignedGreaterThan, <T> (ptr, src) -> ret);
 915         };
 916         _ if intrinsic.starts_with("atomic_umax"), <T> (v ptr, v src) {
 917             atomic_minmax!(fx, IntCC::UnsignedGreaterThan, <T> (ptr, src) -> ret);
 918         };
 919         _ if intrinsic.starts_with("atomic_min"), <T> (v ptr, v src) {
 920             atomic_minmax!(fx, IntCC::SignedLessThan, <T> (ptr, src) -> ret);
 921         };
 922         _ if intrinsic.starts_with("atomic_umin"), <T> (v ptr, v src) {
 923             atomic_minmax!(fx, IntCC::UnsignedLessThan, <T> (ptr, src) -> ret);
 924         };
 925
 926         minnumf32, (v a, v b) {
 927             let val = fx.bcx.ins().fmin(a, b);
 928             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
 929             ret.write_cvalue(fx, val);
 930         };
 931         minnumf64, (v a, v b) {
 932             let val = fx.bcx.ins().fmin(a, b);
 933             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
 934             ret.write_cvalue(fx, val);
 935         };
 936         maxnumf32, (v a, v b) {
 937             let val = fx.bcx.ins().fmax(a, b);
 938             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
 939             ret.write_cvalue(fx, val);
 940         };
 941         maxnumf64, (v a, v b) {
 942             let val = fx.bcx.ins().fmax(a, b);
 943             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
 944             ret.write_cvalue(fx, val);
 945         };
 946
 947         simd_cast, (c a) {
 948             let (lane_layout, lane_count) = lane_type_and_count(fx, a.layout(), intrinsic);
 949             let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx, ret.layout(), intrinsic);
 950             assert_eq!(lane_count, ret_lane_count);
 951
 952             let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
 953
 954             let from_signed = type_sign(lane_layout.ty);
 955             let to_signed = type_sign(ret_lane_layout.ty);
 956
 957             for lane in 0..lane_count {
 958                 let lane = mir::Field::new(lane.try_into().unwrap());
 959
 960                 let a_lane = a.value_field(fx, lane).load_scalar(fx);
 961                 let res = clif_int_or_float_cast(fx, a_lane, from_signed, ret_lane_ty, to_signed);
 962                 ret.place_field(fx, lane).write_cvalue(fx, CValue::by_val(res, ret_lane_layout));
 963             }
 964         };
 965
 966         simd_eq, (c x, c y) {
 967             simd_cmp!(fx, intrinsic, Equal(x, y) -> ret);
 968         };
 969         simd_ne, (c x, c y) {
 970             simd_cmp!(fx, intrinsic, NotEqual(x, y) -> ret);
 971         };
 972         simd_lt, (c x, c y) {
 973             simd_cmp!(fx, intrinsic, UnsignedLessThan|SignedLessThan(x, y) -> ret);
 974         };
 975         simd_le, (c x, c y) {
 976             simd_cmp!(fx, intrinsic, UnsignedLessThanOrEqual|SignedLessThanOrEqual(x, y) -> ret);
 977         };
 978         simd_gt, (c x, c y) {
 979             simd_cmp!(fx, intrinsic, UnsignedGreaterThan|SignedGreaterThan(x, y) -> ret);
 980         };
 981         simd_ge, (c x, c y) {
 982             simd_cmp!(fx, intrinsic, UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual(x, y) -> ret);
 983         };
 984
 985         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
 986         _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) {
 987             let n: u32 = intrinsic["simd_shuffle".len()..].parse().unwrap();
 988
 989             assert_eq!(x.layout(), y.layout());
 990             let layout = x.layout();
 991
 992             let (lane_type, lane_count) = lane_type_and_count(fx, layout, intrinsic);
 993             let (ret_lane_type, ret_lane_count) = lane_type_and_count(fx, ret.layout(), intrinsic);
 994
 995             assert_eq!(lane_type, ret_lane_type);
 996             assert_eq!(n, ret_lane_count);
 997
 998             let total_len = lane_count * 2;
 999
1000             let indexes = {
1001                 use rustc::mir::interpret::*;
1002                 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
1003
1004                 let idx_bytes = match idx_const.val {
1005                     ty::ConstKind::Value(ConstValue::ByRef { alloc, offset }) => {
1006                         let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
1007                         let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
1008                         alloc.get_bytes(fx, ptr, size).unwrap()
1009                     }
1010                     _ => unreachable!("{:?}", idx_const),
1011                 };
1012
1013                 (0..ret_lane_count).map(|i| {
1014                     let i = usize::try_from(i).unwrap();
1015                     let idx = rustc::mir::interpret::read_target_uint(
1016                         fx.tcx.data_layout.endian,
1017                         &idx_bytes[4*i.. 4*i + 4],
1018                     ).expect("read_target_uint");
1019                     u32::try_from(idx).expect("try_from u32")
1020                 }).collect::<Vec<u32>>()
1021             };
1022
1023             for &idx in &indexes {
1024                 assert!(idx < total_len, "idx {} out of range 0..{}", idx, total_len);
1025             }
1026
1027             for (out_idx, in_idx) in indexes.into_iter().enumerate() {
1028                 let in_lane = if in_idx < lane_count {
1029                     x.value_field(fx, mir::Field::new(in_idx.try_into().unwrap()))
1030                 } else {
1031                     y.value_field(fx, mir::Field::new((in_idx - lane_count).try_into().unwrap()))
1032                 };
1033                 let out_lane = ret.place_field(fx, mir::Field::new(out_idx));
1034                 out_lane.write_cvalue(fx, in_lane);
1035             }
1036         };
1037
1038         simd_extract, (c v, o idx) {
1039             let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
1040                 idx_const
1041             } else {
1042                 fx.tcx.sess.span_warn(
1043                     fx.mir.span,
1044                     "`#[rustc_arg_required_const(..)]` is not yet supported. Calling this function will panic.",
1045                 );
1046                 crate::trap::trap_panic(fx, "`#[rustc_arg_required_const(..)]` is not yet supported.");
1047                 return;
1048             };
1049
1050             let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).expect(&format!("kind not scalar: {:?}", idx_const));
1051             let (_lane_type, lane_count) = lane_type_and_count(fx, v.layout(), intrinsic);
1052             if idx >= lane_count.into() {
1053                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
1054             }
1055
1056             let ret_lane = v.value_field(fx, mir::Field::new(idx.try_into().unwrap()));
1057             ret.write_cvalue(fx, ret_lane);
1058         };
1059
1060         simd_add, (c x, c y) {
1061             simd_int_flt_binop!(fx, intrinsic, iadd|fadd(x, y) -> ret);
1062         };
1063         simd_sub, (c x, c y) {
1064             simd_int_flt_binop!(fx, intrinsic, isub|fsub(x, y) -> ret);
1065         };
1066         simd_mul, (c x, c y) {
1067             simd_int_flt_binop!(fx, intrinsic, imul|fmul(x, y) -> ret);
1068         };
1069         simd_div, (c x, c y) {
1070             simd_int_flt_binop!(fx, intrinsic, udiv|sdiv|fdiv(x, y) -> ret);
1071         };
1072         simd_shl, (c x, c y) {
1073             simd_int_binop!(fx, intrinsic, ishl(x, y) -> ret);
1074         };
1075         simd_shr, (c x, c y) {
1076             simd_int_binop!(fx, intrinsic, ushr|sshr(x, y) -> ret);
1077         };
1078         simd_and, (c x, c y) {
1079             simd_int_binop!(fx, intrinsic, band(x, y) -> ret);
1080         };
1081         simd_or, (c x, c y) {
1082             simd_int_binop!(fx, intrinsic, bor(x, y) -> ret);
1083         };
1084         simd_xor, (c x, c y) {
1085             simd_int_binop!(fx, intrinsic, bxor(x, y) -> ret);
1086         };
1087
1088         simd_fmin, (c x, c y) {
1089             simd_flt_binop!(fx, intrinsic, fmin(x, y) -> ret);
1090         };
1091         simd_fmax, (c x, c y) {
1092             simd_flt_binop!(fx, intrinsic, fmax(x, y) -> ret);
1093         };
1094
1095         try, (v f, v data, v _local_ptr) {
1096             // FIXME once unwinding is supported, change this to actually catch panics
1097             let f_sig = fx.bcx.func.import_signature(Signature {
1098                 call_conv: CallConv::triple_default(fx.triple()),
1099                 params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
1100                 returns: vec![],
1101             });
1102
1103             fx.bcx.ins().call_indirect(f_sig, f, &[data]);
1104
1105             let ret_val = CValue::const_val(fx, ret.layout().ty, 0);
1106             ret.write_cvalue(fx, ret_val);
1107         };
1108     }
1109
1110     if let Some((_, dest)) = destination {
1111         let ret_ebb = fx.get_ebb(dest);
1112         fx.bcx.ins().jump(ret_ebb, &[]);
1113     } else {
1114         trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
1115     }
1116 }