compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

   1 //! Codegen `extern "platform-intrinsic"` intrinsics.
   2
   3 use rustc_middle::ty::subst::SubstsRef;
   4 use rustc_span::Symbol;
   5 use rustc_target::abi::Endian;
   6
   7 use super::*;
   8 use crate::prelude::*;
   9
  10 fn report_simd_type_validation_error(
  11     fx: &mut FunctionCx<'_, '_, '_>,
  12     intrinsic: Symbol,
  13     span: Span,
  14     ty: Ty<'_>,
  15 ) {
  16     fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
  17     // Prevent verifier error
  18     fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
  19 }
  20
  21 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
  22     fx: &mut FunctionCx<'_, '_, 'tcx>,
  23     intrinsic: Symbol,
  24     _substs: SubstsRef<'tcx>,
  25     args: &[mir::Operand<'tcx>],
  26     ret: CPlace<'tcx>,
  27     span: Span,
  28 ) {
  29     match intrinsic {
  30         sym::simd_as | sym::simd_cast => {
  31             intrinsic_args!(fx, args => (a); intrinsic);
  32
  33             if !a.layout().ty.is_simd() {
  34                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
  35                 return;
  36             }
  37
  38             simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
  39                 let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
  40
  41                 let from_signed = type_sign(lane_ty);
  42                 let to_signed = type_sign(ret_lane_ty);
  43
  44                 clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
  45             });
  46         }
  47
  48         sym::simd_eq | sym::simd_ne | sym::simd_lt | sym::simd_le | sym::simd_gt | sym::simd_ge => {
  49             intrinsic_args!(fx, args => (x, y); intrinsic);
  50
  51             if !x.layout().ty.is_simd() {
  52                 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
  53                 return;
  54             }
  55
  56             // FIXME use vector instructions when possible
  57             simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
  58                 let res_lane = match (lane_ty.kind(), intrinsic) {
  59                     (ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
  60                     (ty::Uint(_), sym::simd_ne) => {
  61                         fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane)
  62                     }
  63                     (ty::Uint(_), sym::simd_lt) => {
  64                         fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
  65                     }
  66                     (ty::Uint(_), sym::simd_le) => {
  67                         fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane)
  68                     }
  69                     (ty::Uint(_), sym::simd_gt) => {
  70                         fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane)
  71                     }
  72                     (ty::Uint(_), sym::simd_ge) => {
  73                         fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane)
  74                     }
  75
  76                     (ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
  77                     (ty::Int(_), sym::simd_ne) => {
  78                         fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane)
  79                     }
  80                     (ty::Int(_), sym::simd_lt) => {
  81                         fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane)
  82                     }
  83                     (ty::Int(_), sym::simd_le) => {
  84                         fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
  85                     }
  86                     (ty::Int(_), sym::simd_gt) => {
  87                         fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane)
  88                     }
  89                     (ty::Int(_), sym::simd_ge) => {
  90                         fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
  91                     }
  92
  93                     (ty::Float(_), sym::simd_eq) => {
  94                         fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane)
  95                     }
  96                     (ty::Float(_), sym::simd_ne) => {
  97                         fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane)
  98                     }
  99                     (ty::Float(_), sym::simd_lt) => {
 100                         fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane)
 101                     }
 102                     (ty::Float(_), sym::simd_le) => {
 103                         fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
 104                     }
 105                     (ty::Float(_), sym::simd_gt) => {
 106                         fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane)
 107                     }
 108                     (ty::Float(_), sym::simd_ge) => {
 109                         fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
 110                     }
 111
 112                     _ => unreachable!(),
 113                 };
 114
 115                 let ty = fx.clif_type(res_lane_ty).unwrap();
 116
 117                 let res_lane = fx.bcx.ins().bint(ty, res_lane);
 118                 fx.bcx.ins().ineg(res_lane)
 119             });
 120         }
 121
 122         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
 123         _ if intrinsic.as_str().starts_with("simd_shuffle") => {
 124             let (x, y, idx) = match args {
 125                 [x, y, idx] => (x, y, idx),
 126                 _ => {
 127                     bug!("wrong number of args for intrinsic {intrinsic}");
 128                 }
 129             };
 130             let x = codegen_operand(fx, x);
 131             let y = codegen_operand(fx, y);
 132
 133             if !x.layout().ty.is_simd() {
 134                 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
 135                 return;
 136             }
 137
 138             // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
 139             // If there is no suffix, use the index array length.
 140             let n: u16 = if intrinsic == sym::simd_shuffle {
 141                 // Make sure this is actually an array, since typeck only checks the length-suffixed
 142                 // version of this intrinsic.
 143                 let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
 144                 match idx_ty.kind() {
 145                     ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => len
 146                         .try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
 147                         .unwrap_or_else(|| {
 148                             span_bug!(span, "could not evaluate shuffle index array length")
 149                         })
 150                         .try_into()
 151                         .unwrap(),
 152                     _ => {
 153                         fx.tcx.sess.span_err(
 154                             span,
 155                             &format!(
 156                                 "simd_shuffle index must be an array of `u32`, got `{}`",
 157                                 idx_ty,
 158                             ),
 159                         );
 160                         // Prevent verifier error
 161                         fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
 162                         return;
 163                     }
 164                 }
 165             } else {
 166                 // FIXME remove this case
 167                 intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
 168             };
 169
 170             assert_eq!(x.layout(), y.layout());
 171             let layout = x.layout();
 172
 173             let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
 174             let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
 175
 176             assert_eq!(lane_ty, ret_lane_ty);
 177             assert_eq!(u64::from(n), ret_lane_count);
 178
 179             let total_len = lane_count * 2;
 180
 181             let indexes = {
 182                 use rustc_middle::mir::interpret::*;
 183                 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx)
 184                     .expect("simd_shuffle* idx not const");
 185
 186                 let idx_bytes = match idx_const {
 187                     ConstValue::ByRef { alloc, offset } => {
 188                         let size = Size::from_bytes(
 189                             4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */
 190                         );
 191                         alloc
 192                             .inner()
 193                             .get_bytes_strip_provenance(fx, alloc_range(offset, size))
 194                             .unwrap()
 195                     }
 196                     _ => unreachable!("{:?}", idx_const),
 197                 };
 198
 199                 (0..ret_lane_count)
 200                     .map(|i| {
 201                         let i = usize::try_from(i).unwrap();
 202                         let idx = rustc_middle::mir::interpret::read_target_uint(
 203                             fx.tcx.data_layout.endian,
 204                             &idx_bytes[4 * i..4 * i + 4],
 205                         )
 206                         .expect("read_target_uint");
 207                         u16::try_from(idx).expect("try_from u32")
 208                     })
 209                     .collect::<Vec<u16>>()
 210             };
 211
 212             for &idx in &indexes {
 213                 assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
 214             }
 215
 216             for (out_idx, in_idx) in indexes.into_iter().enumerate() {
 217                 let in_lane = if u64::from(in_idx) < lane_count {
 218                     x.value_lane(fx, in_idx.into())
 219                 } else {
 220                     y.value_lane(fx, u64::from(in_idx) - lane_count)
 221                 };
 222                 let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap());
 223                 out_lane.write_cvalue(fx, in_lane);
 224             }
 225         }
 226
 227         sym::simd_insert => {
 228             let (base, idx, val) = match args {
 229                 [base, idx, val] => (base, idx, val),
 230                 _ => {
 231                     bug!("wrong number of args for intrinsic {intrinsic}");
 232                 }
 233             };
 234             let base = codegen_operand(fx, base);
 235             let val = codegen_operand(fx, val);
 236
 237             // FIXME validate
 238             let idx_const = if let Some(idx_const) =
 239                 crate::constant::mir_operand_get_const_val(fx, idx)
 240             {
 241                 idx_const
 242             } else {
 243                 fx.tcx.sess.span_fatal(span, "Index argument for `simd_insert` is not a constant");
 244             };
 245
 246             let idx = idx_const
 247                 .try_to_bits(Size::from_bytes(4 /* u32*/))
 248                 .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
 249             let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
 250             if idx >= lane_count.into() {
 251                 fx.tcx.sess.span_fatal(
 252                     fx.mir.span,
 253                     &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count),
 254                 );
 255             }
 256
 257             ret.write_cvalue(fx, base);
 258             let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
 259             ret_lane.write_cvalue(fx, val);
 260         }
 261
 262         sym::simd_extract => {
 263             let (v, idx) = match args {
 264                 [v, idx] => (v, idx),
 265                 _ => {
 266                     bug!("wrong number of args for intrinsic {intrinsic}");
 267                 }
 268             };
 269             let v = codegen_operand(fx, v);
 270
 271             if !v.layout().ty.is_simd() {
 272                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 273                 return;
 274             }
 275
 276             let idx_const = if let Some(idx_const) =
 277                 crate::constant::mir_operand_get_const_val(fx, idx)
 278             {
 279                 idx_const
 280             } else {
 281                 fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
 282                 let trap_block = fx.bcx.create_block();
 283                 let dummy_block = fx.bcx.create_block();
 284                 let true_ = fx.bcx.ins().iconst(types::I8, 1);
 285                 fx.bcx.ins().brnz(true_, trap_block, &[]);
 286                 fx.bcx.ins().jump(dummy_block, &[]);
 287                 fx.bcx.switch_to_block(trap_block);
 288                 crate::trap::trap_unimplemented(
 289                     fx,
 290                     "Index argument for `simd_extract` is not a constant",
 291                 );
 292                 fx.bcx.switch_to_block(dummy_block);
 293                 return;
 294             };
 295
 296             let idx = idx_const
 297                 .try_to_bits(Size::from_bytes(4 /* u32*/))
 298                 .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
 299             let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
 300             if idx >= lane_count.into() {
 301                 fx.tcx.sess.span_fatal(
 302                     fx.mir.span,
 303                     &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count),
 304                 );
 305             }
 306
 307             let ret_lane = v.value_lane(fx, idx.try_into().unwrap());
 308             ret.write_cvalue(fx, ret_lane);
 309         }
 310
 311         sym::simd_neg => {
 312             intrinsic_args!(fx, args => (a); intrinsic);
 313
 314             if !a.layout().ty.is_simd() {
 315                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 316                 return;
 317             }
 318
 319             simd_for_each_lane(
 320                 fx,
 321                 a,
 322                 ret,
 323                 &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
 324                     ty::Int(_) => fx.bcx.ins().ineg(lane),
 325                     ty::Float(_) => fx.bcx.ins().fneg(lane),
 326                     _ => unreachable!(),
 327                 },
 328             );
 329         }
 330
 331         sym::simd_add
 332         | sym::simd_sub
 333         | sym::simd_mul
 334         | sym::simd_div
 335         | sym::simd_rem
 336         | sym::simd_shl
 337         | sym::simd_shr
 338         | sym::simd_and
 339         | sym::simd_or
 340         | sym::simd_xor => {
 341             intrinsic_args!(fx, args => (x, y); intrinsic);
 342
 343             // FIXME use vector instructions when possible
 344             simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
 345                 match (lane_ty.kind(), intrinsic) {
 346                     (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
 347                     (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
 348                     (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
 349                     (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
 350                     (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
 351
 352                     (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
 353                     (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
 354                     (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
 355                     (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
 356                     (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
 357
 358                     (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
 359                     (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
 360                     (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
 361                     (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
 362                     (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
 363                         "fmodf",
 364                         vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
 365                         vec![AbiParam::new(types::F32)],
 366                         &[x_lane, y_lane],
 367                     )[0],
 368                     (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
 369                         "fmod",
 370                         vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
 371                         vec![AbiParam::new(types::F64)],
 372                         &[x_lane, y_lane],
 373                     )[0],
 374
 375                     (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
 376                     (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
 377                     (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
 378                     (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
 379                     (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
 380
 381                     (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
 382                     (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
 383                     (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
 384                     (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
 385                     (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
 386
 387                     _ => unreachable!(),
 388                 }
 389             });
 390         }
 391
 392         sym::simd_fma => {
 393             intrinsic_args!(fx, args => (a, b, c); intrinsic);
 394
 395             if !a.layout().ty.is_simd() {
 396                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 397                 return;
 398             }
 399             assert_eq!(a.layout(), b.layout());
 400             assert_eq!(a.layout(), c.layout());
 401             assert_eq!(a.layout(), ret.layout());
 402
 403             let layout = a.layout();
 404             let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
 405             let res_lane_layout = fx.layout_of(lane_ty);
 406
 407             for lane in 0..lane_count {
 408                 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
 409                 let b_lane = b.value_lane(fx, lane).load_scalar(fx);
 410                 let c_lane = c.value_lane(fx, lane).load_scalar(fx);
 411
 412                 let res_lane = fx.bcx.ins().fma(a_lane, b_lane, c_lane);
 413                 let res_lane = CValue::by_val(res_lane, res_lane_layout);
 414
 415                 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
 416             }
 417         }
 418
 419         sym::simd_fmin | sym::simd_fmax => {
 420             intrinsic_args!(fx, args => (x, y); intrinsic);
 421
 422             if !x.layout().ty.is_simd() {
 423                 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
 424                 return;
 425             }
 426
 427             // FIXME use vector instructions when possible
 428             simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
 429                 match lane_ty.kind() {
 430                     ty::Float(_) => {}
 431                     _ => unreachable!("{:?}", lane_ty),
 432                 }
 433                 match intrinsic {
 434                     sym::simd_fmin => crate::num::codegen_float_min(fx, x_lane, y_lane),
 435                     sym::simd_fmax => crate::num::codegen_float_max(fx, x_lane, y_lane),
 436                     _ => unreachable!(),
 437                 }
 438             });
 439         }
 440
 441         sym::simd_round => {
 442             intrinsic_args!(fx, args => (a); intrinsic);
 443
 444             if !a.layout().ty.is_simd() {
 445                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 446                 return;
 447             }
 448
 449             simd_for_each_lane(
 450                 fx,
 451                 a,
 452                 ret,
 453                 &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
 454                     ty::Float(FloatTy::F32) => fx.lib_call(
 455                         "roundf",
 456                         vec![AbiParam::new(types::F32)],
 457                         vec![AbiParam::new(types::F32)],
 458                         &[lane],
 459                     )[0],
 460                     ty::Float(FloatTy::F64) => fx.lib_call(
 461                         "round",
 462                         vec![AbiParam::new(types::F64)],
 463                         vec![AbiParam::new(types::F64)],
 464                         &[lane],
 465                     )[0],
 466                     _ => unreachable!("{:?}", lane_ty),
 467                 },
 468             );
 469         }
 470
 471         sym::simd_fabs | sym::simd_fsqrt | sym::simd_ceil | sym::simd_floor | sym::simd_trunc => {
 472             intrinsic_args!(fx, args => (a); intrinsic);
 473
 474             if !a.layout().ty.is_simd() {
 475                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 476                 return;
 477             }
 478
 479             simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
 480                 match lane_ty.kind() {
 481                     ty::Float(_) => {}
 482                     _ => unreachable!("{:?}", lane_ty),
 483                 }
 484                 match intrinsic {
 485                     sym::simd_fabs => fx.bcx.ins().fabs(lane),
 486                     sym::simd_fsqrt => fx.bcx.ins().sqrt(lane),
 487                     sym::simd_ceil => fx.bcx.ins().ceil(lane),
 488                     sym::simd_floor => fx.bcx.ins().floor(lane),
 489                     sym::simd_trunc => fx.bcx.ins().trunc(lane),
 490                     _ => unreachable!(),
 491                 }
 492             });
 493         }
 494
 495         sym::simd_reduce_add_ordered | sym::simd_reduce_add_unordered => {
 496             intrinsic_args!(fx, args => (v, acc); intrinsic);
 497             let acc = acc.load_scalar(fx);
 498
 499             // FIXME there must be no acc param for integer vectors
 500             if !v.layout().ty.is_simd() {
 501                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 502                 return;
 503             }
 504
 505             simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
 506                 if lane_ty.is_floating_point() {
 507                     fx.bcx.ins().fadd(a, b)
 508                 } else {
 509                     fx.bcx.ins().iadd(a, b)
 510                 }
 511             });
 512         }
 513
 514         sym::simd_reduce_mul_ordered | sym::simd_reduce_mul_unordered => {
 515             intrinsic_args!(fx, args => (v, acc); intrinsic);
 516             let acc = acc.load_scalar(fx);
 517
 518             // FIXME there must be no acc param for integer vectors
 519             if !v.layout().ty.is_simd() {
 520                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 521                 return;
 522             }
 523
 524             simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
 525                 if lane_ty.is_floating_point() {
 526                     fx.bcx.ins().fmul(a, b)
 527                 } else {
 528                     fx.bcx.ins().imul(a, b)
 529                 }
 530             });
 531         }
 532
 533         sym::simd_reduce_all => {
 534             intrinsic_args!(fx, args => (v); intrinsic);
 535
 536             if !v.layout().ty.is_simd() {
 537                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 538                 return;
 539             }
 540
 541             simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
 542         }
 543
 544         sym::simd_reduce_any => {
 545             intrinsic_args!(fx, args => (v); intrinsic);
 546
 547             if !v.layout().ty.is_simd() {
 548                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 549                 return;
 550             }
 551
 552             simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
 553         }
 554
 555         sym::simd_reduce_and => {
 556             intrinsic_args!(fx, args => (v); intrinsic);
 557
 558             if !v.layout().ty.is_simd() {
 559                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 560                 return;
 561             }
 562
 563             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
 564         }
 565
 566         sym::simd_reduce_or => {
 567             intrinsic_args!(fx, args => (v); intrinsic);
 568
 569             if !v.layout().ty.is_simd() {
 570                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 571                 return;
 572             }
 573
 574             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
 575         }
 576
 577         sym::simd_reduce_xor => {
 578             intrinsic_args!(fx, args => (v); intrinsic);
 579
 580             if !v.layout().ty.is_simd() {
 581                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 582                 return;
 583             }
 584
 585             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
 586         }
 587
 588         sym::simd_reduce_min => {
 589             intrinsic_args!(fx, args => (v); intrinsic);
 590
 591             if !v.layout().ty.is_simd() {
 592                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 593                 return;
 594             }
 595
 596             simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
 597                 let lt = match ty.kind() {
 598                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
 599                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
 600                     ty::Float(_) => return crate::num::codegen_float_min(fx, a, b),
 601                     _ => unreachable!(),
 602                 };
 603                 fx.bcx.ins().select(lt, a, b)
 604             });
 605         }
 606
 607         sym::simd_reduce_max => {
 608             intrinsic_args!(fx, args => (v); intrinsic);
 609
 610             if !v.layout().ty.is_simd() {
 611                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 612                 return;
 613             }
 614
 615             simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
 616                 let gt = match ty.kind() {
 617                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
 618                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
 619                     ty::Float(_) => return crate::num::codegen_float_max(fx, a, b),
 620                     _ => unreachable!(),
 621                 };
 622                 fx.bcx.ins().select(gt, a, b)
 623             });
 624         }
 625
 626         sym::simd_select => {
 627             intrinsic_args!(fx, args => (m, a, b); intrinsic);
 628
 629             if !m.layout().ty.is_simd() {
 630                 report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
 631                 return;
 632             }
 633             if !a.layout().ty.is_simd() {
 634                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 635                 return;
 636             }
 637             assert_eq!(a.layout(), b.layout());
 638
 639             let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
 640             let lane_layout = fx.layout_of(lane_ty);
 641
 642             for lane in 0..lane_count {
 643                 let m_lane = m.value_lane(fx, lane).load_scalar(fx);
 644                 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
 645                 let b_lane = b.value_lane(fx, lane).load_scalar(fx);
 646
 647                 let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
 648                 let res_lane =
 649                     CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
 650
 651                 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
 652             }
 653         }
 654
 655         sym::simd_select_bitmask => {
 656             intrinsic_args!(fx, args => (m, a, b); intrinsic);
 657
 658             if !a.layout().ty.is_simd() {
 659                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 660                 return;
 661             }
 662             assert_eq!(a.layout(), b.layout());
 663
 664             let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
 665             let lane_layout = fx.layout_of(lane_ty);
 666
 667             let m = m.load_scalar(fx);
 668
 669             for lane in 0..lane_count {
 670                 let m_lane = fx.bcx.ins().ushr_imm(m, u64::from(lane) as i64);
 671                 let m_lane = fx.bcx.ins().band_imm(m_lane, 1);
 672                 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
 673                 let b_lane = b.value_lane(fx, lane).load_scalar(fx);
 674
 675                 let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
 676                 let res_lane =
 677                     CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
 678
 679                 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
 680             }
 681         }
 682
 683         sym::simd_bitmask => {
 684             intrinsic_args!(fx, args => (a); intrinsic);
 685
 686             let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
 687             let lane_clif_ty = fx.clif_type(lane_ty).unwrap();
 688
 689             // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a
 690             // vector mask and returns the most significant bit (MSB) of each lane in the form
 691             // of either:
 692             // * an unsigned integer
 693             // * an array of `u8`
 694             // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits.
 695             //
 696             // The bit order of the result depends on the byte endianness, LSB-first for little
 697             // endian and MSB-first for big endian.
 698             let expected_int_bits = lane_count.max(8);
 699             let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64);
 700
 701             match lane_ty.kind() {
 702                 ty::Int(_) | ty::Uint(_) => {}
 703                 _ => {
 704                     fx.tcx.sess.span_fatal(
 705                         span,
 706                         &format!(
 707                             "invalid monomorphization of `simd_bitmask` intrinsic: \
 708                             vector argument `{}`'s element type `{}`, expected integer element \
 709                             type",
 710                             a.layout().ty,
 711                             lane_ty
 712                         ),
 713                     );
 714                 }
 715             }
 716
 717             let res_type =
 718                 Type::int_with_byte_size(u16::try_from(expected_bytes).unwrap()).unwrap();
 719             let mut res = fx.bcx.ins().iconst(res_type, 0);
 720
 721             let lanes = match fx.tcx.sess.target.endian {
 722                 Endian::Big => Box::new(0..lane_count) as Box<dyn Iterator<Item = u64>>,
 723                 Endian::Little => Box::new((0..lane_count).rev()) as Box<dyn Iterator<Item = u64>>,
 724             };
 725             for lane in lanes {
 726                 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
 727
 728                 // extract sign bit of an int
 729                 let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_clif_ty.bits() - 1));
 730
 731                 // shift sign bit into result
 732                 let a_lane_sign = clif_intcast(fx, a_lane_sign, res_type, false);
 733                 res = fx.bcx.ins().ishl_imm(res, 1);
 734                 res = fx.bcx.ins().bor(res, a_lane_sign);
 735             }
 736
 737             match ret.layout().ty.kind() {
 738                 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {}
 739                 ty::Array(elem, len)
 740                     if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
 741                         && len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
 742                             == Some(expected_bytes) => {}
 743                 _ => {
 744                     fx.tcx.sess.span_fatal(
 745                         span,
 746                         &format!(
 747                             "invalid monomorphization of `simd_bitmask` intrinsic: \
 748                             cannot return `{}`, expected `u{}` or `[u8; {}]`",
 749                             ret.layout().ty,
 750                             expected_int_bits,
 751                             expected_bytes
 752                         ),
 753                     );
 754                 }
 755             }
 756
 757             let res = CValue::by_val(res, ret.layout());
 758             ret.write_cvalue(fx, res);
 759         }
 760
 761         sym::simd_saturating_add | sym::simd_saturating_sub => {
 762             intrinsic_args!(fx, args => (x, y); intrinsic);
 763
 764             let bin_op = match intrinsic {
 765                 sym::simd_saturating_add => BinOp::Add,
 766                 sym::simd_saturating_sub => BinOp::Sub,
 767                 _ => unreachable!(),
 768             };
 769
 770             // FIXME use vector instructions when possible
 771             simd_pair_for_each_lane_typed(fx, x, y, ret, &|fx, x_lane, y_lane| {
 772                 crate::num::codegen_saturating_int_binop(fx, bin_op, x_lane, y_lane)
 773             });
 774         }
 775
 776         // simd_arith_offset
 777         // simd_scatter
 778         // simd_gather
 779         _ => {
 780             fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
 781         }
 782     }
 783 }