compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

   1 //! Codegen `extern "platform-intrinsic"` intrinsics.
   2
   3 use rustc_middle::ty::subst::SubstsRef;
   4 use rustc_span::Symbol;
   5
   6 use super::*;
   7 use crate::prelude::*;
   8
   9 fn report_simd_type_validation_error(
  10     fx: &mut FunctionCx<'_, '_, '_>,
  11     intrinsic: Symbol,
  12     span: Span,
  13     ty: Ty<'_>,
  14 ) {
  15     fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
  16     // Prevent verifier error
  17     fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
  18 }
  19
  20 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
  21     fx: &mut FunctionCx<'_, '_, 'tcx>,
  22     intrinsic: Symbol,
  23     _substs: SubstsRef<'tcx>,
  24     args: &[mir::Operand<'tcx>],
  25     ret: CPlace<'tcx>,
  26     span: Span,
  27 ) {
  28     match intrinsic {
  29         sym::simd_cast => {
  30             intrinsic_args!(fx, args => (a); intrinsic);
  31
  32             if !a.layout().ty.is_simd() {
  33                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
  34                 return;
  35             }
  36
  37             simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
  38                 let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
  39
  40                 let from_signed = type_sign(lane_ty);
  41                 let to_signed = type_sign(ret_lane_ty);
  42
  43                 clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
  44             });
  45         }
  46
  47         sym::simd_eq | sym::simd_ne | sym::simd_lt | sym::simd_le | sym::simd_gt | sym::simd_ge => {
  48             intrinsic_args!(fx, args => (x, y); intrinsic);
  49
  50             if !x.layout().ty.is_simd() {
  51                 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
  52                 return;
  53             }
  54
  55             // FIXME use vector instructions when possible
  56             simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
  57                 let res_lane = match (lane_ty.kind(), intrinsic) {
  58                     (ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
  59                     (ty::Uint(_), sym::simd_ne) => {
  60                         fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane)
  61                     }
  62                     (ty::Uint(_), sym::simd_lt) => {
  63                         fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
  64                     }
  65                     (ty::Uint(_), sym::simd_le) => {
  66                         fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane)
  67                     }
  68                     (ty::Uint(_), sym::simd_gt) => {
  69                         fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane)
  70                     }
  71                     (ty::Uint(_), sym::simd_ge) => {
  72                         fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane)
  73                     }
  74
  75                     (ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
  76                     (ty::Int(_), sym::simd_ne) => {
  77                         fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane)
  78                     }
  79                     (ty::Int(_), sym::simd_lt) => {
  80                         fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane)
  81                     }
  82                     (ty::Int(_), sym::simd_le) => {
  83                         fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
  84                     }
  85                     (ty::Int(_), sym::simd_gt) => {
  86                         fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane)
  87                     }
  88                     (ty::Int(_), sym::simd_ge) => {
  89                         fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
  90                     }
  91
  92                     (ty::Float(_), sym::simd_eq) => {
  93                         fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane)
  94                     }
  95                     (ty::Float(_), sym::simd_ne) => {
  96                         fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane)
  97                     }
  98                     (ty::Float(_), sym::simd_lt) => {
  99                         fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane)
 100                     }
 101                     (ty::Float(_), sym::simd_le) => {
 102                         fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
 103                     }
 104                     (ty::Float(_), sym::simd_gt) => {
 105                         fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane)
 106                     }
 107                     (ty::Float(_), sym::simd_ge) => {
 108                         fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
 109                     }
 110
 111                     _ => unreachable!(),
 112                 };
 113
 114                 let ty = fx.clif_type(res_lane_ty).unwrap();
 115
 116                 let res_lane = fx.bcx.ins().bint(ty, res_lane);
 117                 fx.bcx.ins().ineg(res_lane)
 118             });
 119         }
 120
 121         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
 122         _ if intrinsic.as_str().starts_with("simd_shuffle") => {
 123             let (x, y, idx) = match args {
 124                 [x, y, idx] => (x, y, idx),
 125                 _ => {
 126                     bug!("wrong number of args for intrinsic {intrinsic}");
 127                 }
 128             };
 129             let x = codegen_operand(fx, x);
 130             let y = codegen_operand(fx, y);
 131
 132             if !x.layout().ty.is_simd() {
 133                 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
 134                 return;
 135             }
 136
 137             // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
 138             // If there is no suffix, use the index array length.
 139             let n: u16 = if intrinsic == sym::simd_shuffle {
 140                 // Make sure this is actually an array, since typeck only checks the length-suffixed
 141                 // version of this intrinsic.
 142                 let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
 143                 match idx_ty.kind() {
 144                     ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => len
 145                         .try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
 146                         .unwrap_or_else(|| {
 147                             span_bug!(span, "could not evaluate shuffle index array length")
 148                         })
 149                         .try_into()
 150                         .unwrap(),
 151                     _ => {
 152                         fx.tcx.sess.span_err(
 153                             span,
 154                             &format!(
 155                                 "simd_shuffle index must be an array of `u32`, got `{}`",
 156                                 idx_ty,
 157                             ),
 158                         );
 159                         // Prevent verifier error
 160                         fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
 161                         return;
 162                     }
 163                 }
 164             } else {
 165                 intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
 166             };
 167
 168             assert_eq!(x.layout(), y.layout());
 169             let layout = x.layout();
 170
 171             let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
 172             let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
 173
 174             assert_eq!(lane_ty, ret_lane_ty);
 175             assert_eq!(u64::from(n), ret_lane_count);
 176
 177             let total_len = lane_count * 2;
 178
 179             let indexes = {
 180                 use rustc_middle::mir::interpret::*;
 181                 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx)
 182                     .expect("simd_shuffle* idx not const");
 183
 184                 let idx_bytes = match idx_const {
 185                     ConstValue::ByRef { alloc, offset } => {
 186                         let size = Size::from_bytes(
 187                             4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */
 188                         );
 189                         alloc
 190                             .inner()
 191                             .get_bytes_strip_provenance(fx, alloc_range(offset, size))
 192                             .unwrap()
 193                     }
 194                     _ => unreachable!("{:?}", idx_const),
 195                 };
 196
 197                 (0..ret_lane_count)
 198                     .map(|i| {
 199                         let i = usize::try_from(i).unwrap();
 200                         let idx = rustc_middle::mir::interpret::read_target_uint(
 201                             fx.tcx.data_layout.endian,
 202                             &idx_bytes[4 * i..4 * i + 4],
 203                         )
 204                         .expect("read_target_uint");
 205                         u16::try_from(idx).expect("try_from u32")
 206                     })
 207                     .collect::<Vec<u16>>()
 208             };
 209
 210             for &idx in &indexes {
 211                 assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
 212             }
 213
 214             for (out_idx, in_idx) in indexes.into_iter().enumerate() {
 215                 let in_lane = if u64::from(in_idx) < lane_count {
 216                     x.value_lane(fx, in_idx.into())
 217                 } else {
 218                     y.value_lane(fx, u64::from(in_idx) - lane_count)
 219                 };
 220                 let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap());
 221                 out_lane.write_cvalue(fx, in_lane);
 222             }
 223         }
 224
 225         sym::simd_insert => {
 226             let (base, idx, val) = match args {
 227                 [base, idx, val] => (base, idx, val),
 228                 _ => {
 229                     bug!("wrong number of args for intrinsic {intrinsic}");
 230                 }
 231             };
 232             let base = codegen_operand(fx, base);
 233             let val = codegen_operand(fx, val);
 234
 235             // FIXME validate
 236             let idx_const = if let Some(idx_const) =
 237                 crate::constant::mir_operand_get_const_val(fx, idx)
 238             {
 239                 idx_const
 240             } else {
 241                 fx.tcx.sess.span_fatal(span, "Index argument for `simd_insert` is not a constant");
 242             };
 243
 244             let idx = idx_const
 245                 .try_to_bits(Size::from_bytes(4 /* u32*/))
 246                 .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
 247             let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
 248             if idx >= lane_count.into() {
 249                 fx.tcx.sess.span_fatal(
 250                     fx.mir.span,
 251                     &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count),
 252                 );
 253             }
 254
 255             ret.write_cvalue(fx, base);
 256             let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
 257             ret_lane.write_cvalue(fx, val);
 258         }
 259
 260         sym::simd_extract => {
 261             let (v, idx) = match args {
 262                 [v, idx] => (v, idx),
 263                 _ => {
 264                     bug!("wrong number of args for intrinsic {intrinsic}");
 265                 }
 266             };
 267             let v = codegen_operand(fx, v);
 268
 269             if !v.layout().ty.is_simd() {
 270                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 271                 return;
 272             }
 273
 274             let idx_const = if let Some(idx_const) =
 275                 crate::constant::mir_operand_get_const_val(fx, idx)
 276             {
 277                 idx_const
 278             } else {
 279                 fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
 280                 let trap_block = fx.bcx.create_block();
 281                 let dummy_block = fx.bcx.create_block();
 282                 let true_ = fx.bcx.ins().iconst(types::I8, 1);
 283                 fx.bcx.ins().brnz(true_, trap_block, &[]);
 284                 fx.bcx.ins().jump(dummy_block, &[]);
 285                 fx.bcx.switch_to_block(trap_block);
 286                 crate::trap::trap_unimplemented(
 287                     fx,
 288                     "Index argument for `simd_extract` is not a constant",
 289                 );
 290                 fx.bcx.switch_to_block(dummy_block);
 291                 return;
 292             };
 293
 294             let idx = idx_const
 295                 .try_to_bits(Size::from_bytes(4 /* u32*/))
 296                 .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
 297             let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
 298             if idx >= lane_count.into() {
 299                 fx.tcx.sess.span_fatal(
 300                     fx.mir.span,
 301                     &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count),
 302                 );
 303             }
 304
 305             let ret_lane = v.value_lane(fx, idx.try_into().unwrap());
 306             ret.write_cvalue(fx, ret_lane);
 307         }
 308
 309         sym::simd_neg => {
 310             intrinsic_args!(fx, args => (a); intrinsic);
 311
 312             if !a.layout().ty.is_simd() {
 313                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 314                 return;
 315             }
 316
 317             simd_for_each_lane(
 318                 fx,
 319                 a,
 320                 ret,
 321                 &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
 322                     ty::Int(_) => fx.bcx.ins().ineg(lane),
 323                     ty::Float(_) => fx.bcx.ins().fneg(lane),
 324                     _ => unreachable!(),
 325                 },
 326             );
 327         }
 328
 329         sym::simd_add
 330         | sym::simd_sub
 331         | sym::simd_mul
 332         | sym::simd_div
 333         | sym::simd_rem
 334         | sym::simd_shl
 335         | sym::simd_shr
 336         | sym::simd_and
 337         | sym::simd_or
 338         | sym::simd_xor => {
 339             intrinsic_args!(fx, args => (x, y); intrinsic);
 340
 341             // FIXME use vector instructions when possible
 342             simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
 343                 match (lane_ty.kind(), intrinsic) {
 344                     (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
 345                     (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
 346                     (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
 347                     (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
 348                     (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
 349
 350                     (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
 351                     (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
 352                     (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
 353                     (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
 354                     (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
 355
 356                     (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
 357                     (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
 358                     (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
 359                     (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
 360                     (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
 361                         "fmodf",
 362                         vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
 363                         vec![AbiParam::new(types::F32)],
 364                         &[x_lane, y_lane],
 365                     )[0],
 366                     (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
 367                         "fmod",
 368                         vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
 369                         vec![AbiParam::new(types::F64)],
 370                         &[x_lane, y_lane],
 371                     )[0],
 372
 373                     (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
 374                     (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
 375                     (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
 376                     (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
 377                     (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
 378
 379                     (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
 380                     (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
 381                     (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
 382                     (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
 383                     (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
 384
 385                     _ => unreachable!(),
 386                 }
 387             });
 388         }
 389
 390         sym::simd_fma => {
 391             intrinsic_args!(fx, args => (a, b, c); intrinsic);
 392
 393             if !a.layout().ty.is_simd() {
 394                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 395                 return;
 396             }
 397             assert_eq!(a.layout(), b.layout());
 398             assert_eq!(a.layout(), c.layout());
 399             assert_eq!(a.layout(), ret.layout());
 400
 401             let layout = a.layout();
 402             let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
 403             let res_lane_layout = fx.layout_of(lane_ty);
 404
 405             for lane in 0..lane_count {
 406                 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
 407                 let b_lane = b.value_lane(fx, lane).load_scalar(fx);
 408                 let c_lane = c.value_lane(fx, lane).load_scalar(fx);
 409
 410                 let res_lane = fx.bcx.ins().fma(a_lane, b_lane, c_lane);
 411                 let res_lane = CValue::by_val(res_lane, res_lane_layout);
 412
 413                 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
 414             }
 415         }
 416
 417         sym::simd_fmin | sym::simd_fmax => {
 418             intrinsic_args!(fx, args => (x, y); intrinsic);
 419
 420             if !x.layout().ty.is_simd() {
 421                 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
 422                 return;
 423             }
 424
 425             // FIXME use vector instructions when possible
 426             simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
 427                 match lane_ty.kind() {
 428                     ty::Float(_) => {}
 429                     _ => unreachable!("{:?}", lane_ty),
 430                 }
 431                 match intrinsic {
 432                     sym::simd_fmin => crate::num::codegen_float_min(fx, x_lane, y_lane),
 433                     sym::simd_fmax => crate::num::codegen_float_max(fx, x_lane, y_lane),
 434                     _ => unreachable!(),
 435                 }
 436             });
 437         }
 438
 439         sym::simd_round => {
 440             intrinsic_args!(fx, args => (a); intrinsic);
 441
 442             if !a.layout().ty.is_simd() {
 443                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 444                 return;
 445             }
 446
 447             simd_for_each_lane(
 448                 fx,
 449                 a,
 450                 ret,
 451                 &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
 452                     ty::Float(FloatTy::F32) => fx.lib_call(
 453                         "roundf",
 454                         vec![AbiParam::new(types::F32)],
 455                         vec![AbiParam::new(types::F32)],
 456                         &[lane],
 457                     )[0],
 458                     ty::Float(FloatTy::F64) => fx.lib_call(
 459                         "round",
 460                         vec![AbiParam::new(types::F64)],
 461                         vec![AbiParam::new(types::F64)],
 462                         &[lane],
 463                     )[0],
 464                     _ => unreachable!("{:?}", lane_ty),
 465                 },
 466             );
 467         }
 468
 469         sym::simd_fabs | sym::simd_fsqrt | sym::simd_ceil | sym::simd_floor | sym::simd_trunc => {
 470             intrinsic_args!(fx, args => (a); intrinsic);
 471
 472             if !a.layout().ty.is_simd() {
 473                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 474                 return;
 475             }
 476
 477             simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
 478                 match lane_ty.kind() {
 479                     ty::Float(_) => {}
 480                     _ => unreachable!("{:?}", lane_ty),
 481                 }
 482                 match intrinsic {
 483                     sym::simd_fabs => fx.bcx.ins().fabs(lane),
 484                     sym::simd_fsqrt => fx.bcx.ins().sqrt(lane),
 485                     sym::simd_ceil => fx.bcx.ins().ceil(lane),
 486                     sym::simd_floor => fx.bcx.ins().floor(lane),
 487                     sym::simd_trunc => fx.bcx.ins().trunc(lane),
 488                     _ => unreachable!(),
 489                 }
 490             });
 491         }
 492
 493         sym::simd_reduce_add_ordered | sym::simd_reduce_add_unordered => {
 494             intrinsic_args!(fx, args => (v, acc); intrinsic);
 495             let acc = acc.load_scalar(fx);
 496
 497             // FIXME there must be no acc param for integer vectors
 498             if !v.layout().ty.is_simd() {
 499                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 500                 return;
 501             }
 502
 503             simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
 504                 if lane_ty.is_floating_point() {
 505                     fx.bcx.ins().fadd(a, b)
 506                 } else {
 507                     fx.bcx.ins().iadd(a, b)
 508                 }
 509             });
 510         }
 511
 512         sym::simd_reduce_mul_ordered | sym::simd_reduce_mul_unordered => {
 513             intrinsic_args!(fx, args => (v, acc); intrinsic);
 514             let acc = acc.load_scalar(fx);
 515
 516             // FIXME there must be no acc param for integer vectors
 517             if !v.layout().ty.is_simd() {
 518                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 519                 return;
 520             }
 521
 522             simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
 523                 if lane_ty.is_floating_point() {
 524                     fx.bcx.ins().fmul(a, b)
 525                 } else {
 526                     fx.bcx.ins().imul(a, b)
 527                 }
 528             });
 529         }
 530
 531         sym::simd_reduce_all => {
 532             intrinsic_args!(fx, args => (v); intrinsic);
 533
 534             if !v.layout().ty.is_simd() {
 535                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 536                 return;
 537             }
 538
 539             simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
 540         }
 541
 542         sym::simd_reduce_any => {
 543             intrinsic_args!(fx, args => (v); intrinsic);
 544
 545             if !v.layout().ty.is_simd() {
 546                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 547                 return;
 548             }
 549
 550             simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
 551         }
 552
 553         sym::simd_reduce_and => {
 554             intrinsic_args!(fx, args => (v); intrinsic);
 555
 556             if !v.layout().ty.is_simd() {
 557                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 558                 return;
 559             }
 560
 561             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
 562         }
 563
 564         sym::simd_reduce_or => {
 565             intrinsic_args!(fx, args => (v); intrinsic);
 566
 567             if !v.layout().ty.is_simd() {
 568                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 569                 return;
 570             }
 571
 572             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
 573         }
 574
 575         sym::simd_reduce_xor => {
 576             intrinsic_args!(fx, args => (v); intrinsic);
 577
 578             if !v.layout().ty.is_simd() {
 579                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 580                 return;
 581             }
 582
 583             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
 584         }
 585
 586         sym::simd_reduce_min => {
 587             intrinsic_args!(fx, args => (v); intrinsic);
 588
 589             if !v.layout().ty.is_simd() {
 590                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 591                 return;
 592             }
 593
 594             simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
 595                 let lt = match ty.kind() {
 596                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
 597                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
 598                     ty::Float(_) => return crate::num::codegen_float_min(fx, a, b),
 599                     _ => unreachable!(),
 600                 };
 601                 fx.bcx.ins().select(lt, a, b)
 602             });
 603         }
 604
 605         sym::simd_reduce_max => {
 606             intrinsic_args!(fx, args => (v); intrinsic);
 607
 608             if !v.layout().ty.is_simd() {
 609                 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
 610                 return;
 611             }
 612
 613             simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
 614                 let gt = match ty.kind() {
 615                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
 616                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
 617                     ty::Float(_) => return crate::num::codegen_float_max(fx, a, b),
 618                     _ => unreachable!(),
 619                 };
 620                 fx.bcx.ins().select(gt, a, b)
 621             });
 622         }
 623
 624         sym::simd_select => {
 625             intrinsic_args!(fx, args => (m, a, b); intrinsic);
 626
 627             if !m.layout().ty.is_simd() {
 628                 report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
 629                 return;
 630             }
 631             if !a.layout().ty.is_simd() {
 632                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
 633                 return;
 634             }
 635             assert_eq!(a.layout(), b.layout());
 636
 637             let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
 638             let lane_layout = fx.layout_of(lane_ty);
 639
 640             for lane in 0..lane_count {
 641                 let m_lane = m.value_lane(fx, lane).load_scalar(fx);
 642                 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
 643                 let b_lane = b.value_lane(fx, lane).load_scalar(fx);
 644
 645                 let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
 646                 let res_lane =
 647                     CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
 648
 649                 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
 650             }
 651         }
 652
 653         // simd_saturating_*
 654         // simd_bitmask
 655         // simd_scatter
 656         // simd_gather
 657         _ => {
 658             fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
 659         }
 660     }
 661 }