src/shims/intrinsics.rs

   1 use std::iter;
   2
   3 use log::trace;
   4
   5 use rustc_apfloat::{Float, Round};
   6 use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf};
   7 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
   8 use rustc_target::abi::{Align, Endian, HasDataLayout, Integer, Size};
   9
  10 use crate::*;
  11 use helpers::check_arg_count;
  12
  13 pub enum AtomicOp {
  14     MirOp(mir::BinOp, bool),
  15     Max,
  16     Min,
  17 }
  18
  19 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  20 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  21     fn call_intrinsic(
  22         &mut self,
  23         instance: ty::Instance<'tcx>,
  24         args: &[OpTy<'tcx, Tag>],
  25         ret: Option<(&PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  26         _unwind: StackPopUnwind,
  27     ) -> InterpResult<'tcx> {
  28         let this = self.eval_context_mut();
  29
  30         if this.emulate_intrinsic(instance, args, ret)? {
  31             return Ok(());
  32         }
  33
  34         // All supported intrinsics have a return place.
  35         let intrinsic_name = this.tcx.item_name(instance.def_id());
  36         let intrinsic_name = intrinsic_name.as_str();
  37         let (dest, ret) = match ret {
  38             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  39             Some(p) => p,
  40         };
  41
  42         // Then handle terminating intrinsics.
  43         match intrinsic_name {
  44             // Miri overwriting CTFE intrinsics.
  45             "ptr_guaranteed_eq" => {
  46                 let &[ref left, ref right] = check_arg_count(args)?;
  47                 let left = this.read_immediate(left)?;
  48                 let right = this.read_immediate(right)?;
  49                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  50             }
  51             "ptr_guaranteed_ne" => {
  52                 let &[ref left, ref right] = check_arg_count(args)?;
  53                 let left = this.read_immediate(left)?;
  54                 let right = this.read_immediate(right)?;
  55                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  56             }
  57             "const_allocate" => {
  58                 // For now, for compatibility with the run-time implementation of this, we just return null.
  59                 // See <https://github.com/rust-lang/rust/issues/93935>.
  60                 this.write_null(dest)?;
  61             }
  62             "const_deallocate" => {
  63                 // complete NOP
  64             }
  65
  66             // Raw memory accesses
  67             "volatile_load" => {
  68                 let &[ref place] = check_arg_count(args)?;
  69                 let place = this.deref_operand(place)?;
  70                 this.copy_op(&place.into(), dest)?;
  71             }
  72             "volatile_store" => {
  73                 let &[ref place, ref dest] = check_arg_count(args)?;
  74                 let place = this.deref_operand(place)?;
  75                 this.copy_op(dest, &place.into())?;
  76             }
  77
  78             "write_bytes" | "volatile_set_memory" => {
  79                 let &[ref ptr, ref val_byte, ref count] = check_arg_count(args)?;
  80                 let ty = instance.substs.type_at(0);
  81                 let ty_layout = this.layout_of(ty)?;
  82                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
  83                 let ptr = this.read_pointer(ptr)?;
  84                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  85                 // `checked_mul` enforces a too small bound (the correct one would probably be machine_isize_max),
  86                 // but no actual allocation can be big enough for the difference to be noticeable.
  87                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
  88                     err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
  89                 })?;
  90                 this.write_bytes_ptr(
  91                     ptr,
  92                     iter::repeat(val_byte).take(byte_count.bytes() as usize),
  93                 )?;
  94             }
  95
  96             // Floating-point operations
  97             "fabsf32" => {
  98                 let &[ref f] = check_arg_count(args)?;
  99                 let f = this.read_scalar(f)?.to_f32()?;
 100                 // Can be implemented in soft-floats.
 101                 this.write_scalar(Scalar::from_f32(f.abs()), dest)?;
 102             }
 103             "fabsf64" => {
 104                 let &[ref f] = check_arg_count(args)?;
 105                 let f = this.read_scalar(f)?.to_f64()?;
 106                 // Can be implemented in soft-floats.
 107                 this.write_scalar(Scalar::from_f64(f.abs()), dest)?;
 108             }
 109             #[rustfmt::skip]
 110             | "sinf32"
 111             | "cosf32"
 112             | "sqrtf32"
 113             | "expf32"
 114             | "exp2f32"
 115             | "logf32"
 116             | "log10f32"
 117             | "log2f32"
 118             | "floorf32"
 119             | "ceilf32"
 120             | "truncf32"
 121             | "roundf32"
 122             => {
 123                 let &[ref f] = check_arg_count(args)?;
 124                 // FIXME: Using host floats.
 125                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 126                 let f = match intrinsic_name {
 127                     "sinf32" => f.sin(),
 128                     "cosf32" => f.cos(),
 129                     "sqrtf32" => f.sqrt(),
 130                     "expf32" => f.exp(),
 131                     "exp2f32" => f.exp2(),
 132                     "logf32" => f.ln(),
 133                     "log10f32" => f.log10(),
 134                     "log2f32" => f.log2(),
 135                     "floorf32" => f.floor(),
 136                     "ceilf32" => f.ceil(),
 137                     "truncf32" => f.trunc(),
 138                     "roundf32" => f.round(),
 139                     _ => bug!(),
 140                 };
 141                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 142             }
 143
 144             #[rustfmt::skip]
 145             | "sinf64"
 146             | "cosf64"
 147             | "sqrtf64"
 148             | "expf64"
 149             | "exp2f64"
 150             | "logf64"
 151             | "log10f64"
 152             | "log2f64"
 153             | "floorf64"
 154             | "ceilf64"
 155             | "truncf64"
 156             | "roundf64"
 157             => {
 158                 let &[ref f] = check_arg_count(args)?;
 159                 // FIXME: Using host floats.
 160                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 161                 let f = match intrinsic_name {
 162                     "sinf64" => f.sin(),
 163                     "cosf64" => f.cos(),
 164                     "sqrtf64" => f.sqrt(),
 165                     "expf64" => f.exp(),
 166                     "exp2f64" => f.exp2(),
 167                     "logf64" => f.ln(),
 168                     "log10f64" => f.log10(),
 169                     "log2f64" => f.log2(),
 170                     "floorf64" => f.floor(),
 171                     "ceilf64" => f.ceil(),
 172                     "truncf64" => f.trunc(),
 173                     "roundf64" => f.round(),
 174                     _ => bug!(),
 175                 };
 176                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 177             }
 178
 179             #[rustfmt::skip]
 180             | "fadd_fast"
 181             | "fsub_fast"
 182             | "fmul_fast"
 183             | "fdiv_fast"
 184             | "frem_fast"
 185             => {
 186                 let &[ref a, ref b] = check_arg_count(args)?;
 187                 let a = this.read_immediate(a)?;
 188                 let b = this.read_immediate(b)?;
 189                 let op = match intrinsic_name {
 190                     "fadd_fast" => mir::BinOp::Add,
 191                     "fsub_fast" => mir::BinOp::Sub,
 192                     "fmul_fast" => mir::BinOp::Mul,
 193                     "fdiv_fast" => mir::BinOp::Div,
 194                     "frem_fast" => mir::BinOp::Rem,
 195                     _ => bug!(),
 196                 };
 197                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 198                     Ok(match x.layout.ty.kind() {
 199                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 200                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 201                         _ => bug!(
 202                             "`{}` called with non-float input type {:?}",
 203                             intrinsic_name,
 204                             x.layout.ty
 205                         ),
 206                     })
 207                 };
 208                 match (float_finite(a)?, float_finite(b)?) {
 209                     (false, false) => throw_ub_format!(
 210                         "`{}` intrinsic called with non-finite value as both parameters",
 211                         intrinsic_name,
 212                     ),
 213                     (false, _) => throw_ub_format!(
 214                         "`{}` intrinsic called with non-finite value as first parameter",
 215                         intrinsic_name,
 216                     ),
 217                     (_, false) => throw_ub_format!(
 218                         "`{}` intrinsic called with non-finite value as second parameter",
 219                         intrinsic_name,
 220                     ),
 221                     _ => {}
 222                 }
 223                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 224             }
 225
 226             #[rustfmt::skip]
 227             | "minnumf32"
 228             | "maxnumf32"
 229             | "copysignf32"
 230             => {
 231                 let &[ref a, ref b] = check_arg_count(args)?;
 232                 let a = this.read_scalar(a)?.to_f32()?;
 233                 let b = this.read_scalar(b)?.to_f32()?;
 234                 let res = match intrinsic_name {
 235                     "minnumf32" => a.min(b),
 236                     "maxnumf32" => a.max(b),
 237                     "copysignf32" => a.copy_sign(b),
 238                     _ => bug!(),
 239                 };
 240                 this.write_scalar(Scalar::from_f32(res), dest)?;
 241             }
 242
 243             #[rustfmt::skip]
 244             | "minnumf64"
 245             | "maxnumf64"
 246             | "copysignf64"
 247             => {
 248                 let &[ref a, ref b] = check_arg_count(args)?;
 249                 let a = this.read_scalar(a)?.to_f64()?;
 250                 let b = this.read_scalar(b)?.to_f64()?;
 251                 let res = match intrinsic_name {
 252                     "minnumf64" => a.min(b),
 253                     "maxnumf64" => a.max(b),
 254                     "copysignf64" => a.copy_sign(b),
 255                     _ => bug!(),
 256                 };
 257                 this.write_scalar(Scalar::from_f64(res), dest)?;
 258             }
 259
 260             "powf32" => {
 261                 let &[ref f, ref f2] = check_arg_count(args)?;
 262                 // FIXME: Using host floats.
 263                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 264                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 265                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 266             }
 267
 268             "powf64" => {
 269                 let &[ref f, ref f2] = check_arg_count(args)?;
 270                 // FIXME: Using host floats.
 271                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 272                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 273                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 274             }
 275
 276             "fmaf32" => {
 277                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 278                 let a = this.read_scalar(a)?.to_f32()?;
 279                 let b = this.read_scalar(b)?.to_f32()?;
 280                 let c = this.read_scalar(c)?.to_f32()?;
 281                 let res = a.mul_add(b, c).value;
 282                 this.write_scalar(Scalar::from_f32(res), dest)?;
 283             }
 284
 285             "fmaf64" => {
 286                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 287                 let a = this.read_scalar(a)?.to_f64()?;
 288                 let b = this.read_scalar(b)?.to_f64()?;
 289                 let c = this.read_scalar(c)?.to_f64()?;
 290                 let res = a.mul_add(b, c).value;
 291                 this.write_scalar(Scalar::from_f64(res), dest)?;
 292             }
 293
 294             "powif32" => {
 295                 let &[ref f, ref i] = check_arg_count(args)?;
 296                 // FIXME: Using host floats.
 297                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 298                 let i = this.read_scalar(i)?.to_i32()?;
 299                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 300             }
 301
 302             "powif64" => {
 303                 let &[ref f, ref i] = check_arg_count(args)?;
 304                 // FIXME: Using host floats.
 305                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 306                 let i = this.read_scalar(i)?.to_i32()?;
 307                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 308             }
 309
 310             "float_to_int_unchecked" => {
 311                 let &[ref val] = check_arg_count(args)?;
 312                 let val = this.read_immediate(val)?;
 313
 314                 let res = match val.layout.ty.kind() {
 315                     ty::Float(FloatTy::F32) =>
 316                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 317                     ty::Float(FloatTy::F64) =>
 318                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 319                     _ =>
 320                         bug!(
 321                             "`float_to_int_unchecked` called with non-float input type {:?}",
 322                             val.layout.ty
 323                         ),
 324                 };
 325
 326                 this.write_scalar(res, dest)?;
 327             }
 328
 329             // SIMD operations
 330             #[rustfmt::skip]
 331             | "simd_neg"
 332             | "simd_fabs"
 333             | "simd_ceil"
 334             | "simd_floor"
 335             | "simd_round"
 336             | "simd_trunc"
 337             | "simd_fsqrt" => {
 338                 let &[ref op] = check_arg_count(args)?;
 339                 let (op, op_len) = this.operand_to_simd(op)?;
 340                 let (dest, dest_len) = this.place_to_simd(dest)?;
 341
 342                 assert_eq!(dest_len, op_len);
 343
 344                 #[derive(Copy, Clone)]
 345                 enum HostFloatOp {
 346                     Ceil,
 347                     Floor,
 348                     Round,
 349                     Trunc,
 350                     Sqrt,
 351                 }
 352                 #[derive(Copy, Clone)]
 353                 enum Op {
 354                     MirOp(mir::UnOp),
 355                     Abs,
 356                     HostOp(HostFloatOp),
 357                 }
 358                 let which = match intrinsic_name {
 359                     "simd_neg" => Op::MirOp(mir::UnOp::Neg),
 360                     "simd_fabs" => Op::Abs,
 361                     "simd_ceil" => Op::HostOp(HostFloatOp::Ceil),
 362                     "simd_floor" => Op::HostOp(HostFloatOp::Floor),
 363                     "simd_round" => Op::HostOp(HostFloatOp::Round),
 364                     "simd_trunc" => Op::HostOp(HostFloatOp::Trunc),
 365                     "simd_fsqrt" => Op::HostOp(HostFloatOp::Sqrt),
 366                     _ => unreachable!(),
 367                 };
 368
 369                 for i in 0..dest_len {
 370                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 371                     let dest = this.mplace_index(&dest, i)?;
 372                     let val = match which {
 373                         Op::MirOp(mir_op) => this.unary_op(mir_op, &op)?.to_scalar()?,
 374                         Op::Abs => {
 375                             // Works for f32 and f64.
 376                             let ty::Float(float_ty) = op.layout.ty.kind() else {
 377                                 bug!("{} operand is not a float", intrinsic_name)
 378                             };
 379                             let op = op.to_scalar()?;
 380                             match float_ty {
 381                                 FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()),
 382                                 FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()),
 383                             }
 384                         }
 385                         Op::HostOp(host_op) => {
 386                             let ty::Float(float_ty) = op.layout.ty.kind() else {
 387                                 bug!("{} operand is not a float", intrinsic_name)
 388                             };
 389                             // FIXME using host floats
 390                             match float_ty {
 391                                 FloatTy::F32 => {
 392                                     let f = f32::from_bits(op.to_scalar()?.to_u32()?);
 393                                     let res = match host_op {
 394                                         HostFloatOp::Ceil => f.ceil(),
 395                                         HostFloatOp::Floor => f.floor(),
 396                                         HostFloatOp::Round => f.round(),
 397                                         HostFloatOp::Trunc => f.trunc(),
 398                                         HostFloatOp::Sqrt => f.sqrt(),
 399                                     };
 400                                     Scalar::from_u32(res.to_bits())
 401                                 }
 402                                 FloatTy::F64 => {
 403                                     let f = f64::from_bits(op.to_scalar()?.to_u64()?);
 404                                     let res = match host_op {
 405                                         HostFloatOp::Ceil => f.ceil(),
 406                                         HostFloatOp::Floor => f.floor(),
 407                                         HostFloatOp::Round => f.round(),
 408                                         HostFloatOp::Trunc => f.trunc(),
 409                                         HostFloatOp::Sqrt => f.sqrt(),
 410                                     };
 411                                     Scalar::from_u64(res.to_bits())
 412                                 }
 413                             }
 414
 415                         }
 416                     };
 417                     this.write_scalar(val, &dest.into())?;
 418                 }
 419             }
 420             #[rustfmt::skip]
 421             | "simd_add"
 422             | "simd_sub"
 423             | "simd_mul"
 424             | "simd_div"
 425             | "simd_rem"
 426             | "simd_shl"
 427             | "simd_shr"
 428             | "simd_and"
 429             | "simd_or"
 430             | "simd_xor"
 431             | "simd_eq"
 432             | "simd_ne"
 433             | "simd_lt"
 434             | "simd_le"
 435             | "simd_gt"
 436             | "simd_ge"
 437             | "simd_fmax"
 438             | "simd_fmin"
 439             | "simd_saturating_add"
 440             | "simd_saturating_sub" => {
 441                 use mir::BinOp;
 442
 443                 let &[ref left, ref right] = check_arg_count(args)?;
 444                 let (left, left_len) = this.operand_to_simd(left)?;
 445                 let (right, right_len) = this.operand_to_simd(right)?;
 446                 let (dest, dest_len) = this.place_to_simd(dest)?;
 447
 448                 assert_eq!(dest_len, left_len);
 449                 assert_eq!(dest_len, right_len);
 450
 451                 enum Op {
 452                     MirOp(BinOp),
 453                     SaturatingOp(BinOp),
 454                     FMax,
 455                     FMin,
 456                 }
 457                 let which = match intrinsic_name {
 458                     "simd_add" => Op::MirOp(BinOp::Add),
 459                     "simd_sub" => Op::MirOp(BinOp::Sub),
 460                     "simd_mul" => Op::MirOp(BinOp::Mul),
 461                     "simd_div" => Op::MirOp(BinOp::Div),
 462                     "simd_rem" => Op::MirOp(BinOp::Rem),
 463                     "simd_shl" => Op::MirOp(BinOp::Shl),
 464                     "simd_shr" => Op::MirOp(BinOp::Shr),
 465                     "simd_and" => Op::MirOp(BinOp::BitAnd),
 466                     "simd_or" => Op::MirOp(BinOp::BitOr),
 467                     "simd_xor" => Op::MirOp(BinOp::BitXor),
 468                     "simd_eq" => Op::MirOp(BinOp::Eq),
 469                     "simd_ne" => Op::MirOp(BinOp::Ne),
 470                     "simd_lt" => Op::MirOp(BinOp::Lt),
 471                     "simd_le" => Op::MirOp(BinOp::Le),
 472                     "simd_gt" => Op::MirOp(BinOp::Gt),
 473                     "simd_ge" => Op::MirOp(BinOp::Ge),
 474                     "simd_fmax" => Op::FMax,
 475                     "simd_fmin" => Op::FMin,
 476                     "simd_saturating_add" => Op::SaturatingOp(BinOp::Add),
 477                     "simd_saturating_sub" => Op::SaturatingOp(BinOp::Sub),
 478                     _ => unreachable!(),
 479                 };
 480
 481                 for i in 0..dest_len {
 482                     let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
 483                     let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
 484                     let dest = this.mplace_index(&dest, i)?;
 485                     let val = match which {
 486                         Op::MirOp(mir_op) => {
 487                             let (val, overflowed, ty) = this.overflowing_binary_op(mir_op, &left, &right)?;
 488                             if matches!(mir_op, BinOp::Shl | BinOp::Shr) {
 489                                 // Shifts have extra UB as SIMD operations that the MIR binop does not have.
 490                                 // See <https://github.com/rust-lang/rust/issues/91237>.
 491                                 if overflowed {
 492                                     let r_val = right.to_scalar()?.to_bits(right.layout.size)?;
 493                                     throw_ub_format!("overflowing shift by {} in `{}` in SIMD lane {}", r_val, intrinsic_name, i);
 494                                 }
 495                             }
 496                             if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) {
 497                                 // Special handling for boolean-returning operations
 498                                 assert_eq!(ty, this.tcx.types.bool);
 499                                 let val = val.to_bool().unwrap();
 500                                 bool_to_simd_element(val, dest.layout.size)
 501                             } else {
 502                                 assert_ne!(ty, this.tcx.types.bool);
 503                                 assert_eq!(ty, dest.layout.ty);
 504                                 val
 505                             }
 506                         }
 507                         Op::FMax => {
 508                             fmax_op(&left, &right)?
 509                         }
 510                         Op::FMin => {
 511                             fmin_op(&left, &right)?
 512                         }
 513                         Op::SaturatingOp(mir_op) => {
 514                             this.saturating_arith(mir_op, &left, &right)?
 515                         }
 516                     };
 517                     this.write_scalar(val, &dest.into())?;
 518                 }
 519             }
 520             "simd_fma" => {
 521                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 522                 let (a, a_len) = this.operand_to_simd(a)?;
 523                 let (b, b_len) = this.operand_to_simd(b)?;
 524                 let (c, c_len) = this.operand_to_simd(c)?;
 525                 let (dest, dest_len) = this.place_to_simd(dest)?;
 526
 527                 assert_eq!(dest_len, a_len);
 528                 assert_eq!(dest_len, b_len);
 529                 assert_eq!(dest_len, c_len);
 530
 531                 for i in 0..dest_len {
 532                     let a = this.read_immediate(&this.mplace_index(&a, i)?.into())?.to_scalar()?;
 533                     let b = this.read_immediate(&this.mplace_index(&b, i)?.into())?.to_scalar()?;
 534                     let c = this.read_immediate(&this.mplace_index(&c, i)?.into())?.to_scalar()?;
 535                     let dest = this.mplace_index(&dest, i)?;
 536
 537                     // Works for f32 and f64.
 538                     let ty::Float(float_ty) = dest.layout.ty.kind() else {
 539                         bug!("{} operand is not a float", intrinsic_name)
 540                     };
 541                     let val = match float_ty {
 542                         FloatTy::F32 =>
 543                             Scalar::from_f32(a.to_f32()?.mul_add(b.to_f32()?, c.to_f32()?).value),
 544                         FloatTy::F64 =>
 545                             Scalar::from_f64(a.to_f64()?.mul_add(b.to_f64()?, c.to_f64()?).value),
 546                     };
 547                     this.write_scalar(val, &dest.into())?;
 548                 }
 549             }
 550             #[rustfmt::skip]
 551             | "simd_reduce_and"
 552             | "simd_reduce_or"
 553             | "simd_reduce_xor"
 554             | "simd_reduce_any"
 555             | "simd_reduce_all"
 556             | "simd_reduce_max"
 557             | "simd_reduce_min" => {
 558                 use mir::BinOp;
 559
 560                 let &[ref op] = check_arg_count(args)?;
 561                 let (op, op_len) = this.operand_to_simd(op)?;
 562
 563                 let imm_from_bool =
 564                     |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool);
 565
 566                 enum Op {
 567                     MirOp(BinOp),
 568                     MirOpBool(BinOp),
 569                     Max,
 570                     Min,
 571                 }
 572                 let which = match intrinsic_name {
 573                     "simd_reduce_and" => Op::MirOp(BinOp::BitAnd),
 574                     "simd_reduce_or" => Op::MirOp(BinOp::BitOr),
 575                     "simd_reduce_xor" => Op::MirOp(BinOp::BitXor),
 576                     "simd_reduce_any" => Op::MirOpBool(BinOp::BitOr),
 577                     "simd_reduce_all" => Op::MirOpBool(BinOp::BitAnd),
 578                     "simd_reduce_max" => Op::Max,
 579                     "simd_reduce_min" => Op::Min,
 580                     _ => unreachable!(),
 581                 };
 582
 583                 // Initialize with first lane, then proceed with the rest.
 584                 let mut res = this.read_immediate(&this.mplace_index(&op, 0)?.into())?;
 585                 if matches!(which, Op::MirOpBool(_)) {
 586                     // Convert to `bool` scalar.
 587                     res = imm_from_bool(simd_element_to_bool(res)?);
 588                 }
 589                 for i in 1..op_len {
 590                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 591                     res = match which {
 592                         Op::MirOp(mir_op) => {
 593                             this.binary_op(mir_op, &res, &op)?
 594                         }
 595                         Op::MirOpBool(mir_op) => {
 596                             let op = imm_from_bool(simd_element_to_bool(op)?);
 597                             this.binary_op(mir_op, &res, &op)?
 598                         }
 599                         Op::Max => {
 600                             if matches!(res.layout.ty.kind(), ty::Float(_)) {
 601                                 ImmTy::from_scalar(fmax_op(&res, &op)?, res.layout)
 602                             } else {
 603                                 // Just boring integers, so NaNs to worry about
 604                                 if this.binary_op(BinOp::Ge, &res, &op)?.to_scalar()?.to_bool()? {
 605                                     res
 606                                 } else {
 607                                     op
 608                                 }
 609                             }
 610                         }
 611                         Op::Min => {
 612                             if matches!(res.layout.ty.kind(), ty::Float(_)) {
 613                                 ImmTy::from_scalar(fmin_op(&res, &op)?, res.layout)
 614                             } else {
 615                                 // Just boring integers, so NaNs to worry about
 616                                 if this.binary_op(BinOp::Le, &res, &op)?.to_scalar()?.to_bool()? {
 617                                     res
 618                                 } else {
 619                                     op
 620                                 }
 621                             }
 622                         }
 623                     };
 624                 }
 625                 this.write_immediate(*res, dest)?;
 626             }
 627             #[rustfmt::skip]
 628             | "simd_reduce_add_ordered"
 629             | "simd_reduce_mul_ordered" => {
 630                 use mir::BinOp;
 631
 632                 let &[ref op, ref init] = check_arg_count(args)?;
 633                 let (op, op_len) = this.operand_to_simd(op)?;
 634                 let init = this.read_immediate(init)?;
 635
 636                 let mir_op = match intrinsic_name {
 637                     "simd_reduce_add_ordered" => BinOp::Add,
 638                     "simd_reduce_mul_ordered" => BinOp::Mul,
 639                     _ => unreachable!(),
 640                 };
 641
 642                 let mut res = init;
 643                 for i in 0..op_len {
 644                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 645                     res = this.binary_op(mir_op, &res, &op)?;
 646                 }
 647                 this.write_immediate(*res, dest)?;
 648             }
 649             "simd_select" => {
 650                 let &[ref mask, ref yes, ref no] = check_arg_count(args)?;
 651                 let (mask, mask_len) = this.operand_to_simd(mask)?;
 652                 let (yes, yes_len) = this.operand_to_simd(yes)?;
 653                 let (no, no_len) = this.operand_to_simd(no)?;
 654                 let (dest, dest_len) = this.place_to_simd(dest)?;
 655
 656                 assert_eq!(dest_len, mask_len);
 657                 assert_eq!(dest_len, yes_len);
 658                 assert_eq!(dest_len, no_len);
 659
 660                 for i in 0..dest_len {
 661                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
 662                     let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
 663                     let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
 664                     let dest = this.mplace_index(&dest, i)?;
 665
 666                     let val = if simd_element_to_bool(mask)? { yes } else { no };
 667                     this.write_immediate(*val, &dest.into())?;
 668                 }
 669             }
 670             "simd_select_bitmask" => {
 671                 let &[ref mask, ref yes, ref no] = check_arg_count(args)?;
 672                 let (yes, yes_len) = this.operand_to_simd(yes)?;
 673                 let (no, no_len) = this.operand_to_simd(no)?;
 674                 let (dest, dest_len) = this.place_to_simd(dest)?;
 675                 let bitmask_len = dest_len.max(8);
 676
 677                 assert!(mask.layout.ty.is_integral());
 678                 assert!(bitmask_len <= 64);
 679                 assert_eq!(bitmask_len, mask.layout.size.bits());
 680                 assert_eq!(dest_len, yes_len);
 681                 assert_eq!(dest_len, no_len);
 682
 683                 let mask: u64 = this
 684                     .read_scalar(mask)?
 685                     .check_init()?
 686                     .to_bits(mask.layout.size)?
 687                     .try_into()
 688                     .unwrap();
 689                 for i in 0..dest_len {
 690                     let mask =
 691                         mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian));
 692                     let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
 693                     let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
 694                     let dest = this.mplace_index(&dest, i)?;
 695
 696                     let val = if mask != 0 { yes } else { no };
 697                     this.write_immediate(*val, &dest.into())?;
 698                 }
 699                 for i in dest_len..bitmask_len {
 700                     // If the mask is "padded", ensure that padding is all-zero.
 701                     let mask = mask & (1 << i);
 702                     if mask != 0 {
 703                         throw_ub_format!(
 704                             "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits"
 705                         );
 706                     }
 707                 }
 708             }
 709             #[rustfmt::skip]
 710             "simd_cast" | "simd_as" => {
 711                 let &[ref op] = check_arg_count(args)?;
 712                 let (op, op_len) = this.operand_to_simd(op)?;
 713                 let (dest, dest_len) = this.place_to_simd(dest)?;
 714
 715                 assert_eq!(dest_len, op_len);
 716
 717                 let safe_cast = intrinsic_name == "simd_as";
 718
 719                 for i in 0..dest_len {
 720                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 721                     let dest = this.mplace_index(&dest, i)?;
 722
 723                     let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) {
 724                         // Int-to-(int|float): always safe
 725                         (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) =>
 726                             this.misc_cast(&op, dest.layout.ty)?,
 727                         // Float-to-float: always safe
 728                         (ty::Float(_), ty::Float(_)) =>
 729                             this.misc_cast(&op, dest.layout.ty)?,
 730                         // Float-to-int in safe mode
 731                         (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast =>
 732                             this.misc_cast(&op, dest.layout.ty)?,
 733                         // Float-to-int in unchecked mode
 734                         (ty::Float(FloatTy::F32), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
 735                             this.float_to_int_unchecked(op.to_scalar()?.to_f32()?, dest.layout.ty)?.into(),
 736                         (ty::Float(FloatTy::F64), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
 737                             this.float_to_int_unchecked(op.to_scalar()?.to_f64()?, dest.layout.ty)?.into(),
 738                         _ =>
 739                             throw_unsup_format!(
 740                                 "Unsupported SIMD cast from element type {} to {}",
 741                                 op.layout.ty,
 742                                 dest.layout.ty
 743                             ),
 744                     };
 745                     this.write_immediate(val, &dest.into())?;
 746                 }
 747             }
 748             "simd_shuffle" => {
 749                 let &[ref left, ref right, ref index] = check_arg_count(args)?;
 750                 let (left, left_len) = this.operand_to_simd(left)?;
 751                 let (right, right_len) = this.operand_to_simd(right)?;
 752                 let (dest, dest_len) = this.place_to_simd(dest)?;
 753
 754                 // `index` is an array, not a SIMD type
 755                 let ty::Array(_, index_len) = index.layout.ty.kind() else {
 756                     bug!("simd_shuffle index argument has non-array type {}", index.layout.ty)
 757                 };
 758                 let index_len = index_len.eval_usize(*this.tcx, this.param_env());
 759
 760                 assert_eq!(left_len, right_len);
 761                 assert_eq!(index_len, dest_len);
 762
 763                 for i in 0..dest_len {
 764                     let src_index: u64 = this
 765                         .read_immediate(&this.operand_index(&index, i)?.into())?
 766                         .to_scalar()?
 767                         .to_u32()?
 768                         .into();
 769                     let dest = this.mplace_index(&dest, i)?;
 770
 771                     let val = if src_index < left_len {
 772                         this.read_immediate(&this.mplace_index(&left, src_index)?.into())?
 773                     } else if src_index < left_len.checked_add(right_len).unwrap() {
 774                         this.read_immediate(
 775                             &this.mplace_index(&right, src_index - left_len)?.into(),
 776                         )?
 777                     } else {
 778                         bug!(
 779                             "simd_shuffle index {} is out of bounds for 2 vectors of size {}",
 780                             src_index,
 781                             left_len
 782                         );
 783                     };
 784                     this.write_immediate(*val, &dest.into())?;
 785                 }
 786             }
 787             "simd_gather" => {
 788                 let &[ref passthru, ref ptrs, ref mask] = check_arg_count(args)?;
 789                 let (passthru, passthru_len) = this.operand_to_simd(passthru)?;
 790                 let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
 791                 let (mask, mask_len) = this.operand_to_simd(mask)?;
 792                 let (dest, dest_len) = this.place_to_simd(dest)?;
 793
 794                 assert_eq!(dest_len, passthru_len);
 795                 assert_eq!(dest_len, ptrs_len);
 796                 assert_eq!(dest_len, mask_len);
 797
 798                 for i in 0..dest_len {
 799                     let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?;
 800                     let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
 801                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
 802                     let dest = this.mplace_index(&dest, i)?;
 803
 804                     let val = if simd_element_to_bool(mask)? {
 805                         let place = this.deref_operand(&ptr.into())?;
 806                         this.read_immediate(&place.into())?
 807                     } else {
 808                         passthru
 809                     };
 810                     this.write_immediate(*val, &dest.into())?;
 811                 }
 812             }
 813             "simd_scatter" => {
 814                 let &[ref value, ref ptrs, ref mask] = check_arg_count(args)?;
 815                 let (value, value_len) = this.operand_to_simd(value)?;
 816                 let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
 817                 let (mask, mask_len) = this.operand_to_simd(mask)?;
 818
 819                 assert_eq!(ptrs_len, value_len);
 820                 assert_eq!(ptrs_len, mask_len);
 821
 822                 for i in 0..ptrs_len {
 823                     let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?;
 824                     let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
 825                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
 826
 827                     if simd_element_to_bool(mask)? {
 828                         let place = this.deref_operand(&ptr.into())?;
 829                         this.write_immediate(*value, &place.into())?;
 830                     }
 831                 }
 832             }
 833             "simd_bitmask" => {
 834                 let &[ref op] = check_arg_count(args)?;
 835                 let (op, op_len) = this.operand_to_simd(op)?;
 836                 let bitmask_len = op_len.max(8);
 837
 838                 assert!(dest.layout.ty.is_integral());
 839                 assert!(bitmask_len <= 64);
 840                 assert_eq!(bitmask_len, dest.layout.size.bits());
 841
 842                 let mut res = 0u64;
 843                 for i in 0..op_len {
 844                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 845                     if simd_element_to_bool(op)? {
 846                         res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian);
 847                     }
 848                 }
 849                 this.write_int(res, dest)?;
 850             }
 851
 852             // Atomic operations
 853             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
 854             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
 855             "atomic_load_acq" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
 856
 857             "atomic_store" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
 858             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
 859             "atomic_store_rel" => this.atomic_store(args, AtomicWriteOp::Release)?,
 860
 861             "atomic_fence_acq" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
 862             "atomic_fence_rel" => this.atomic_fence(args, AtomicFenceOp::Release)?,
 863             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
 864             "atomic_fence" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
 865
 866             "atomic_singlethreadfence_acq" => this.compiler_fence(args, AtomicFenceOp::Acquire)?,
 867             "atomic_singlethreadfence_rel" => this.compiler_fence(args, AtomicFenceOp::Release)?,
 868             "atomic_singlethreadfence_acqrel" =>
 869                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
 870             "atomic_singlethreadfence" => this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
 871
 872             "atomic_xchg" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
 873             "atomic_xchg_acq" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
 874             "atomic_xchg_rel" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
 875             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
 876             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
 877
 878             #[rustfmt::skip]
 879             "atomic_cxchg" =>
 880                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 881             #[rustfmt::skip]
 882             "atomic_cxchg_acq" =>
 883                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 884             #[rustfmt::skip]
 885             "atomic_cxchg_rel" =>
 886                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 887             #[rustfmt::skip]
 888             "atomic_cxchg_acqrel" =>
 889                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 890             #[rustfmt::skip]
 891             "atomic_cxchg_relaxed" =>
 892                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 893             #[rustfmt::skip]
 894             "atomic_cxchg_acq_failrelaxed" =>
 895                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 896             #[rustfmt::skip]
 897             "atomic_cxchg_acqrel_failrelaxed" =>
 898                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 899             #[rustfmt::skip]
 900             "atomic_cxchg_failrelaxed" =>
 901                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 902             #[rustfmt::skip]
 903             "atomic_cxchg_failacq" =>
 904                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 905
 906             #[rustfmt::skip]
 907             "atomic_cxchgweak" =>
 908                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 909             #[rustfmt::skip]
 910             "atomic_cxchgweak_acq" =>
 911                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 912             #[rustfmt::skip]
 913             "atomic_cxchgweak_rel" =>
 914                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 915             #[rustfmt::skip]
 916             "atomic_cxchgweak_acqrel" =>
 917                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 918             #[rustfmt::skip]
 919             "atomic_cxchgweak_relaxed" =>
 920                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 921             #[rustfmt::skip]
 922             "atomic_cxchgweak_acq_failrelaxed" =>
 923                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 924             #[rustfmt::skip]
 925             "atomic_cxchgweak_acqrel_failrelaxed" =>
 926                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 927             #[rustfmt::skip]
 928             "atomic_cxchgweak_failrelaxed" =>
 929                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 930             #[rustfmt::skip]
 931             "atomic_cxchgweak_failacq" =>
 932                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 933
 934             #[rustfmt::skip]
 935             "atomic_or" =>
 936                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::SeqCst)?,
 937             #[rustfmt::skip]
 938             "atomic_or_acq" =>
 939                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Acquire)?,
 940             #[rustfmt::skip]
 941             "atomic_or_rel" =>
 942                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Release)?,
 943             #[rustfmt::skip]
 944             "atomic_or_acqrel" =>
 945                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::AcqRel)?,
 946             #[rustfmt::skip]
 947             "atomic_or_relaxed" =>
 948                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Relaxed)?,
 949             #[rustfmt::skip]
 950             "atomic_xor" =>
 951                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::SeqCst)?,
 952             #[rustfmt::skip]
 953             "atomic_xor_acq" =>
 954                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Acquire)?,
 955             #[rustfmt::skip]
 956             "atomic_xor_rel" =>
 957                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Release)?,
 958             #[rustfmt::skip]
 959             "atomic_xor_acqrel" =>
 960                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::AcqRel)?,
 961             #[rustfmt::skip]
 962             "atomic_xor_relaxed" =>
 963                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Relaxed)?,
 964             #[rustfmt::skip]
 965             "atomic_and" =>
 966                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::SeqCst)?,
 967             #[rustfmt::skip]
 968             "atomic_and_acq" =>
 969                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Acquire)?,
 970             #[rustfmt::skip]
 971             "atomic_and_rel" =>
 972                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Release)?,
 973             #[rustfmt::skip]
 974             "atomic_and_acqrel" =>
 975                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::AcqRel)?,
 976             #[rustfmt::skip]
 977             "atomic_and_relaxed" =>
 978                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Relaxed)?,
 979             #[rustfmt::skip]
 980             "atomic_nand" =>
 981                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::SeqCst)?,
 982             #[rustfmt::skip]
 983             "atomic_nand_acq" =>
 984                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Acquire)?,
 985             #[rustfmt::skip]
 986             "atomic_nand_rel" =>
 987                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Release)?,
 988             #[rustfmt::skip]
 989             "atomic_nand_acqrel" =>
 990                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::AcqRel)?,
 991             #[rustfmt::skip]
 992             "atomic_nand_relaxed" =>
 993                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Relaxed)?,
 994             #[rustfmt::skip]
 995             "atomic_xadd" =>
 996                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::SeqCst)?,
 997             #[rustfmt::skip]
 998             "atomic_xadd_acq" =>
 999                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Acquire)?,
1000             #[rustfmt::skip]
1001             "atomic_xadd_rel" =>
1002                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Release)?,
1003             #[rustfmt::skip]
1004             "atomic_xadd_acqrel" =>
1005                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::AcqRel)?,
1006             #[rustfmt::skip]
1007             "atomic_xadd_relaxed" =>
1008                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Relaxed)?,
1009             #[rustfmt::skip]
1010             "atomic_xsub" =>
1011                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::SeqCst)?,
1012             #[rustfmt::skip]
1013             "atomic_xsub_acq" =>
1014                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Acquire)?,
1015             #[rustfmt::skip]
1016             "atomic_xsub_rel" =>
1017                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Release)?,
1018             #[rustfmt::skip]
1019             "atomic_xsub_acqrel" =>
1020                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::AcqRel)?,
1021             #[rustfmt::skip]
1022             "atomic_xsub_relaxed" =>
1023                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Relaxed)?,
1024             "atomic_min" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
1025             "atomic_min_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
1026             "atomic_min_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
1027             "atomic_min_acqrel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
1028             "atomic_min_relaxed" =>
1029                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
1030             "atomic_max" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
1031             "atomic_max_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
1032             "atomic_max_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
1033             "atomic_max_acqrel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
1034             "atomic_max_relaxed" =>
1035                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
1036             "atomic_umin" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
1037             "atomic_umin_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
1038             "atomic_umin_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
1039             "atomic_umin_acqrel" =>
1040                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
1041             "atomic_umin_relaxed" =>
1042                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
1043             "atomic_umax" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
1044             "atomic_umax_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
1045             "atomic_umax_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
1046             "atomic_umax_acqrel" =>
1047                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
1048             "atomic_umax_relaxed" =>
1049                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
1050
1051             // Other
1052             "exact_div" => {
1053                 let &[ref num, ref denom] = check_arg_count(args)?;
1054                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
1055             }
1056
1057             "try" => return this.handle_try(args, dest, ret),
1058
1059             "breakpoint" => {
1060                 let &[] = check_arg_count(args)?;
1061                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
1062                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
1063             }
1064
1065             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
1066         }
1067
1068         trace!("{:?}", this.dump_place(**dest));
1069         this.go_to_block(ret);
1070         Ok(())
1071     }
1072
1073     fn atomic_load(
1074         &mut self,
1075         args: &[OpTy<'tcx, Tag>],
1076         dest: &PlaceTy<'tcx, Tag>,
1077         atomic: AtomicReadOp,
1078     ) -> InterpResult<'tcx> {
1079         let this = self.eval_context_mut();
1080
1081         let &[ref place] = check_arg_count(args)?;
1082         let place = this.deref_operand(place)?;
1083
1084         // make sure it fits into a scalar; otherwise it cannot be atomic
1085         let val = this.read_scalar_atomic(&place, atomic)?;
1086
1087         // Check alignment requirements. Atomics must always be aligned to their size,
1088         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1089         // be 8-aligned).
1090         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1091         this.check_ptr_access_align(
1092             place.ptr,
1093             place.layout.size,
1094             align,
1095             CheckInAllocMsg::MemoryAccessTest,
1096         )?;
1097         // Perform regular access.
1098         this.write_scalar(val, dest)?;
1099         Ok(())
1100     }
1101
1102     fn atomic_store(
1103         &mut self,
1104         args: &[OpTy<'tcx, Tag>],
1105         atomic: AtomicWriteOp,
1106     ) -> InterpResult<'tcx> {
1107         let this = self.eval_context_mut();
1108
1109         let &[ref place, ref val] = check_arg_count(args)?;
1110         let place = this.deref_operand(place)?;
1111         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
1112
1113         // Check alignment requirements. Atomics must always be aligned to their size,
1114         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1115         // be 8-aligned).
1116         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1117         this.check_ptr_access_align(
1118             place.ptr,
1119             place.layout.size,
1120             align,
1121             CheckInAllocMsg::MemoryAccessTest,
1122         )?;
1123
1124         // Perform atomic store
1125         this.write_scalar_atomic(val, &place, atomic)?;
1126         Ok(())
1127     }
1128
1129     fn compiler_fence(
1130         &mut self,
1131         args: &[OpTy<'tcx, Tag>],
1132         atomic: AtomicFenceOp,
1133     ) -> InterpResult<'tcx> {
1134         let &[] = check_arg_count(args)?;
1135         let _ = atomic;
1136         //FIXME: compiler fences are currently ignored
1137         Ok(())
1138     }
1139
1140     fn atomic_fence(
1141         &mut self,
1142         args: &[OpTy<'tcx, Tag>],
1143         atomic: AtomicFenceOp,
1144     ) -> InterpResult<'tcx> {
1145         let this = self.eval_context_mut();
1146         let &[] = check_arg_count(args)?;
1147         this.validate_atomic_fence(atomic)?;
1148         Ok(())
1149     }
1150
1151     fn atomic_op(
1152         &mut self,
1153         args: &[OpTy<'tcx, Tag>],
1154         dest: &PlaceTy<'tcx, Tag>,
1155         atomic_op: AtomicOp,
1156         atomic: AtomicRwOp,
1157     ) -> InterpResult<'tcx> {
1158         let this = self.eval_context_mut();
1159
1160         let &[ref place, ref rhs] = check_arg_count(args)?;
1161         let place = this.deref_operand(place)?;
1162
1163         if !place.layout.ty.is_integral() {
1164             bug!("Atomic arithmetic operations only work on integer types");
1165         }
1166         let rhs = this.read_immediate(rhs)?;
1167
1168         // Check alignment requirements. Atomics must always be aligned to their size,
1169         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1170         // be 8-aligned).
1171         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1172         this.check_ptr_access_align(
1173             place.ptr,
1174             place.layout.size,
1175             align,
1176             CheckInAllocMsg::MemoryAccessTest,
1177         )?;
1178
1179         match atomic_op {
1180             AtomicOp::Min => {
1181                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
1182                 this.write_immediate(*old, &dest)?; // old value is returned
1183                 Ok(())
1184             }
1185             AtomicOp::Max => {
1186                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
1187                 this.write_immediate(*old, &dest)?; // old value is returned
1188                 Ok(())
1189             }
1190             AtomicOp::MirOp(op, neg) => {
1191                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
1192                 this.write_immediate(*old, dest)?; // old value is returned
1193                 Ok(())
1194             }
1195         }
1196     }
1197
1198     fn atomic_exchange(
1199         &mut self,
1200         args: &[OpTy<'tcx, Tag>],
1201         dest: &PlaceTy<'tcx, Tag>,
1202         atomic: AtomicRwOp,
1203     ) -> InterpResult<'tcx> {
1204         let this = self.eval_context_mut();
1205
1206         let &[ref place, ref new] = check_arg_count(args)?;
1207         let place = this.deref_operand(place)?;
1208         let new = this.read_scalar(new)?;
1209
1210         // Check alignment requirements. Atomics must always be aligned to their size,
1211         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1212         // be 8-aligned).
1213         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1214         this.check_ptr_access_align(
1215             place.ptr,
1216             place.layout.size,
1217             align,
1218             CheckInAllocMsg::MemoryAccessTest,
1219         )?;
1220
1221         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
1222         this.write_scalar(old, dest)?; // old value is returned
1223         Ok(())
1224     }
1225
1226     fn atomic_compare_exchange_impl(
1227         &mut self,
1228         args: &[OpTy<'tcx, Tag>],
1229         dest: &PlaceTy<'tcx, Tag>,
1230         success: AtomicRwOp,
1231         fail: AtomicReadOp,
1232         can_fail_spuriously: bool,
1233     ) -> InterpResult<'tcx> {
1234         let this = self.eval_context_mut();
1235
1236         let &[ref place, ref expect_old, ref new] = check_arg_count(args)?;
1237         let place = this.deref_operand(place)?;
1238         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
1239         let new = this.read_scalar(new)?;
1240
1241         // Check alignment requirements. Atomics must always be aligned to their size,
1242         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1243         // be 8-aligned).
1244         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1245         this.check_ptr_access_align(
1246             place.ptr,
1247             place.layout.size,
1248             align,
1249             CheckInAllocMsg::MemoryAccessTest,
1250         )?;
1251
1252         let old = this.atomic_compare_exchange_scalar(
1253             &place,
1254             &expect_old,
1255             new,
1256             success,
1257             fail,
1258             can_fail_spuriously,
1259         )?;
1260
1261         // Return old value.
1262         this.write_immediate(old, dest)?;
1263         Ok(())
1264     }
1265
1266     fn atomic_compare_exchange(
1267         &mut self,
1268         args: &[OpTy<'tcx, Tag>],
1269         dest: &PlaceTy<'tcx, Tag>,
1270         success: AtomicRwOp,
1271         fail: AtomicReadOp,
1272     ) -> InterpResult<'tcx> {
1273         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
1274     }
1275
1276     fn atomic_compare_exchange_weak(
1277         &mut self,
1278         args: &[OpTy<'tcx, Tag>],
1279         dest: &PlaceTy<'tcx, Tag>,
1280         success: AtomicRwOp,
1281         fail: AtomicReadOp,
1282     ) -> InterpResult<'tcx> {
1283         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
1284     }
1285
1286     fn float_to_int_unchecked<F>(
1287         &self,
1288         f: F,
1289         dest_ty: ty::Ty<'tcx>,
1290     ) -> InterpResult<'tcx, Scalar<Tag>>
1291     where
1292         F: Float + Into<Scalar<Tag>>,
1293     {
1294         let this = self.eval_context_ref();
1295
1296         // Step 1: cut off the fractional part of `f`. The result of this is
1297         // guaranteed to be precisely representable in IEEE floats.
1298         let f = f.round_to_integral(Round::TowardZero).value;
1299
1300         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
1301         Ok(match dest_ty.kind() {
1302             // Unsigned
1303             ty::Uint(t) => {
1304                 let size = Integer::from_uint_ty(this, *t).size();
1305                 let res = f.to_u128(size.bits_usize());
1306                 if res.status.is_empty() {
1307                     // No status flags means there was no further rounding or other loss of precision.
1308                     Scalar::from_uint(res.value, size)
1309                 } else {
1310                     // `f` was not representable in this integer type.
1311                     throw_ub_format!(
1312                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
1313                         f,
1314                         dest_ty,
1315                     );
1316                 }
1317             }
1318             // Signed
1319             ty::Int(t) => {
1320                 let size = Integer::from_int_ty(this, *t).size();
1321                 let res = f.to_i128(size.bits_usize());
1322                 if res.status.is_empty() {
1323                     // No status flags means there was no further rounding or other loss of precision.
1324                     Scalar::from_int(res.value, size)
1325                 } else {
1326                     // `f` was not representable in this integer type.
1327                     throw_ub_format!(
1328                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
1329                         f,
1330                         dest_ty,
1331                     );
1332                 }
1333             }
1334             // Nothing else
1335             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
1336         })
1337     }
1338 }
1339
1340 fn fmax_op<'tcx>(
1341     left: &ImmTy<'tcx, Tag>,
1342     right: &ImmTy<'tcx, Tag>,
1343 ) -> InterpResult<'tcx, Scalar<Tag>> {
1344     assert_eq!(left.layout.ty, right.layout.ty);
1345     let ty::Float(float_ty) = left.layout.ty.kind() else {
1346         bug!("fmax operand is not a float")
1347     };
1348     let left = left.to_scalar()?;
1349     let right = right.to_scalar()?;
1350     Ok(match float_ty {
1351         FloatTy::F32 => Scalar::from_f32(left.to_f32()?.max(right.to_f32()?)),
1352         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.max(right.to_f64()?)),
1353     })
1354 }
1355
1356 fn fmin_op<'tcx>(
1357     left: &ImmTy<'tcx, Tag>,
1358     right: &ImmTy<'tcx, Tag>,
1359 ) -> InterpResult<'tcx, Scalar<Tag>> {
1360     assert_eq!(left.layout.ty, right.layout.ty);
1361     let ty::Float(float_ty) = left.layout.ty.kind() else {
1362         bug!("fmin operand is not a float")
1363     };
1364     let left = left.to_scalar()?;
1365     let right = right.to_scalar()?;
1366     Ok(match float_ty {
1367         FloatTy::F32 => Scalar::from_f32(left.to_f32()?.min(right.to_f32()?)),
1368         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)),
1369     })
1370 }
1371
1372 fn bool_to_simd_element(b: bool, size: Size) -> Scalar<Tag> {
1373     // SIMD uses all-1 as pattern for "true"
1374     let val = if b { -1 } else { 0 };
1375     Scalar::from_int(val, size)
1376 }
1377
1378 fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool> {
1379     let val = elem.to_scalar()?.to_int(elem.layout.size)?;
1380     Ok(match val {
1381         0 => false,
1382         -1 => true,
1383         _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"),
1384     })
1385 }
1386
1387 fn simd_bitmask_index(idx: u64, vec_len: u64, endianess: Endian) -> u64 {
1388     assert!(idx < vec_len);
1389     match endianess {
1390         Endian::Little => idx,
1391         Endian::Big => vec_len - 1 - idx, // reverse order of bits
1392     }
1393 }