src/shims/intrinsics.rs

   1 use std::convert::TryInto;
   2 use std::iter;
   3
   4 use log::trace;
   5
   6 use rustc_apfloat::{Float, Round};
   7 use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf};
   8 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
   9 use rustc_target::abi::{Align, Endian, HasDataLayout, Integer, Size};
  10
  11 use crate::*;
  12 use helpers::check_arg_count;
  13
  14 pub enum AtomicOp {
  15     MirOp(mir::BinOp, bool),
  16     Max,
  17     Min,
  18 }
  19
  20 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  21 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  22     fn call_intrinsic(
  23         &mut self,
  24         instance: ty::Instance<'tcx>,
  25         args: &[OpTy<'tcx, Tag>],
  26         ret: Option<(&PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  27         _unwind: StackPopUnwind,
  28     ) -> InterpResult<'tcx> {
  29         let this = self.eval_context_mut();
  30
  31         if this.emulate_intrinsic(instance, args, ret)? {
  32             return Ok(());
  33         }
  34
  35         // All supported intrinsics have a return place.
  36         let intrinsic_name = this.tcx.item_name(instance.def_id());
  37         let intrinsic_name = intrinsic_name.as_str();
  38         let (dest, ret) = match ret {
  39             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  40             Some(p) => p,
  41         };
  42
  43         // Then handle terminating intrinsics.
  44         match intrinsic_name {
  45             // Miri overwriting CTFE intrinsics.
  46             "ptr_guaranteed_eq" => {
  47                 let &[ref left, ref right] = check_arg_count(args)?;
  48                 let left = this.read_immediate(left)?;
  49                 let right = this.read_immediate(right)?;
  50                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  51             }
  52             "ptr_guaranteed_ne" => {
  53                 let &[ref left, ref right] = check_arg_count(args)?;
  54                 let left = this.read_immediate(left)?;
  55                 let right = this.read_immediate(right)?;
  56                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  57             }
  58             "const_allocate" => {
  59                 // For now, for compatibility with the run-time implementation of this, we just return null.
  60                 // See <https://github.com/rust-lang/rust/issues/93935>.
  61                 this.write_null(dest)?;
  62             }
  63             "const_deallocate" => {
  64                 // complete NOP
  65             }
  66
  67             // Raw memory accesses
  68             "volatile_load" => {
  69                 let &[ref place] = check_arg_count(args)?;
  70                 let place = this.deref_operand(place)?;
  71                 this.copy_op(&place.into(), dest)?;
  72             }
  73             "volatile_store" => {
  74                 let &[ref place, ref dest] = check_arg_count(args)?;
  75                 let place = this.deref_operand(place)?;
  76                 this.copy_op(dest, &place.into())?;
  77             }
  78
  79             "write_bytes" | "volatile_set_memory" => {
  80                 let &[ref ptr, ref val_byte, ref count] = check_arg_count(args)?;
  81                 let ty = instance.substs.type_at(0);
  82                 let ty_layout = this.layout_of(ty)?;
  83                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
  84                 let ptr = this.read_pointer(ptr)?;
  85                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  86                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
  87                     err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
  88                 })?;
  89                 this.memory
  90                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
  91             }
  92
  93             // Floating-point operations
  94             "fabsf32" => {
  95                 let &[ref f] = check_arg_count(args)?;
  96                 let f = this.read_scalar(f)?.to_f32()?;
  97                 // Can be implemented in soft-floats.
  98                 this.write_scalar(Scalar::from_f32(f.abs()), dest)?;
  99             }
 100             "fabsf64" => {
 101                 let &[ref f] = check_arg_count(args)?;
 102                 let f = this.read_scalar(f)?.to_f64()?;
 103                 // Can be implemented in soft-floats.
 104                 this.write_scalar(Scalar::from_f64(f.abs()), dest)?;
 105             }
 106             #[rustfmt::skip]
 107             | "sinf32"
 108             | "cosf32"
 109             | "sqrtf32"
 110             | "expf32"
 111             | "exp2f32"
 112             | "logf32"
 113             | "log10f32"
 114             | "log2f32"
 115             | "floorf32"
 116             | "ceilf32"
 117             | "truncf32"
 118             | "roundf32"
 119             => {
 120                 let &[ref f] = check_arg_count(args)?;
 121                 // FIXME: Using host floats.
 122                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 123                 let f = match intrinsic_name {
 124                     "sinf32" => f.sin(),
 125                     "cosf32" => f.cos(),
 126                     "sqrtf32" => f.sqrt(),
 127                     "expf32" => f.exp(),
 128                     "exp2f32" => f.exp2(),
 129                     "logf32" => f.ln(),
 130                     "log10f32" => f.log10(),
 131                     "log2f32" => f.log2(),
 132                     "floorf32" => f.floor(),
 133                     "ceilf32" => f.ceil(),
 134                     "truncf32" => f.trunc(),
 135                     "roundf32" => f.round(),
 136                     _ => bug!(),
 137                 };
 138                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 139             }
 140
 141             #[rustfmt::skip]
 142             | "sinf64"
 143             | "cosf64"
 144             | "sqrtf64"
 145             | "expf64"
 146             | "exp2f64"
 147             | "logf64"
 148             | "log10f64"
 149             | "log2f64"
 150             | "floorf64"
 151             | "ceilf64"
 152             | "truncf64"
 153             | "roundf64"
 154             => {
 155                 let &[ref f] = check_arg_count(args)?;
 156                 // FIXME: Using host floats.
 157                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 158                 let f = match intrinsic_name {
 159                     "sinf64" => f.sin(),
 160                     "cosf64" => f.cos(),
 161                     "sqrtf64" => f.sqrt(),
 162                     "expf64" => f.exp(),
 163                     "exp2f64" => f.exp2(),
 164                     "logf64" => f.ln(),
 165                     "log10f64" => f.log10(),
 166                     "log2f64" => f.log2(),
 167                     "floorf64" => f.floor(),
 168                     "ceilf64" => f.ceil(),
 169                     "truncf64" => f.trunc(),
 170                     "roundf64" => f.round(),
 171                     _ => bug!(),
 172                 };
 173                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 174             }
 175
 176             #[rustfmt::skip]
 177             | "fadd_fast"
 178             | "fsub_fast"
 179             | "fmul_fast"
 180             | "fdiv_fast"
 181             | "frem_fast"
 182             => {
 183                 let &[ref a, ref b] = check_arg_count(args)?;
 184                 let a = this.read_immediate(a)?;
 185                 let b = this.read_immediate(b)?;
 186                 let op = match intrinsic_name {
 187                     "fadd_fast" => mir::BinOp::Add,
 188                     "fsub_fast" => mir::BinOp::Sub,
 189                     "fmul_fast" => mir::BinOp::Mul,
 190                     "fdiv_fast" => mir::BinOp::Div,
 191                     "frem_fast" => mir::BinOp::Rem,
 192                     _ => bug!(),
 193                 };
 194                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 195                     Ok(match x.layout.ty.kind() {
 196                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 197                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 198                         _ => bug!(
 199                             "`{}` called with non-float input type {:?}",
 200                             intrinsic_name,
 201                             x.layout.ty
 202                         ),
 203                     })
 204                 };
 205                 match (float_finite(a)?, float_finite(b)?) {
 206                     (false, false) => throw_ub_format!(
 207                         "`{}` intrinsic called with non-finite value as both parameters",
 208                         intrinsic_name,
 209                     ),
 210                     (false, _) => throw_ub_format!(
 211                         "`{}` intrinsic called with non-finite value as first parameter",
 212                         intrinsic_name,
 213                     ),
 214                     (_, false) => throw_ub_format!(
 215                         "`{}` intrinsic called with non-finite value as second parameter",
 216                         intrinsic_name,
 217                     ),
 218                     _ => {}
 219                 }
 220                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 221             }
 222
 223             #[rustfmt::skip]
 224             | "minnumf32"
 225             | "maxnumf32"
 226             | "copysignf32"
 227             => {
 228                 let &[ref a, ref b] = check_arg_count(args)?;
 229                 let a = this.read_scalar(a)?.to_f32()?;
 230                 let b = this.read_scalar(b)?.to_f32()?;
 231                 let res = match intrinsic_name {
 232                     "minnumf32" => a.min(b),
 233                     "maxnumf32" => a.max(b),
 234                     "copysignf32" => a.copy_sign(b),
 235                     _ => bug!(),
 236                 };
 237                 this.write_scalar(Scalar::from_f32(res), dest)?;
 238             }
 239
 240             #[rustfmt::skip]
 241             | "minnumf64"
 242             | "maxnumf64"
 243             | "copysignf64"
 244             => {
 245                 let &[ref a, ref b] = check_arg_count(args)?;
 246                 let a = this.read_scalar(a)?.to_f64()?;
 247                 let b = this.read_scalar(b)?.to_f64()?;
 248                 let res = match intrinsic_name {
 249                     "minnumf64" => a.min(b),
 250                     "maxnumf64" => a.max(b),
 251                     "copysignf64" => a.copy_sign(b),
 252                     _ => bug!(),
 253                 };
 254                 this.write_scalar(Scalar::from_f64(res), dest)?;
 255             }
 256
 257             "powf32" => {
 258                 let &[ref f, ref f2] = check_arg_count(args)?;
 259                 // FIXME: Using host floats.
 260                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 261                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 262                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 263             }
 264
 265             "powf64" => {
 266                 let &[ref f, ref f2] = check_arg_count(args)?;
 267                 // FIXME: Using host floats.
 268                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 269                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 270                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 271             }
 272
 273             "fmaf32" => {
 274                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 275                 let a = this.read_scalar(a)?.to_f32()?;
 276                 let b = this.read_scalar(b)?.to_f32()?;
 277                 let c = this.read_scalar(c)?.to_f32()?;
 278                 let res = a.mul_add(b, c).value;
 279                 this.write_scalar(Scalar::from_f32(res), dest)?;
 280             }
 281
 282             "fmaf64" => {
 283                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 284                 let a = this.read_scalar(a)?.to_f64()?;
 285                 let b = this.read_scalar(b)?.to_f64()?;
 286                 let c = this.read_scalar(c)?.to_f64()?;
 287                 let res = a.mul_add(b, c).value;
 288                 this.write_scalar(Scalar::from_f64(res), dest)?;
 289             }
 290
 291             "powif32" => {
 292                 let &[ref f, ref i] = check_arg_count(args)?;
 293                 // FIXME: Using host floats.
 294                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 295                 let i = this.read_scalar(i)?.to_i32()?;
 296                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 297             }
 298
 299             "powif64" => {
 300                 let &[ref f, ref i] = check_arg_count(args)?;
 301                 // FIXME: Using host floats.
 302                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 303                 let i = this.read_scalar(i)?.to_i32()?;
 304                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 305             }
 306
 307             "float_to_int_unchecked" => {
 308                 let &[ref val] = check_arg_count(args)?;
 309                 let val = this.read_immediate(val)?;
 310
 311                 let res = match val.layout.ty.kind() {
 312                     ty::Float(FloatTy::F32) =>
 313                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 314                     ty::Float(FloatTy::F64) =>
 315                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 316                     _ =>
 317                         bug!(
 318                             "`float_to_int_unchecked` called with non-float input type {:?}",
 319                             val.layout.ty
 320                         ),
 321                 };
 322
 323                 this.write_scalar(res, dest)?;
 324             }
 325
 326             // SIMD operations
 327             #[rustfmt::skip]
 328             | "simd_neg"
 329             | "simd_fabs"
 330             | "simd_ceil"
 331             | "simd_floor"
 332             | "simd_round"
 333             | "simd_trunc"
 334             | "simd_fsqrt" => {
 335                 let &[ref op] = check_arg_count(args)?;
 336                 let (op, op_len) = this.operand_to_simd(op)?;
 337                 let (dest, dest_len) = this.place_to_simd(dest)?;
 338
 339                 assert_eq!(dest_len, op_len);
 340
 341                 #[derive(Copy, Clone)]
 342                 enum HostFloatOp {
 343                     Ceil,
 344                     Floor,
 345                     Round,
 346                     Trunc,
 347                     Sqrt,
 348                 }
 349                 #[derive(Copy, Clone)]
 350                 enum Op {
 351                     MirOp(mir::UnOp),
 352                     Abs,
 353                     HostOp(HostFloatOp),
 354                 }
 355                 let which = match intrinsic_name {
 356                     "simd_neg" => Op::MirOp(mir::UnOp::Neg),
 357                     "simd_fabs" => Op::Abs,
 358                     "simd_ceil" => Op::HostOp(HostFloatOp::Ceil),
 359                     "simd_floor" => Op::HostOp(HostFloatOp::Floor),
 360                     "simd_round" => Op::HostOp(HostFloatOp::Round),
 361                     "simd_trunc" => Op::HostOp(HostFloatOp::Trunc),
 362                     "simd_fsqrt" => Op::HostOp(HostFloatOp::Sqrt),
 363                     _ => unreachable!(),
 364                 };
 365
 366                 for i in 0..dest_len {
 367                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 368                     let dest = this.mplace_index(&dest, i)?;
 369                     let val = match which {
 370                         Op::MirOp(mir_op) => this.unary_op(mir_op, &op)?.to_scalar()?,
 371                         Op::Abs => {
 372                             // Works for f32 and f64.
 373                             let ty::Float(float_ty) = op.layout.ty.kind() else {
 374                                 bug!("{} operand is not a float", intrinsic_name)
 375                             };
 376                             let op = op.to_scalar()?;
 377                             match float_ty {
 378                                 FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()),
 379                                 FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()),
 380                             }
 381                         }
 382                         Op::HostOp(host_op) => {
 383                             let ty::Float(float_ty) = op.layout.ty.kind() else {
 384                                 bug!("{} operand is not a float", intrinsic_name)
 385                             };
 386                             // FIXME using host floats
 387                             match float_ty {
 388                                 FloatTy::F32 => {
 389                                     let f = f32::from_bits(op.to_scalar()?.to_u32()?);
 390                                     let res = match host_op {
 391                                         HostFloatOp::Ceil => f.ceil(),
 392                                         HostFloatOp::Floor => f.floor(),
 393                                         HostFloatOp::Round => f.round(),
 394                                         HostFloatOp::Trunc => f.trunc(),
 395                                         HostFloatOp::Sqrt => f.sqrt(),
 396                                     };
 397                                     Scalar::from_u32(res.to_bits())
 398                                 }
 399                                 FloatTy::F64 => {
 400                                     let f = f64::from_bits(op.to_scalar()?.to_u64()?);
 401                                     let res = match host_op {
 402                                         HostFloatOp::Ceil => f.ceil(),
 403                                         HostFloatOp::Floor => f.floor(),
 404                                         HostFloatOp::Round => f.round(),
 405                                         HostFloatOp::Trunc => f.trunc(),
 406                                         HostFloatOp::Sqrt => f.sqrt(),
 407                                     };
 408                                     Scalar::from_u64(res.to_bits())
 409                                 }
 410                             }
 411
 412                         }
 413                     };
 414                     this.write_scalar(val, &dest.into())?;
 415                 }
 416             }
 417             #[rustfmt::skip]
 418             | "simd_add"
 419             | "simd_sub"
 420             | "simd_mul"
 421             | "simd_div"
 422             | "simd_rem"
 423             | "simd_shl"
 424             | "simd_shr"
 425             | "simd_and"
 426             | "simd_or"
 427             | "simd_xor"
 428             | "simd_eq"
 429             | "simd_ne"
 430             | "simd_lt"
 431             | "simd_le"
 432             | "simd_gt"
 433             | "simd_ge"
 434             | "simd_fmax"
 435             | "simd_fmin"
 436             | "simd_saturating_add"
 437             | "simd_saturating_sub" => {
 438                 use mir::BinOp;
 439
 440                 let &[ref left, ref right] = check_arg_count(args)?;
 441                 let (left, left_len) = this.operand_to_simd(left)?;
 442                 let (right, right_len) = this.operand_to_simd(right)?;
 443                 let (dest, dest_len) = this.place_to_simd(dest)?;
 444
 445                 assert_eq!(dest_len, left_len);
 446                 assert_eq!(dest_len, right_len);
 447
 448                 enum Op {
 449                     MirOp(BinOp),
 450                     SaturatingOp(BinOp),
 451                     FMax,
 452                     FMin,
 453                 }
 454                 let which = match intrinsic_name {
 455                     "simd_add" => Op::MirOp(BinOp::Add),
 456                     "simd_sub" => Op::MirOp(BinOp::Sub),
 457                     "simd_mul" => Op::MirOp(BinOp::Mul),
 458                     "simd_div" => Op::MirOp(BinOp::Div),
 459                     "simd_rem" => Op::MirOp(BinOp::Rem),
 460                     "simd_shl" => Op::MirOp(BinOp::Shl),
 461                     "simd_shr" => Op::MirOp(BinOp::Shr),
 462                     "simd_and" => Op::MirOp(BinOp::BitAnd),
 463                     "simd_or" => Op::MirOp(BinOp::BitOr),
 464                     "simd_xor" => Op::MirOp(BinOp::BitXor),
 465                     "simd_eq" => Op::MirOp(BinOp::Eq),
 466                     "simd_ne" => Op::MirOp(BinOp::Ne),
 467                     "simd_lt" => Op::MirOp(BinOp::Lt),
 468                     "simd_le" => Op::MirOp(BinOp::Le),
 469                     "simd_gt" => Op::MirOp(BinOp::Gt),
 470                     "simd_ge" => Op::MirOp(BinOp::Ge),
 471                     "simd_fmax" => Op::FMax,
 472                     "simd_fmin" => Op::FMin,
 473                     "simd_saturating_add" => Op::SaturatingOp(BinOp::Add),
 474                     "simd_saturating_sub" => Op::SaturatingOp(BinOp::Sub),
 475                     _ => unreachable!(),
 476                 };
 477
 478                 for i in 0..dest_len {
 479                     let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
 480                     let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
 481                     let dest = this.mplace_index(&dest, i)?;
 482                     let val = match which {
 483                         Op::MirOp(mir_op) => {
 484                             let (val, overflowed, ty) = this.overflowing_binary_op(mir_op, &left, &right)?;
 485                             if matches!(mir_op, BinOp::Shl | BinOp::Shr) {
 486                                 // Shifts have extra UB as SIMD operations that the MIR binop does not have.
 487                                 // See <https://github.com/rust-lang/rust/issues/91237>.
 488                                 if overflowed {
 489                                     let r_val = right.to_scalar()?.to_bits(right.layout.size)?;
 490                                     throw_ub_format!("overflowing shift by {} in `{}` in SIMD lane {}", r_val, intrinsic_name, i);
 491                                 }
 492                             }
 493                             if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) {
 494                                 // Special handling for boolean-returning operations
 495                                 assert_eq!(ty, this.tcx.types.bool);
 496                                 let val = val.to_bool().unwrap();
 497                                 bool_to_simd_element(val, dest.layout.size)
 498                             } else {
 499                                 assert_ne!(ty, this.tcx.types.bool);
 500                                 assert_eq!(ty, dest.layout.ty);
 501                                 val
 502                             }
 503                         }
 504                         Op::FMax => {
 505                             fmax_op(&left, &right)?
 506                         }
 507                         Op::FMin => {
 508                             fmin_op(&left, &right)?
 509                         }
 510                         Op::SaturatingOp(mir_op) => {
 511                             this.saturating_arith(mir_op, &left, &right)?
 512                         }
 513                     };
 514                     this.write_scalar(val, &dest.into())?;
 515                 }
 516             }
 517             "simd_fma" => {
 518                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 519                 let (a, a_len) = this.operand_to_simd(a)?;
 520                 let (b, b_len) = this.operand_to_simd(b)?;
 521                 let (c, c_len) = this.operand_to_simd(c)?;
 522                 let (dest, dest_len) = this.place_to_simd(dest)?;
 523
 524                 assert_eq!(dest_len, a_len);
 525                 assert_eq!(dest_len, b_len);
 526                 assert_eq!(dest_len, c_len);
 527
 528                 for i in 0..dest_len {
 529                     let a = this.read_immediate(&this.mplace_index(&a, i)?.into())?.to_scalar()?;
 530                     let b = this.read_immediate(&this.mplace_index(&b, i)?.into())?.to_scalar()?;
 531                     let c = this.read_immediate(&this.mplace_index(&c, i)?.into())?.to_scalar()?;
 532                     let dest = this.mplace_index(&dest, i)?;
 533
 534                     // Works for f32 and f64.
 535                     let ty::Float(float_ty) = dest.layout.ty.kind() else {
 536                         bug!("{} operand is not a float", intrinsic_name)
 537                     };
 538                     let val = match float_ty {
 539                         FloatTy::F32 =>
 540                             Scalar::from_f32(a.to_f32()?.mul_add(b.to_f32()?, c.to_f32()?).value),
 541                         FloatTy::F64 =>
 542                             Scalar::from_f64(a.to_f64()?.mul_add(b.to_f64()?, c.to_f64()?).value),
 543                     };
 544                     this.write_scalar(val, &dest.into())?;
 545                 }
 546             }
 547             #[rustfmt::skip]
 548             | "simd_reduce_and"
 549             | "simd_reduce_or"
 550             | "simd_reduce_xor"
 551             | "simd_reduce_any"
 552             | "simd_reduce_all"
 553             | "simd_reduce_max"
 554             | "simd_reduce_min" => {
 555                 use mir::BinOp;
 556
 557                 let &[ref op] = check_arg_count(args)?;
 558                 let (op, op_len) = this.operand_to_simd(op)?;
 559
 560                 let imm_from_bool =
 561                     |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool);
 562
 563                 enum Op {
 564                     MirOp(BinOp),
 565                     MirOpBool(BinOp),
 566                     Max,
 567                     Min,
 568                 }
 569                 let which = match intrinsic_name {
 570                     "simd_reduce_and" => Op::MirOp(BinOp::BitAnd),
 571                     "simd_reduce_or" => Op::MirOp(BinOp::BitOr),
 572                     "simd_reduce_xor" => Op::MirOp(BinOp::BitXor),
 573                     "simd_reduce_any" => Op::MirOpBool(BinOp::BitOr),
 574                     "simd_reduce_all" => Op::MirOpBool(BinOp::BitAnd),
 575                     "simd_reduce_max" => Op::Max,
 576                     "simd_reduce_min" => Op::Min,
 577                     _ => unreachable!(),
 578                 };
 579
 580                 // Initialize with first lane, then proceed with the rest.
 581                 let mut res = this.read_immediate(&this.mplace_index(&op, 0)?.into())?;
 582                 if matches!(which, Op::MirOpBool(_)) {
 583                     // Convert to `bool` scalar.
 584                     res = imm_from_bool(simd_element_to_bool(res)?);
 585                 }
 586                 for i in 1..op_len {
 587                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 588                     res = match which {
 589                         Op::MirOp(mir_op) => {
 590                             this.binary_op(mir_op, &res, &op)?
 591                         }
 592                         Op::MirOpBool(mir_op) => {
 593                             let op = imm_from_bool(simd_element_to_bool(op)?);
 594                             this.binary_op(mir_op, &res, &op)?
 595                         }
 596                         Op::Max => {
 597                             if matches!(res.layout.ty.kind(), ty::Float(_)) {
 598                                 ImmTy::from_scalar(fmax_op(&res, &op)?, res.layout)
 599                             } else {
 600                                 // Just boring integers, so NaNs to worry about
 601                                 if this.binary_op(BinOp::Ge, &res, &op)?.to_scalar()?.to_bool()? {
 602                                     res
 603                                 } else {
 604                                     op
 605                                 }
 606                             }
 607                         }
 608                         Op::Min => {
 609                             if matches!(res.layout.ty.kind(), ty::Float(_)) {
 610                                 ImmTy::from_scalar(fmin_op(&res, &op)?, res.layout)
 611                             } else {
 612                                 // Just boring integers, so NaNs to worry about
 613                                 if this.binary_op(BinOp::Le, &res, &op)?.to_scalar()?.to_bool()? {
 614                                     res
 615                                 } else {
 616                                     op
 617                                 }
 618                             }
 619                         }
 620                     };
 621                 }
 622                 this.write_immediate(*res, dest)?;
 623             }
 624             #[rustfmt::skip]
 625             | "simd_reduce_add_ordered"
 626             | "simd_reduce_mul_ordered" => {
 627                 use mir::BinOp;
 628
 629                 let &[ref op, ref init] = check_arg_count(args)?;
 630                 let (op, op_len) = this.operand_to_simd(op)?;
 631                 let init = this.read_immediate(init)?;
 632
 633                 let mir_op = match intrinsic_name {
 634                     "simd_reduce_add_ordered" => BinOp::Add,
 635                     "simd_reduce_mul_ordered" => BinOp::Mul,
 636                     _ => unreachable!(),
 637                 };
 638
 639                 let mut res = init;
 640                 for i in 0..op_len {
 641                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 642                     res = this.binary_op(mir_op, &res, &op)?;
 643                 }
 644                 this.write_immediate(*res, dest)?;
 645             }
 646             "simd_select" => {
 647                 let &[ref mask, ref yes, ref no] = check_arg_count(args)?;
 648                 let (mask, mask_len) = this.operand_to_simd(mask)?;
 649                 let (yes, yes_len) = this.operand_to_simd(yes)?;
 650                 let (no, no_len) = this.operand_to_simd(no)?;
 651                 let (dest, dest_len) = this.place_to_simd(dest)?;
 652
 653                 assert_eq!(dest_len, mask_len);
 654                 assert_eq!(dest_len, yes_len);
 655                 assert_eq!(dest_len, no_len);
 656
 657                 for i in 0..dest_len {
 658                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
 659                     let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
 660                     let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
 661                     let dest = this.mplace_index(&dest, i)?;
 662
 663                     let val = if simd_element_to_bool(mask)? { yes } else { no };
 664                     this.write_immediate(*val, &dest.into())?;
 665                 }
 666             }
 667             "simd_select_bitmask" => {
 668                 let &[ref mask, ref yes, ref no] = check_arg_count(args)?;
 669                 let (yes, yes_len) = this.operand_to_simd(yes)?;
 670                 let (no, no_len) = this.operand_to_simd(no)?;
 671                 let (dest, dest_len) = this.place_to_simd(dest)?;
 672
 673                 assert!(mask.layout.ty.is_integral());
 674                 assert_eq!(dest_len.max(8), mask.layout.size.bits());
 675                 assert!(dest_len <= 64);
 676                 assert_eq!(dest_len, yes_len);
 677                 assert_eq!(dest_len, no_len);
 678
 679                 let mask: u64 = this
 680                     .read_scalar(mask)?
 681                     .check_init()?
 682                     .to_bits(mask.layout.size)?
 683                     .try_into()
 684                     .unwrap();
 685                 for i in 0..dest_len {
 686                     let mask =
 687                         mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian));
 688                     let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
 689                     let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
 690                     let dest = this.mplace_index(&dest, i)?;
 691
 692                     let val = if mask != 0 { yes } else { no };
 693                     this.write_immediate(*val, &dest.into())?;
 694                 }
 695             }
 696             #[rustfmt::skip]
 697             "simd_cast" | "simd_as" => {
 698                 let &[ref op] = check_arg_count(args)?;
 699                 let (op, op_len) = this.operand_to_simd(op)?;
 700                 let (dest, dest_len) = this.place_to_simd(dest)?;
 701
 702                 assert_eq!(dest_len, op_len);
 703
 704                 let safe_cast = intrinsic_name == "simd_as";
 705
 706                 for i in 0..dest_len {
 707                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 708                     let dest = this.mplace_index(&dest, i)?;
 709
 710                     let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) {
 711                         // Int-to-(int|float): always safe
 712                         (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) =>
 713                             this.misc_cast(&op, dest.layout.ty)?,
 714                         // Float-to-float: always safe
 715                         (ty::Float(_), ty::Float(_)) =>
 716                             this.misc_cast(&op, dest.layout.ty)?,
 717                         // Float-to-int in safe mode
 718                         (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast =>
 719                             this.misc_cast(&op, dest.layout.ty)?,
 720                         // Float-to-int in unchecked mode
 721                         (ty::Float(FloatTy::F32), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
 722                             this.float_to_int_unchecked(op.to_scalar()?.to_f32()?, dest.layout.ty)?.into(),
 723                         (ty::Float(FloatTy::F64), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
 724                             this.float_to_int_unchecked(op.to_scalar()?.to_f64()?, dest.layout.ty)?.into(),
 725                         _ =>
 726                             throw_unsup_format!(
 727                                 "Unsupported SIMD cast from element type {} to {}",
 728                                 op.layout.ty,
 729                                 dest.layout.ty
 730                             ),
 731                     };
 732                     this.write_immediate(val, &dest.into())?;
 733                 }
 734             }
 735             "simd_shuffle" => {
 736                 let &[ref left, ref right, ref index] = check_arg_count(args)?;
 737                 let (left, left_len) = this.operand_to_simd(left)?;
 738                 let (right, right_len) = this.operand_to_simd(right)?;
 739                 let (dest, dest_len) = this.place_to_simd(dest)?;
 740
 741                 // `index` is an array, not a SIMD type
 742                 let ty::Array(_, index_len) = index.layout.ty.kind() else {
 743                     bug!("simd_shuffle index argument has non-array type {}", index.layout.ty)
 744                 };
 745                 let index_len = index_len.eval_usize(*this.tcx, this.param_env());
 746
 747                 assert_eq!(left_len, right_len);
 748                 assert_eq!(index_len, dest_len);
 749
 750                 for i in 0..dest_len {
 751                     let src_index: u64 = this
 752                         .read_immediate(&this.operand_index(&index, i)?.into())?
 753                         .to_scalar()?
 754                         .to_u32()?
 755                         .into();
 756                     let dest = this.mplace_index(&dest, i)?;
 757
 758                     let val = if src_index < left_len {
 759                         this.read_immediate(&this.mplace_index(&left, src_index)?.into())?
 760                     } else if src_index < left_len.checked_add(right_len).unwrap() {
 761                         this.read_immediate(
 762                             &this.mplace_index(&right, src_index - left_len)?.into(),
 763                         )?
 764                     } else {
 765                         bug!(
 766                             "simd_shuffle index {} is out of bounds for 2 vectors of size {}",
 767                             src_index,
 768                             left_len
 769                         );
 770                     };
 771                     this.write_immediate(*val, &dest.into())?;
 772                 }
 773             }
 774             "simd_gather" => {
 775                 let &[ref passthru, ref ptrs, ref mask] = check_arg_count(args)?;
 776                 let (passthru, passthru_len) = this.operand_to_simd(passthru)?;
 777                 let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
 778                 let (mask, mask_len) = this.operand_to_simd(mask)?;
 779                 let (dest, dest_len) = this.place_to_simd(dest)?;
 780
 781                 assert_eq!(dest_len, passthru_len);
 782                 assert_eq!(dest_len, ptrs_len);
 783                 assert_eq!(dest_len, mask_len);
 784
 785                 for i in 0..dest_len {
 786                     let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?;
 787                     let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
 788                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
 789                     let dest = this.mplace_index(&dest, i)?;
 790
 791                     let val = if simd_element_to_bool(mask)? {
 792                         let place = this.deref_operand(&ptr.into())?;
 793                         this.read_immediate(&place.into())?
 794                     } else {
 795                         passthru
 796                     };
 797                     this.write_immediate(*val, &dest.into())?;
 798                 }
 799             }
 800             "simd_scatter" => {
 801                 let &[ref value, ref ptrs, ref mask] = check_arg_count(args)?;
 802                 let (value, value_len) = this.operand_to_simd(value)?;
 803                 let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
 804                 let (mask, mask_len) = this.operand_to_simd(mask)?;
 805
 806                 assert_eq!(ptrs_len, value_len);
 807                 assert_eq!(ptrs_len, mask_len);
 808
 809                 for i in 0..ptrs_len {
 810                     let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?;
 811                     let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
 812                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
 813
 814                     if simd_element_to_bool(mask)? {
 815                         let place = this.deref_operand(&ptr.into())?;
 816                         this.write_immediate(*value, &place.into())?;
 817                     }
 818                 }
 819             }
 820             "simd_bitmask" => {
 821                 let &[ref op] = check_arg_count(args)?;
 822                 let (op, op_len) = this.operand_to_simd(op)?;
 823
 824                 assert!(dest.layout.ty.is_integral());
 825                 assert_eq!(op_len.max(8), dest.layout.size.bits());
 826                 assert!(op_len <= 64);
 827
 828                 let mut res = 0u64;
 829                 for i in 0..op_len {
 830                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 831                     if simd_element_to_bool(op)? {
 832                         res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian);
 833                     }
 834                 }
 835                 this.write_int(res, dest)?;
 836             }
 837
 838             // Atomic operations
 839             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
 840             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
 841             "atomic_load_acq" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
 842
 843             "atomic_store" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
 844             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
 845             "atomic_store_rel" => this.atomic_store(args, AtomicWriteOp::Release)?,
 846
 847             "atomic_fence_acq" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
 848             "atomic_fence_rel" => this.atomic_fence(args, AtomicFenceOp::Release)?,
 849             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
 850             "atomic_fence" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
 851
 852             "atomic_singlethreadfence_acq" => this.compiler_fence(args, AtomicFenceOp::Acquire)?,
 853             "atomic_singlethreadfence_rel" => this.compiler_fence(args, AtomicFenceOp::Release)?,
 854             "atomic_singlethreadfence_acqrel" =>
 855                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
 856             "atomic_singlethreadfence" => this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
 857
 858             "atomic_xchg" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
 859             "atomic_xchg_acq" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
 860             "atomic_xchg_rel" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
 861             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
 862             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
 863
 864             #[rustfmt::skip]
 865             "atomic_cxchg" =>
 866                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 867             #[rustfmt::skip]
 868             "atomic_cxchg_acq" =>
 869                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 870             #[rustfmt::skip]
 871             "atomic_cxchg_rel" =>
 872                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 873             #[rustfmt::skip]
 874             "atomic_cxchg_acqrel" =>
 875                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 876             #[rustfmt::skip]
 877             "atomic_cxchg_relaxed" =>
 878                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 879             #[rustfmt::skip]
 880             "atomic_cxchg_acq_failrelaxed" =>
 881                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 882             #[rustfmt::skip]
 883             "atomic_cxchg_acqrel_failrelaxed" =>
 884                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 885             #[rustfmt::skip]
 886             "atomic_cxchg_failrelaxed" =>
 887                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 888             #[rustfmt::skip]
 889             "atomic_cxchg_failacq" =>
 890                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 891
 892             #[rustfmt::skip]
 893             "atomic_cxchgweak" =>
 894                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 895             #[rustfmt::skip]
 896             "atomic_cxchgweak_acq" =>
 897                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 898             #[rustfmt::skip]
 899             "atomic_cxchgweak_rel" =>
 900                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 901             #[rustfmt::skip]
 902             "atomic_cxchgweak_acqrel" =>
 903                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 904             #[rustfmt::skip]
 905             "atomic_cxchgweak_relaxed" =>
 906                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 907             #[rustfmt::skip]
 908             "atomic_cxchgweak_acq_failrelaxed" =>
 909                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 910             #[rustfmt::skip]
 911             "atomic_cxchgweak_acqrel_failrelaxed" =>
 912                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 913             #[rustfmt::skip]
 914             "atomic_cxchgweak_failrelaxed" =>
 915                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 916             #[rustfmt::skip]
 917             "atomic_cxchgweak_failacq" =>
 918                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 919
 920             #[rustfmt::skip]
 921             "atomic_or" =>
 922                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::SeqCst)?,
 923             #[rustfmt::skip]
 924             "atomic_or_acq" =>
 925                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Acquire)?,
 926             #[rustfmt::skip]
 927             "atomic_or_rel" =>
 928                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Release)?,
 929             #[rustfmt::skip]
 930             "atomic_or_acqrel" =>
 931                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::AcqRel)?,
 932             #[rustfmt::skip]
 933             "atomic_or_relaxed" =>
 934                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Relaxed)?,
 935             #[rustfmt::skip]
 936             "atomic_xor" =>
 937                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::SeqCst)?,
 938             #[rustfmt::skip]
 939             "atomic_xor_acq" =>
 940                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Acquire)?,
 941             #[rustfmt::skip]
 942             "atomic_xor_rel" =>
 943                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Release)?,
 944             #[rustfmt::skip]
 945             "atomic_xor_acqrel" =>
 946                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::AcqRel)?,
 947             #[rustfmt::skip]
 948             "atomic_xor_relaxed" =>
 949                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Relaxed)?,
 950             #[rustfmt::skip]
 951             "atomic_and" =>
 952                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::SeqCst)?,
 953             #[rustfmt::skip]
 954             "atomic_and_acq" =>
 955                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Acquire)?,
 956             #[rustfmt::skip]
 957             "atomic_and_rel" =>
 958                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Release)?,
 959             #[rustfmt::skip]
 960             "atomic_and_acqrel" =>
 961                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::AcqRel)?,
 962             #[rustfmt::skip]
 963             "atomic_and_relaxed" =>
 964                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Relaxed)?,
 965             #[rustfmt::skip]
 966             "atomic_nand" =>
 967                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::SeqCst)?,
 968             #[rustfmt::skip]
 969             "atomic_nand_acq" =>
 970                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Acquire)?,
 971             #[rustfmt::skip]
 972             "atomic_nand_rel" =>
 973                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Release)?,
 974             #[rustfmt::skip]
 975             "atomic_nand_acqrel" =>
 976                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::AcqRel)?,
 977             #[rustfmt::skip]
 978             "atomic_nand_relaxed" =>
 979                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Relaxed)?,
 980             #[rustfmt::skip]
 981             "atomic_xadd" =>
 982                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::SeqCst)?,
 983             #[rustfmt::skip]
 984             "atomic_xadd_acq" =>
 985                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Acquire)?,
 986             #[rustfmt::skip]
 987             "atomic_xadd_rel" =>
 988                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Release)?,
 989             #[rustfmt::skip]
 990             "atomic_xadd_acqrel" =>
 991                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::AcqRel)?,
 992             #[rustfmt::skip]
 993             "atomic_xadd_relaxed" =>
 994                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Relaxed)?,
 995             #[rustfmt::skip]
 996             "atomic_xsub" =>
 997                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::SeqCst)?,
 998             #[rustfmt::skip]
 999             "atomic_xsub_acq" =>
1000                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Acquire)?,
1001             #[rustfmt::skip]
1002             "atomic_xsub_rel" =>
1003                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Release)?,
1004             #[rustfmt::skip]
1005             "atomic_xsub_acqrel" =>
1006                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::AcqRel)?,
1007             #[rustfmt::skip]
1008             "atomic_xsub_relaxed" =>
1009                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Relaxed)?,
1010             "atomic_min" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
1011             "atomic_min_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
1012             "atomic_min_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
1013             "atomic_min_acqrel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
1014             "atomic_min_relaxed" =>
1015                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
1016             "atomic_max" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
1017             "atomic_max_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
1018             "atomic_max_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
1019             "atomic_max_acqrel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
1020             "atomic_max_relaxed" =>
1021                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
1022             "atomic_umin" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
1023             "atomic_umin_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
1024             "atomic_umin_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
1025             "atomic_umin_acqrel" =>
1026                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
1027             "atomic_umin_relaxed" =>
1028                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
1029             "atomic_umax" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
1030             "atomic_umax_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
1031             "atomic_umax_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
1032             "atomic_umax_acqrel" =>
1033                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
1034             "atomic_umax_relaxed" =>
1035                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
1036
1037             // Other
1038             "exact_div" => {
1039                 let &[ref num, ref denom] = check_arg_count(args)?;
1040                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
1041             }
1042
1043             "try" => return this.handle_try(args, dest, ret),
1044
1045             "breakpoint" => {
1046                 let &[] = check_arg_count(args)?;
1047                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
1048                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
1049             }
1050
1051             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
1052         }
1053
1054         trace!("{:?}", this.dump_place(**dest));
1055         this.go_to_block(ret);
1056         Ok(())
1057     }
1058
1059     fn atomic_load(
1060         &mut self,
1061         args: &[OpTy<'tcx, Tag>],
1062         dest: &PlaceTy<'tcx, Tag>,
1063         atomic: AtomicReadOp,
1064     ) -> InterpResult<'tcx> {
1065         let this = self.eval_context_mut();
1066
1067         let &[ref place] = check_arg_count(args)?;
1068         let place = this.deref_operand(place)?;
1069
1070         // make sure it fits into a scalar; otherwise it cannot be atomic
1071         let val = this.read_scalar_atomic(&place, atomic)?;
1072
1073         // Check alignment requirements. Atomics must always be aligned to their size,
1074         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1075         // be 8-aligned).
1076         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1077         this.memory.check_ptr_access_align(
1078             place.ptr,
1079             place.layout.size,
1080             align,
1081             CheckInAllocMsg::MemoryAccessTest,
1082         )?;
1083         // Perform regular access.
1084         this.write_scalar(val, dest)?;
1085         Ok(())
1086     }
1087
1088     fn atomic_store(
1089         &mut self,
1090         args: &[OpTy<'tcx, Tag>],
1091         atomic: AtomicWriteOp,
1092     ) -> InterpResult<'tcx> {
1093         let this = self.eval_context_mut();
1094
1095         let &[ref place, ref val] = check_arg_count(args)?;
1096         let place = this.deref_operand(place)?;
1097         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
1098
1099         // Check alignment requirements. Atomics must always be aligned to their size,
1100         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1101         // be 8-aligned).
1102         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1103         this.memory.check_ptr_access_align(
1104             place.ptr,
1105             place.layout.size,
1106             align,
1107             CheckInAllocMsg::MemoryAccessTest,
1108         )?;
1109
1110         // Perform atomic store
1111         this.write_scalar_atomic(val, &place, atomic)?;
1112         Ok(())
1113     }
1114
1115     fn compiler_fence(
1116         &mut self,
1117         args: &[OpTy<'tcx, Tag>],
1118         atomic: AtomicFenceOp,
1119     ) -> InterpResult<'tcx> {
1120         let &[] = check_arg_count(args)?;
1121         let _ = atomic;
1122         //FIXME: compiler fences are currently ignored
1123         Ok(())
1124     }
1125
1126     fn atomic_fence(
1127         &mut self,
1128         args: &[OpTy<'tcx, Tag>],
1129         atomic: AtomicFenceOp,
1130     ) -> InterpResult<'tcx> {
1131         let this = self.eval_context_mut();
1132         let &[] = check_arg_count(args)?;
1133         this.validate_atomic_fence(atomic)?;
1134         Ok(())
1135     }
1136
1137     fn atomic_op(
1138         &mut self,
1139         args: &[OpTy<'tcx, Tag>],
1140         dest: &PlaceTy<'tcx, Tag>,
1141         atomic_op: AtomicOp,
1142         atomic: AtomicRwOp,
1143     ) -> InterpResult<'tcx> {
1144         let this = self.eval_context_mut();
1145
1146         let &[ref place, ref rhs] = check_arg_count(args)?;
1147         let place = this.deref_operand(place)?;
1148
1149         if !place.layout.ty.is_integral() {
1150             bug!("Atomic arithmetic operations only work on integer types");
1151         }
1152         let rhs = this.read_immediate(rhs)?;
1153
1154         // Check alignment requirements. Atomics must always be aligned to their size,
1155         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1156         // be 8-aligned).
1157         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1158         this.memory.check_ptr_access_align(
1159             place.ptr,
1160             place.layout.size,
1161             align,
1162             CheckInAllocMsg::MemoryAccessTest,
1163         )?;
1164
1165         match atomic_op {
1166             AtomicOp::Min => {
1167                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
1168                 this.write_immediate(*old, &dest)?; // old value is returned
1169                 Ok(())
1170             }
1171             AtomicOp::Max => {
1172                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
1173                 this.write_immediate(*old, &dest)?; // old value is returned
1174                 Ok(())
1175             }
1176             AtomicOp::MirOp(op, neg) => {
1177                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
1178                 this.write_immediate(*old, dest)?; // old value is returned
1179                 Ok(())
1180             }
1181         }
1182     }
1183
1184     fn atomic_exchange(
1185         &mut self,
1186         args: &[OpTy<'tcx, Tag>],
1187         dest: &PlaceTy<'tcx, Tag>,
1188         atomic: AtomicRwOp,
1189     ) -> InterpResult<'tcx> {
1190         let this = self.eval_context_mut();
1191
1192         let &[ref place, ref new] = check_arg_count(args)?;
1193         let place = this.deref_operand(place)?;
1194         let new = this.read_scalar(new)?;
1195
1196         // Check alignment requirements. Atomics must always be aligned to their size,
1197         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1198         // be 8-aligned).
1199         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1200         this.memory.check_ptr_access_align(
1201             place.ptr,
1202             place.layout.size,
1203             align,
1204             CheckInAllocMsg::MemoryAccessTest,
1205         )?;
1206
1207         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
1208         this.write_scalar(old, dest)?; // old value is returned
1209         Ok(())
1210     }
1211
1212     fn atomic_compare_exchange_impl(
1213         &mut self,
1214         args: &[OpTy<'tcx, Tag>],
1215         dest: &PlaceTy<'tcx, Tag>,
1216         success: AtomicRwOp,
1217         fail: AtomicReadOp,
1218         can_fail_spuriously: bool,
1219     ) -> InterpResult<'tcx> {
1220         let this = self.eval_context_mut();
1221
1222         let &[ref place, ref expect_old, ref new] = check_arg_count(args)?;
1223         let place = this.deref_operand(place)?;
1224         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
1225         let new = this.read_scalar(new)?;
1226
1227         // Check alignment requirements. Atomics must always be aligned to their size,
1228         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1229         // be 8-aligned).
1230         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1231         this.memory.check_ptr_access_align(
1232             place.ptr,
1233             place.layout.size,
1234             align,
1235             CheckInAllocMsg::MemoryAccessTest,
1236         )?;
1237
1238         let old = this.atomic_compare_exchange_scalar(
1239             &place,
1240             &expect_old,
1241             new,
1242             success,
1243             fail,
1244             can_fail_spuriously,
1245         )?;
1246
1247         // Return old value.
1248         this.write_immediate(old, dest)?;
1249         Ok(())
1250     }
1251
1252     fn atomic_compare_exchange(
1253         &mut self,
1254         args: &[OpTy<'tcx, Tag>],
1255         dest: &PlaceTy<'tcx, Tag>,
1256         success: AtomicRwOp,
1257         fail: AtomicReadOp,
1258     ) -> InterpResult<'tcx> {
1259         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
1260     }
1261
1262     fn atomic_compare_exchange_weak(
1263         &mut self,
1264         args: &[OpTy<'tcx, Tag>],
1265         dest: &PlaceTy<'tcx, Tag>,
1266         success: AtomicRwOp,
1267         fail: AtomicReadOp,
1268     ) -> InterpResult<'tcx> {
1269         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
1270     }
1271
1272     fn float_to_int_unchecked<F>(
1273         &self,
1274         f: F,
1275         dest_ty: ty::Ty<'tcx>,
1276     ) -> InterpResult<'tcx, Scalar<Tag>>
1277     where
1278         F: Float + Into<Scalar<Tag>>,
1279     {
1280         let this = self.eval_context_ref();
1281
1282         // Step 1: cut off the fractional part of `f`. The result of this is
1283         // guaranteed to be precisely representable in IEEE floats.
1284         let f = f.round_to_integral(Round::TowardZero).value;
1285
1286         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
1287         Ok(match dest_ty.kind() {
1288             // Unsigned
1289             ty::Uint(t) => {
1290                 let size = Integer::from_uint_ty(this, *t).size();
1291                 let res = f.to_u128(size.bits_usize());
1292                 if res.status.is_empty() {
1293                     // No status flags means there was no further rounding or other loss of precision.
1294                     Scalar::from_uint(res.value, size)
1295                 } else {
1296                     // `f` was not representable in this integer type.
1297                     throw_ub_format!(
1298                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
1299                         f,
1300                         dest_ty,
1301                     );
1302                 }
1303             }
1304             // Signed
1305             ty::Int(t) => {
1306                 let size = Integer::from_int_ty(this, *t).size();
1307                 let res = f.to_i128(size.bits_usize());
1308                 if res.status.is_empty() {
1309                     // No status flags means there was no further rounding or other loss of precision.
1310                     Scalar::from_int(res.value, size)
1311                 } else {
1312                     // `f` was not representable in this integer type.
1313                     throw_ub_format!(
1314                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
1315                         f,
1316                         dest_ty,
1317                     );
1318                 }
1319             }
1320             // Nothing else
1321             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
1322         })
1323     }
1324 }
1325
1326 fn fmax_op<'tcx>(
1327     left: &ImmTy<'tcx, Tag>,
1328     right: &ImmTy<'tcx, Tag>,
1329 ) -> InterpResult<'tcx, Scalar<Tag>> {
1330     assert_eq!(left.layout.ty, right.layout.ty);
1331     let ty::Float(float_ty) = left.layout.ty.kind() else {
1332         bug!("fmax operand is not a float")
1333     };
1334     let left = left.to_scalar()?;
1335     let right = right.to_scalar()?;
1336     Ok(match float_ty {
1337         FloatTy::F32 => Scalar::from_f32(left.to_f32()?.max(right.to_f32()?)),
1338         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.max(right.to_f64()?)),
1339     })
1340 }
1341
1342 fn fmin_op<'tcx>(
1343     left: &ImmTy<'tcx, Tag>,
1344     right: &ImmTy<'tcx, Tag>,
1345 ) -> InterpResult<'tcx, Scalar<Tag>> {
1346     assert_eq!(left.layout.ty, right.layout.ty);
1347     let ty::Float(float_ty) = left.layout.ty.kind() else {
1348         bug!("fmin operand is not a float")
1349     };
1350     let left = left.to_scalar()?;
1351     let right = right.to_scalar()?;
1352     Ok(match float_ty {
1353         FloatTy::F32 => Scalar::from_f32(left.to_f32()?.min(right.to_f32()?)),
1354         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)),
1355     })
1356 }
1357
1358 fn bool_to_simd_element(b: bool, size: Size) -> Scalar<Tag> {
1359     // SIMD uses all-1 as pattern for "true"
1360     let val = if b { -1 } else { 0 };
1361     Scalar::from_int(val, size)
1362 }
1363
1364 fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool> {
1365     let val = elem.to_scalar()?.to_int(elem.layout.size)?;
1366     Ok(match val {
1367         0 => false,
1368         -1 => true,
1369         _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"),
1370     })
1371 }
1372
1373 fn simd_bitmask_index(idx: u64, len: u64, endianess: Endian) -> u64 {
1374     assert!(idx < len);
1375     match endianess {
1376         Endian::Little => idx,
1377         Endian::Big => len.max(8) - 1 - idx, // reverse order of bits
1378     }
1379 }