src/shims/intrinsics.rs

   1 use std::iter;
   2
   3 use log::trace;
   4
   5 use rustc_apfloat::{Float, Round};
   6 use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf};
   7 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
   8 use rustc_target::abi::{Align, Endian, HasDataLayout, Integer, Size};
   9
  10 use crate::*;
  11 use helpers::check_arg_count;
  12
  13 pub enum AtomicOp {
  14     MirOp(mir::BinOp, bool),
  15     Max,
  16     Min,
  17 }
  18
  19 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  20 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  21     fn call_intrinsic(
  22         &mut self,
  23         instance: ty::Instance<'tcx>,
  24         args: &[OpTy<'tcx, Tag>],
  25         dest: &PlaceTy<'tcx, Tag>,
  26         ret: Option<mir::BasicBlock>,
  27         _unwind: StackPopUnwind,
  28     ) -> InterpResult<'tcx> {
  29         let this = self.eval_context_mut();
  30
  31         if this.emulate_intrinsic(instance, args, dest, ret)? {
  32             return Ok(());
  33         }
  34
  35         // All supported intrinsics have a return place.
  36         let intrinsic_name = this.tcx.item_name(instance.def_id());
  37         let intrinsic_name = intrinsic_name.as_str();
  38         let ret = match ret {
  39             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  40             Some(p) => p,
  41         };
  42
  43         // Then handle terminating intrinsics.
  44         match intrinsic_name {
  45             // Miri overwriting CTFE intrinsics.
  46             "ptr_guaranteed_eq" => {
  47                 let [left, right] = check_arg_count(args)?;
  48                 let left = this.read_immediate(left)?;
  49                 let right = this.read_immediate(right)?;
  50                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  51             }
  52             "ptr_guaranteed_ne" => {
  53                 let [left, right] = check_arg_count(args)?;
  54                 let left = this.read_immediate(left)?;
  55                 let right = this.read_immediate(right)?;
  56                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  57             }
  58             "const_allocate" => {
  59                 // For now, for compatibility with the run-time implementation of this, we just return null.
  60                 // See <https://github.com/rust-lang/rust/issues/93935>.
  61                 this.write_null(dest)?;
  62             }
  63             "const_deallocate" => {
  64                 // complete NOP
  65             }
  66
  67             // Raw memory accesses
  68             "volatile_load" => {
  69                 let [place] = check_arg_count(args)?;
  70                 let place = this.deref_operand(place)?;
  71                 this.copy_op(&place.into(), dest)?;
  72             }
  73             "volatile_store" => {
  74                 let [place, dest] = check_arg_count(args)?;
  75                 let place = this.deref_operand(place)?;
  76                 this.copy_op(dest, &place.into())?;
  77             }
  78
  79             "write_bytes" | "volatile_set_memory" => {
  80                 let [ptr, val_byte, count] = check_arg_count(args)?;
  81                 let ty = instance.substs.type_at(0);
  82                 let ty_layout = this.layout_of(ty)?;
  83                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
  84                 let ptr = this.read_pointer(ptr)?;
  85                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  86                 // `checked_mul` enforces a too small bound (the correct one would probably be machine_isize_max),
  87                 // but no actual allocation can be big enough for the difference to be noticeable.
  88                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
  89                     err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
  90                 })?;
  91                 this.write_bytes_ptr(
  92                     ptr,
  93                     iter::repeat(val_byte).take(byte_count.bytes() as usize),
  94                 )?;
  95             }
  96
  97             // Floating-point operations
  98             "fabsf32" => {
  99                 let [f] = check_arg_count(args)?;
 100                 let f = this.read_scalar(f)?.to_f32()?;
 101                 // Can be implemented in soft-floats.
 102                 this.write_scalar(Scalar::from_f32(f.abs()), dest)?;
 103             }
 104             "fabsf64" => {
 105                 let [f] = check_arg_count(args)?;
 106                 let f = this.read_scalar(f)?.to_f64()?;
 107                 // Can be implemented in soft-floats.
 108                 this.write_scalar(Scalar::from_f64(f.abs()), dest)?;
 109             }
 110             #[rustfmt::skip]
 111             | "sinf32"
 112             | "cosf32"
 113             | "sqrtf32"
 114             | "expf32"
 115             | "exp2f32"
 116             | "logf32"
 117             | "log10f32"
 118             | "log2f32"
 119             | "floorf32"
 120             | "ceilf32"
 121             | "truncf32"
 122             | "roundf32"
 123             => {
 124                 let [f] = check_arg_count(args)?;
 125                 // FIXME: Using host floats.
 126                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 127                 let f = match intrinsic_name {
 128                     "sinf32" => f.sin(),
 129                     "cosf32" => f.cos(),
 130                     "sqrtf32" => f.sqrt(),
 131                     "expf32" => f.exp(),
 132                     "exp2f32" => f.exp2(),
 133                     "logf32" => f.ln(),
 134                     "log10f32" => f.log10(),
 135                     "log2f32" => f.log2(),
 136                     "floorf32" => f.floor(),
 137                     "ceilf32" => f.ceil(),
 138                     "truncf32" => f.trunc(),
 139                     "roundf32" => f.round(),
 140                     _ => bug!(),
 141                 };
 142                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 143             }
 144
 145             #[rustfmt::skip]
 146             | "sinf64"
 147             | "cosf64"
 148             | "sqrtf64"
 149             | "expf64"
 150             | "exp2f64"
 151             | "logf64"
 152             | "log10f64"
 153             | "log2f64"
 154             | "floorf64"
 155             | "ceilf64"
 156             | "truncf64"
 157             | "roundf64"
 158             => {
 159                 let [f] = check_arg_count(args)?;
 160                 // FIXME: Using host floats.
 161                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 162                 let f = match intrinsic_name {
 163                     "sinf64" => f.sin(),
 164                     "cosf64" => f.cos(),
 165                     "sqrtf64" => f.sqrt(),
 166                     "expf64" => f.exp(),
 167                     "exp2f64" => f.exp2(),
 168                     "logf64" => f.ln(),
 169                     "log10f64" => f.log10(),
 170                     "log2f64" => f.log2(),
 171                     "floorf64" => f.floor(),
 172                     "ceilf64" => f.ceil(),
 173                     "truncf64" => f.trunc(),
 174                     "roundf64" => f.round(),
 175                     _ => bug!(),
 176                 };
 177                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 178             }
 179
 180             #[rustfmt::skip]
 181             | "fadd_fast"
 182             | "fsub_fast"
 183             | "fmul_fast"
 184             | "fdiv_fast"
 185             | "frem_fast"
 186             => {
 187                 let [a, b] = check_arg_count(args)?;
 188                 let a = this.read_immediate(a)?;
 189                 let b = this.read_immediate(b)?;
 190                 let op = match intrinsic_name {
 191                     "fadd_fast" => mir::BinOp::Add,
 192                     "fsub_fast" => mir::BinOp::Sub,
 193                     "fmul_fast" => mir::BinOp::Mul,
 194                     "fdiv_fast" => mir::BinOp::Div,
 195                     "frem_fast" => mir::BinOp::Rem,
 196                     _ => bug!(),
 197                 };
 198                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 199                     Ok(match x.layout.ty.kind() {
 200                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 201                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 202                         _ => bug!(
 203                             "`{}` called with non-float input type {:?}",
 204                             intrinsic_name,
 205                             x.layout.ty
 206                         ),
 207                     })
 208                 };
 209                 match (float_finite(a)?, float_finite(b)?) {
 210                     (false, false) => throw_ub_format!(
 211                         "`{}` intrinsic called with non-finite value as both parameters",
 212                         intrinsic_name,
 213                     ),
 214                     (false, _) => throw_ub_format!(
 215                         "`{}` intrinsic called with non-finite value as first parameter",
 216                         intrinsic_name,
 217                     ),
 218                     (_, false) => throw_ub_format!(
 219                         "`{}` intrinsic called with non-finite value as second parameter",
 220                         intrinsic_name,
 221                     ),
 222                     _ => {}
 223                 }
 224                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 225             }
 226
 227             #[rustfmt::skip]
 228             | "minnumf32"
 229             | "maxnumf32"
 230             | "copysignf32"
 231             => {
 232                 let [a, b] = check_arg_count(args)?;
 233                 let a = this.read_scalar(a)?.to_f32()?;
 234                 let b = this.read_scalar(b)?.to_f32()?;
 235                 let res = match intrinsic_name {
 236                     "minnumf32" => a.min(b),
 237                     "maxnumf32" => a.max(b),
 238                     "copysignf32" => a.copy_sign(b),
 239                     _ => bug!(),
 240                 };
 241                 this.write_scalar(Scalar::from_f32(res), dest)?;
 242             }
 243
 244             #[rustfmt::skip]
 245             | "minnumf64"
 246             | "maxnumf64"
 247             | "copysignf64"
 248             => {
 249                 let [a, b] = check_arg_count(args)?;
 250                 let a = this.read_scalar(a)?.to_f64()?;
 251                 let b = this.read_scalar(b)?.to_f64()?;
 252                 let res = match intrinsic_name {
 253                     "minnumf64" => a.min(b),
 254                     "maxnumf64" => a.max(b),
 255                     "copysignf64" => a.copy_sign(b),
 256                     _ => bug!(),
 257                 };
 258                 this.write_scalar(Scalar::from_f64(res), dest)?;
 259             }
 260
 261             "powf32" => {
 262                 let [f, f2] = check_arg_count(args)?;
 263                 // FIXME: Using host floats.
 264                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 265                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 266                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 267             }
 268
 269             "powf64" => {
 270                 let [f, f2] = check_arg_count(args)?;
 271                 // FIXME: Using host floats.
 272                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 273                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 274                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 275             }
 276
 277             "fmaf32" => {
 278                 let [a, b, c] = check_arg_count(args)?;
 279                 let a = this.read_scalar(a)?.to_f32()?;
 280                 let b = this.read_scalar(b)?.to_f32()?;
 281                 let c = this.read_scalar(c)?.to_f32()?;
 282                 let res = a.mul_add(b, c).value;
 283                 this.write_scalar(Scalar::from_f32(res), dest)?;
 284             }
 285
 286             "fmaf64" => {
 287                 let [a, b, c] = check_arg_count(args)?;
 288                 let a = this.read_scalar(a)?.to_f64()?;
 289                 let b = this.read_scalar(b)?.to_f64()?;
 290                 let c = this.read_scalar(c)?.to_f64()?;
 291                 let res = a.mul_add(b, c).value;
 292                 this.write_scalar(Scalar::from_f64(res), dest)?;
 293             }
 294
 295             "powif32" => {
 296                 let [f, i] = check_arg_count(args)?;
 297                 // FIXME: Using host floats.
 298                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 299                 let i = this.read_scalar(i)?.to_i32()?;
 300                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 301             }
 302
 303             "powif64" => {
 304                 let [f, i] = check_arg_count(args)?;
 305                 // FIXME: Using host floats.
 306                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 307                 let i = this.read_scalar(i)?.to_i32()?;
 308                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 309             }
 310
 311             "float_to_int_unchecked" => {
 312                 let [val] = check_arg_count(args)?;
 313                 let val = this.read_immediate(val)?;
 314
 315                 let res = match val.layout.ty.kind() {
 316                     ty::Float(FloatTy::F32) =>
 317                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 318                     ty::Float(FloatTy::F64) =>
 319                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 320                     _ =>
 321                         bug!(
 322                             "`float_to_int_unchecked` called with non-float input type {:?}",
 323                             val.layout.ty
 324                         ),
 325                 };
 326
 327                 this.write_scalar(res, dest)?;
 328             }
 329
 330             // SIMD operations
 331             #[rustfmt::skip]
 332             | "simd_neg"
 333             | "simd_fabs"
 334             | "simd_ceil"
 335             | "simd_floor"
 336             | "simd_round"
 337             | "simd_trunc"
 338             | "simd_fsqrt" => {
 339                 let [op] = check_arg_count(args)?;
 340                 let (op, op_len) = this.operand_to_simd(op)?;
 341                 let (dest, dest_len) = this.place_to_simd(dest)?;
 342
 343                 assert_eq!(dest_len, op_len);
 344
 345                 #[derive(Copy, Clone)]
 346                 enum HostFloatOp {
 347                     Ceil,
 348                     Floor,
 349                     Round,
 350                     Trunc,
 351                     Sqrt,
 352                 }
 353                 #[derive(Copy, Clone)]
 354                 enum Op {
 355                     MirOp(mir::UnOp),
 356                     Abs,
 357                     HostOp(HostFloatOp),
 358                 }
 359                 let which = match intrinsic_name {
 360                     "simd_neg" => Op::MirOp(mir::UnOp::Neg),
 361                     "simd_fabs" => Op::Abs,
 362                     "simd_ceil" => Op::HostOp(HostFloatOp::Ceil),
 363                     "simd_floor" => Op::HostOp(HostFloatOp::Floor),
 364                     "simd_round" => Op::HostOp(HostFloatOp::Round),
 365                     "simd_trunc" => Op::HostOp(HostFloatOp::Trunc),
 366                     "simd_fsqrt" => Op::HostOp(HostFloatOp::Sqrt),
 367                     _ => unreachable!(),
 368                 };
 369
 370                 for i in 0..dest_len {
 371                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 372                     let dest = this.mplace_index(&dest, i)?;
 373                     let val = match which {
 374                         Op::MirOp(mir_op) => this.unary_op(mir_op, &op)?.to_scalar()?,
 375                         Op::Abs => {
 376                             // Works for f32 and f64.
 377                             let ty::Float(float_ty) = op.layout.ty.kind() else {
 378                                 bug!("{} operand is not a float", intrinsic_name)
 379                             };
 380                             let op = op.to_scalar()?;
 381                             match float_ty {
 382                                 FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()),
 383                                 FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()),
 384                             }
 385                         }
 386                         Op::HostOp(host_op) => {
 387                             let ty::Float(float_ty) = op.layout.ty.kind() else {
 388                                 bug!("{} operand is not a float", intrinsic_name)
 389                             };
 390                             // FIXME using host floats
 391                             match float_ty {
 392                                 FloatTy::F32 => {
 393                                     let f = f32::from_bits(op.to_scalar()?.to_u32()?);
 394                                     let res = match host_op {
 395                                         HostFloatOp::Ceil => f.ceil(),
 396                                         HostFloatOp::Floor => f.floor(),
 397                                         HostFloatOp::Round => f.round(),
 398                                         HostFloatOp::Trunc => f.trunc(),
 399                                         HostFloatOp::Sqrt => f.sqrt(),
 400                                     };
 401                                     Scalar::from_u32(res.to_bits())
 402                                 }
 403                                 FloatTy::F64 => {
 404                                     let f = f64::from_bits(op.to_scalar()?.to_u64()?);
 405                                     let res = match host_op {
 406                                         HostFloatOp::Ceil => f.ceil(),
 407                                         HostFloatOp::Floor => f.floor(),
 408                                         HostFloatOp::Round => f.round(),
 409                                         HostFloatOp::Trunc => f.trunc(),
 410                                         HostFloatOp::Sqrt => f.sqrt(),
 411                                     };
 412                                     Scalar::from_u64(res.to_bits())
 413                                 }
 414                             }
 415
 416                         }
 417                     };
 418                     this.write_scalar(val, &dest.into())?;
 419                 }
 420             }
 421             #[rustfmt::skip]
 422             | "simd_add"
 423             | "simd_sub"
 424             | "simd_mul"
 425             | "simd_div"
 426             | "simd_rem"
 427             | "simd_shl"
 428             | "simd_shr"
 429             | "simd_and"
 430             | "simd_or"
 431             | "simd_xor"
 432             | "simd_eq"
 433             | "simd_ne"
 434             | "simd_lt"
 435             | "simd_le"
 436             | "simd_gt"
 437             | "simd_ge"
 438             | "simd_fmax"
 439             | "simd_fmin"
 440             | "simd_saturating_add"
 441             | "simd_saturating_sub"
 442             | "simd_arith_offset" => {
 443                 use mir::BinOp;
 444
 445                 let [left, right] = check_arg_count(args)?;
 446                 let (left, left_len) = this.operand_to_simd(left)?;
 447                 let (right, right_len) = this.operand_to_simd(right)?;
 448                 let (dest, dest_len) = this.place_to_simd(dest)?;
 449
 450                 assert_eq!(dest_len, left_len);
 451                 assert_eq!(dest_len, right_len);
 452
 453                 enum Op {
 454                     MirOp(BinOp),
 455                     SaturatingOp(BinOp),
 456                     FMax,
 457                     FMin,
 458                     WrappingOffset,
 459                 }
 460                 let which = match intrinsic_name {
 461                     "simd_add" => Op::MirOp(BinOp::Add),
 462                     "simd_sub" => Op::MirOp(BinOp::Sub),
 463                     "simd_mul" => Op::MirOp(BinOp::Mul),
 464                     "simd_div" => Op::MirOp(BinOp::Div),
 465                     "simd_rem" => Op::MirOp(BinOp::Rem),
 466                     "simd_shl" => Op::MirOp(BinOp::Shl),
 467                     "simd_shr" => Op::MirOp(BinOp::Shr),
 468                     "simd_and" => Op::MirOp(BinOp::BitAnd),
 469                     "simd_or" => Op::MirOp(BinOp::BitOr),
 470                     "simd_xor" => Op::MirOp(BinOp::BitXor),
 471                     "simd_eq" => Op::MirOp(BinOp::Eq),
 472                     "simd_ne" => Op::MirOp(BinOp::Ne),
 473                     "simd_lt" => Op::MirOp(BinOp::Lt),
 474                     "simd_le" => Op::MirOp(BinOp::Le),
 475                     "simd_gt" => Op::MirOp(BinOp::Gt),
 476                     "simd_ge" => Op::MirOp(BinOp::Ge),
 477                     "simd_fmax" => Op::FMax,
 478                     "simd_fmin" => Op::FMin,
 479                     "simd_saturating_add" => Op::SaturatingOp(BinOp::Add),
 480                     "simd_saturating_sub" => Op::SaturatingOp(BinOp::Sub),
 481                     "simd_arith_offset" => Op::WrappingOffset,
 482                     _ => unreachable!(),
 483                 };
 484
 485                 for i in 0..dest_len {
 486                     let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
 487                     let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
 488                     let dest = this.mplace_index(&dest, i)?;
 489                     let val = match which {
 490                         Op::MirOp(mir_op) => {
 491                             let (val, overflowed, ty) = this.overflowing_binary_op(mir_op, &left, &right)?;
 492                             if matches!(mir_op, BinOp::Shl | BinOp::Shr) {
 493                                 // Shifts have extra UB as SIMD operations that the MIR binop does not have.
 494                                 // See <https://github.com/rust-lang/rust/issues/91237>.
 495                                 if overflowed {
 496                                     let r_val = right.to_scalar()?.to_bits(right.layout.size)?;
 497                                     throw_ub_format!("overflowing shift by {} in `{}` in SIMD lane {}", r_val, intrinsic_name, i);
 498                                 }
 499                             }
 500                             if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) {
 501                                 // Special handling for boolean-returning operations
 502                                 assert_eq!(ty, this.tcx.types.bool);
 503                                 let val = val.to_bool().unwrap();
 504                                 bool_to_simd_element(val, dest.layout.size)
 505                             } else {
 506                                 assert_ne!(ty, this.tcx.types.bool);
 507                                 assert_eq!(ty, dest.layout.ty);
 508                                 val
 509                             }
 510                         }
 511                         Op::SaturatingOp(mir_op) => {
 512                             this.saturating_arith(mir_op, &left, &right)?
 513                         }
 514                         Op::WrappingOffset => {
 515                             let ptr = this.scalar_to_ptr(left.to_scalar()?)?;
 516                             let offset_count = right.to_scalar()?.to_machine_isize(this)?;
 517                             let pointee_ty = left.layout.ty.builtin_deref(true).unwrap().ty;
 518
 519                             let pointee_size = i64::try_from(this.layout_of(pointee_ty)?.size.bytes()).unwrap();
 520                             let offset_bytes = offset_count.wrapping_mul(pointee_size);
 521                             let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, this);
 522                             Scalar::from_maybe_pointer(offset_ptr, this)
 523                         }
 524                         Op::FMax => {
 525                             fmax_op(&left, &right)?
 526                         }
 527                         Op::FMin => {
 528                             fmin_op(&left, &right)?
 529                         }
 530                     };
 531                     this.write_scalar(val, &dest.into())?;
 532                 }
 533             }
 534             "simd_fma" => {
 535                 let [a, b, c] = check_arg_count(args)?;
 536                 let (a, a_len) = this.operand_to_simd(a)?;
 537                 let (b, b_len) = this.operand_to_simd(b)?;
 538                 let (c, c_len) = this.operand_to_simd(c)?;
 539                 let (dest, dest_len) = this.place_to_simd(dest)?;
 540
 541                 assert_eq!(dest_len, a_len);
 542                 assert_eq!(dest_len, b_len);
 543                 assert_eq!(dest_len, c_len);
 544
 545                 for i in 0..dest_len {
 546                     let a = this.read_immediate(&this.mplace_index(&a, i)?.into())?.to_scalar()?;
 547                     let b = this.read_immediate(&this.mplace_index(&b, i)?.into())?.to_scalar()?;
 548                     let c = this.read_immediate(&this.mplace_index(&c, i)?.into())?.to_scalar()?;
 549                     let dest = this.mplace_index(&dest, i)?;
 550
 551                     // Works for f32 and f64.
 552                     let ty::Float(float_ty) = dest.layout.ty.kind() else {
 553                         bug!("{} operand is not a float", intrinsic_name)
 554                     };
 555                     let val = match float_ty {
 556                         FloatTy::F32 =>
 557                             Scalar::from_f32(a.to_f32()?.mul_add(b.to_f32()?, c.to_f32()?).value),
 558                         FloatTy::F64 =>
 559                             Scalar::from_f64(a.to_f64()?.mul_add(b.to_f64()?, c.to_f64()?).value),
 560                     };
 561                     this.write_scalar(val, &dest.into())?;
 562                 }
 563             }
 564             #[rustfmt::skip]
 565             | "simd_reduce_and"
 566             | "simd_reduce_or"
 567             | "simd_reduce_xor"
 568             | "simd_reduce_any"
 569             | "simd_reduce_all"
 570             | "simd_reduce_max"
 571             | "simd_reduce_min" => {
 572                 use mir::BinOp;
 573
 574                 let [op] = check_arg_count(args)?;
 575                 let (op, op_len) = this.operand_to_simd(op)?;
 576
 577                 let imm_from_bool =
 578                     |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool);
 579
 580                 enum Op {
 581                     MirOp(BinOp),
 582                     MirOpBool(BinOp),
 583                     Max,
 584                     Min,
 585                 }
 586                 let which = match intrinsic_name {
 587                     "simd_reduce_and" => Op::MirOp(BinOp::BitAnd),
 588                     "simd_reduce_or" => Op::MirOp(BinOp::BitOr),
 589                     "simd_reduce_xor" => Op::MirOp(BinOp::BitXor),
 590                     "simd_reduce_any" => Op::MirOpBool(BinOp::BitOr),
 591                     "simd_reduce_all" => Op::MirOpBool(BinOp::BitAnd),
 592                     "simd_reduce_max" => Op::Max,
 593                     "simd_reduce_min" => Op::Min,
 594                     _ => unreachable!(),
 595                 };
 596
 597                 // Initialize with first lane, then proceed with the rest.
 598                 let mut res = this.read_immediate(&this.mplace_index(&op, 0)?.into())?;
 599                 if matches!(which, Op::MirOpBool(_)) {
 600                     // Convert to `bool` scalar.
 601                     res = imm_from_bool(simd_element_to_bool(res)?);
 602                 }
 603                 for i in 1..op_len {
 604                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 605                     res = match which {
 606                         Op::MirOp(mir_op) => {
 607                             this.binary_op(mir_op, &res, &op)?
 608                         }
 609                         Op::MirOpBool(mir_op) => {
 610                             let op = imm_from_bool(simd_element_to_bool(op)?);
 611                             this.binary_op(mir_op, &res, &op)?
 612                         }
 613                         Op::Max => {
 614                             if matches!(res.layout.ty.kind(), ty::Float(_)) {
 615                                 ImmTy::from_scalar(fmax_op(&res, &op)?, res.layout)
 616                             } else {
 617                                 // Just boring integers, so NaNs to worry about
 618                                 if this.binary_op(BinOp::Ge, &res, &op)?.to_scalar()?.to_bool()? {
 619                                     res
 620                                 } else {
 621                                     op
 622                                 }
 623                             }
 624                         }
 625                         Op::Min => {
 626                             if matches!(res.layout.ty.kind(), ty::Float(_)) {
 627                                 ImmTy::from_scalar(fmin_op(&res, &op)?, res.layout)
 628                             } else {
 629                                 // Just boring integers, so NaNs to worry about
 630                                 if this.binary_op(BinOp::Le, &res, &op)?.to_scalar()?.to_bool()? {
 631                                     res
 632                                 } else {
 633                                     op
 634                                 }
 635                             }
 636                         }
 637                     };
 638                 }
 639                 this.write_immediate(*res, dest)?;
 640             }
 641             #[rustfmt::skip]
 642             | "simd_reduce_add_ordered"
 643             | "simd_reduce_mul_ordered" => {
 644                 use mir::BinOp;
 645
 646                 let [op, init] = check_arg_count(args)?;
 647                 let (op, op_len) = this.operand_to_simd(op)?;
 648                 let init = this.read_immediate(init)?;
 649
 650                 let mir_op = match intrinsic_name {
 651                     "simd_reduce_add_ordered" => BinOp::Add,
 652                     "simd_reduce_mul_ordered" => BinOp::Mul,
 653                     _ => unreachable!(),
 654                 };
 655
 656                 let mut res = init;
 657                 for i in 0..op_len {
 658                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 659                     res = this.binary_op(mir_op, &res, &op)?;
 660                 }
 661                 this.write_immediate(*res, dest)?;
 662             }
 663             "simd_select" => {
 664                 let [mask, yes, no] = check_arg_count(args)?;
 665                 let (mask, mask_len) = this.operand_to_simd(mask)?;
 666                 let (yes, yes_len) = this.operand_to_simd(yes)?;
 667                 let (no, no_len) = this.operand_to_simd(no)?;
 668                 let (dest, dest_len) = this.place_to_simd(dest)?;
 669
 670                 assert_eq!(dest_len, mask_len);
 671                 assert_eq!(dest_len, yes_len);
 672                 assert_eq!(dest_len, no_len);
 673
 674                 for i in 0..dest_len {
 675                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
 676                     let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
 677                     let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
 678                     let dest = this.mplace_index(&dest, i)?;
 679
 680                     let val = if simd_element_to_bool(mask)? { yes } else { no };
 681                     this.write_immediate(*val, &dest.into())?;
 682                 }
 683             }
 684             "simd_select_bitmask" => {
 685                 let [mask, yes, no] = check_arg_count(args)?;
 686                 let (yes, yes_len) = this.operand_to_simd(yes)?;
 687                 let (no, no_len) = this.operand_to_simd(no)?;
 688                 let (dest, dest_len) = this.place_to_simd(dest)?;
 689                 let bitmask_len = dest_len.max(8);
 690
 691                 assert!(mask.layout.ty.is_integral());
 692                 assert!(bitmask_len <= 64);
 693                 assert_eq!(bitmask_len, mask.layout.size.bits());
 694                 assert_eq!(dest_len, yes_len);
 695                 assert_eq!(dest_len, no_len);
 696
 697                 let mask: u64 = this
 698                     .read_scalar(mask)?
 699                     .check_init()?
 700                     .to_bits(mask.layout.size)?
 701                     .try_into()
 702                     .unwrap();
 703                 for i in 0..dest_len {
 704                     let mask =
 705                         mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian));
 706                     let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
 707                     let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
 708                     let dest = this.mplace_index(&dest, i)?;
 709
 710                     let val = if mask != 0 { yes } else { no };
 711                     this.write_immediate(*val, &dest.into())?;
 712                 }
 713                 for i in dest_len..bitmask_len {
 714                     // If the mask is "padded", ensure that padding is all-zero.
 715                     let mask = mask & (1 << i);
 716                     if mask != 0 {
 717                         throw_ub_format!(
 718                             "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits"
 719                         );
 720                     }
 721                 }
 722             }
 723             #[rustfmt::skip]
 724             "simd_cast" | "simd_as" => {
 725                 let [op] = check_arg_count(args)?;
 726                 let (op, op_len) = this.operand_to_simd(op)?;
 727                 let (dest, dest_len) = this.place_to_simd(dest)?;
 728
 729                 assert_eq!(dest_len, op_len);
 730
 731                 let safe_cast = intrinsic_name == "simd_as";
 732
 733                 for i in 0..dest_len {
 734                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 735                     let dest = this.mplace_index(&dest, i)?;
 736
 737                     let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) {
 738                         // Int-to-(int|float): always safe
 739                         (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) =>
 740                             this.misc_cast(&op, dest.layout.ty)?,
 741                         // Float-to-float: always safe
 742                         (ty::Float(_), ty::Float(_)) =>
 743                             this.misc_cast(&op, dest.layout.ty)?,
 744                         // Float-to-int in safe mode
 745                         (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast =>
 746                             this.misc_cast(&op, dest.layout.ty)?,
 747                         // Float-to-int in unchecked mode
 748                         (ty::Float(FloatTy::F32), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
 749                             this.float_to_int_unchecked(op.to_scalar()?.to_f32()?, dest.layout.ty)?.into(),
 750                         (ty::Float(FloatTy::F64), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
 751                             this.float_to_int_unchecked(op.to_scalar()?.to_f64()?, dest.layout.ty)?.into(),
 752                         _ =>
 753                             throw_unsup_format!(
 754                                 "Unsupported SIMD cast from element type {} to {}",
 755                                 op.layout.ty,
 756                                 dest.layout.ty
 757                             ),
 758                     };
 759                     this.write_immediate(val, &dest.into())?;
 760                 }
 761             }
 762             "simd_shuffle" => {
 763                 let [left, right, index] = check_arg_count(args)?;
 764                 let (left, left_len) = this.operand_to_simd(left)?;
 765                 let (right, right_len) = this.operand_to_simd(right)?;
 766                 let (dest, dest_len) = this.place_to_simd(dest)?;
 767
 768                 // `index` is an array, not a SIMD type
 769                 let ty::Array(_, index_len) = index.layout.ty.kind() else {
 770                     bug!("simd_shuffle index argument has non-array type {}", index.layout.ty)
 771                 };
 772                 let index_len = index_len.eval_usize(*this.tcx, this.param_env());
 773
 774                 assert_eq!(left_len, right_len);
 775                 assert_eq!(index_len, dest_len);
 776
 777                 for i in 0..dest_len {
 778                     let src_index: u64 = this
 779                         .read_immediate(&this.operand_index(index, i)?)?
 780                         .to_scalar()?
 781                         .to_u32()?
 782                         .into();
 783                     let dest = this.mplace_index(&dest, i)?;
 784
 785                     let val = if src_index < left_len {
 786                         this.read_immediate(&this.mplace_index(&left, src_index)?.into())?
 787                     } else if src_index < left_len.checked_add(right_len).unwrap() {
 788                         this.read_immediate(
 789                             &this.mplace_index(&right, src_index - left_len)?.into(),
 790                         )?
 791                     } else {
 792                         bug!(
 793                             "simd_shuffle index {} is out of bounds for 2 vectors of size {}",
 794                             src_index,
 795                             left_len
 796                         );
 797                     };
 798                     this.write_immediate(*val, &dest.into())?;
 799                 }
 800             }
 801             "simd_gather" => {
 802                 let [passthru, ptrs, mask] = check_arg_count(args)?;
 803                 let (passthru, passthru_len) = this.operand_to_simd(passthru)?;
 804                 let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
 805                 let (mask, mask_len) = this.operand_to_simd(mask)?;
 806                 let (dest, dest_len) = this.place_to_simd(dest)?;
 807
 808                 assert_eq!(dest_len, passthru_len);
 809                 assert_eq!(dest_len, ptrs_len);
 810                 assert_eq!(dest_len, mask_len);
 811
 812                 for i in 0..dest_len {
 813                     let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?;
 814                     let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
 815                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
 816                     let dest = this.mplace_index(&dest, i)?;
 817
 818                     let val = if simd_element_to_bool(mask)? {
 819                         let place = this.deref_operand(&ptr.into())?;
 820                         this.read_immediate(&place.into())?
 821                     } else {
 822                         passthru
 823                     };
 824                     this.write_immediate(*val, &dest.into())?;
 825                 }
 826             }
 827             "simd_scatter" => {
 828                 let [value, ptrs, mask] = check_arg_count(args)?;
 829                 let (value, value_len) = this.operand_to_simd(value)?;
 830                 let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
 831                 let (mask, mask_len) = this.operand_to_simd(mask)?;
 832
 833                 assert_eq!(ptrs_len, value_len);
 834                 assert_eq!(ptrs_len, mask_len);
 835
 836                 for i in 0..ptrs_len {
 837                     let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?;
 838                     let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
 839                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
 840
 841                     if simd_element_to_bool(mask)? {
 842                         let place = this.deref_operand(&ptr.into())?;
 843                         this.write_immediate(*value, &place.into())?;
 844                     }
 845                 }
 846             }
 847             "simd_bitmask" => {
 848                 let [op] = check_arg_count(args)?;
 849                 let (op, op_len) = this.operand_to_simd(op)?;
 850                 let bitmask_len = op_len.max(8);
 851
 852                 assert!(dest.layout.ty.is_integral());
 853                 assert!(bitmask_len <= 64);
 854                 assert_eq!(bitmask_len, dest.layout.size.bits());
 855
 856                 let mut res = 0u64;
 857                 for i in 0..op_len {
 858                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 859                     if simd_element_to_bool(op)? {
 860                         res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian);
 861                     }
 862                 }
 863                 this.write_int(res, dest)?;
 864             }
 865
 866             // Atomic operations
 867             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
 868             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
 869             "atomic_load_acq" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
 870
 871             "atomic_store" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
 872             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
 873             "atomic_store_rel" => this.atomic_store(args, AtomicWriteOp::Release)?,
 874
 875             "atomic_fence_acq" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
 876             "atomic_fence_rel" => this.atomic_fence(args, AtomicFenceOp::Release)?,
 877             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
 878             "atomic_fence" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
 879
 880             "atomic_singlethreadfence_acq" => this.compiler_fence(args, AtomicFenceOp::Acquire)?,
 881             "atomic_singlethreadfence_rel" => this.compiler_fence(args, AtomicFenceOp::Release)?,
 882             "atomic_singlethreadfence_acqrel" =>
 883                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
 884             "atomic_singlethreadfence" => this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
 885
 886             "atomic_xchg" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
 887             "atomic_xchg_acq" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
 888             "atomic_xchg_rel" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
 889             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
 890             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
 891
 892             #[rustfmt::skip]
 893             "atomic_cxchg" =>
 894                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 895             #[rustfmt::skip]
 896             "atomic_cxchg_acq" =>
 897                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 898             #[rustfmt::skip]
 899             "atomic_cxchg_rel" =>
 900                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 901             #[rustfmt::skip]
 902             "atomic_cxchg_acqrel" =>
 903                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 904             #[rustfmt::skip]
 905             "atomic_cxchg_relaxed" =>
 906                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 907             #[rustfmt::skip]
 908             "atomic_cxchg_acq_failrelaxed" =>
 909                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 910             #[rustfmt::skip]
 911             "atomic_cxchg_acqrel_failrelaxed" =>
 912                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 913             #[rustfmt::skip]
 914             "atomic_cxchg_failrelaxed" =>
 915                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 916             #[rustfmt::skip]
 917             "atomic_cxchg_failacq" =>
 918                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 919
 920             #[rustfmt::skip]
 921             "atomic_cxchgweak" =>
 922                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 923             #[rustfmt::skip]
 924             "atomic_cxchgweak_acq" =>
 925                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 926             #[rustfmt::skip]
 927             "atomic_cxchgweak_rel" =>
 928                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 929             #[rustfmt::skip]
 930             "atomic_cxchgweak_acqrel" =>
 931                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 932             #[rustfmt::skip]
 933             "atomic_cxchgweak_relaxed" =>
 934                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 935             #[rustfmt::skip]
 936             "atomic_cxchgweak_acq_failrelaxed" =>
 937                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 938             #[rustfmt::skip]
 939             "atomic_cxchgweak_acqrel_failrelaxed" =>
 940                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 941             #[rustfmt::skip]
 942             "atomic_cxchgweak_failrelaxed" =>
 943                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 944             #[rustfmt::skip]
 945             "atomic_cxchgweak_failacq" =>
 946                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 947
 948             #[rustfmt::skip]
 949             "atomic_or" =>
 950                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::SeqCst)?,
 951             #[rustfmt::skip]
 952             "atomic_or_acq" =>
 953                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Acquire)?,
 954             #[rustfmt::skip]
 955             "atomic_or_rel" =>
 956                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Release)?,
 957             #[rustfmt::skip]
 958             "atomic_or_acqrel" =>
 959                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::AcqRel)?,
 960             #[rustfmt::skip]
 961             "atomic_or_relaxed" =>
 962                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Relaxed)?,
 963             #[rustfmt::skip]
 964             "atomic_xor" =>
 965                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::SeqCst)?,
 966             #[rustfmt::skip]
 967             "atomic_xor_acq" =>
 968                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Acquire)?,
 969             #[rustfmt::skip]
 970             "atomic_xor_rel" =>
 971                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Release)?,
 972             #[rustfmt::skip]
 973             "atomic_xor_acqrel" =>
 974                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::AcqRel)?,
 975             #[rustfmt::skip]
 976             "atomic_xor_relaxed" =>
 977                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Relaxed)?,
 978             #[rustfmt::skip]
 979             "atomic_and" =>
 980                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::SeqCst)?,
 981             #[rustfmt::skip]
 982             "atomic_and_acq" =>
 983                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Acquire)?,
 984             #[rustfmt::skip]
 985             "atomic_and_rel" =>
 986                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Release)?,
 987             #[rustfmt::skip]
 988             "atomic_and_acqrel" =>
 989                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::AcqRel)?,
 990             #[rustfmt::skip]
 991             "atomic_and_relaxed" =>
 992                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Relaxed)?,
 993             #[rustfmt::skip]
 994             "atomic_nand" =>
 995                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::SeqCst)?,
 996             #[rustfmt::skip]
 997             "atomic_nand_acq" =>
 998                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Acquire)?,
 999             #[rustfmt::skip]
1000             "atomic_nand_rel" =>
1001                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Release)?,
1002             #[rustfmt::skip]
1003             "atomic_nand_acqrel" =>
1004                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::AcqRel)?,
1005             #[rustfmt::skip]
1006             "atomic_nand_relaxed" =>
1007                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Relaxed)?,
1008             #[rustfmt::skip]
1009             "atomic_xadd" =>
1010                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::SeqCst)?,
1011             #[rustfmt::skip]
1012             "atomic_xadd_acq" =>
1013                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Acquire)?,
1014             #[rustfmt::skip]
1015             "atomic_xadd_rel" =>
1016                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Release)?,
1017             #[rustfmt::skip]
1018             "atomic_xadd_acqrel" =>
1019                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::AcqRel)?,
1020             #[rustfmt::skip]
1021             "atomic_xadd_relaxed" =>
1022                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Relaxed)?,
1023             #[rustfmt::skip]
1024             "atomic_xsub" =>
1025                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::SeqCst)?,
1026             #[rustfmt::skip]
1027             "atomic_xsub_acq" =>
1028                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Acquire)?,
1029             #[rustfmt::skip]
1030             "atomic_xsub_rel" =>
1031                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Release)?,
1032             #[rustfmt::skip]
1033             "atomic_xsub_acqrel" =>
1034                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::AcqRel)?,
1035             #[rustfmt::skip]
1036             "atomic_xsub_relaxed" =>
1037                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Relaxed)?,
1038             "atomic_min" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
1039             "atomic_min_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
1040             "atomic_min_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
1041             "atomic_min_acqrel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
1042             "atomic_min_relaxed" =>
1043                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
1044             "atomic_max" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
1045             "atomic_max_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
1046             "atomic_max_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
1047             "atomic_max_acqrel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
1048             "atomic_max_relaxed" =>
1049                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
1050             "atomic_umin" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
1051             "atomic_umin_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
1052             "atomic_umin_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
1053             "atomic_umin_acqrel" =>
1054                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
1055             "atomic_umin_relaxed" =>
1056                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
1057             "atomic_umax" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
1058             "atomic_umax_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
1059             "atomic_umax_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
1060             "atomic_umax_acqrel" =>
1061                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
1062             "atomic_umax_relaxed" =>
1063                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
1064
1065             // Other
1066             "exact_div" => {
1067                 let [num, denom] = check_arg_count(args)?;
1068                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
1069             }
1070
1071             "try" => return this.handle_try(args, dest, ret),
1072
1073             "breakpoint" => {
1074                 let [] = check_arg_count(args)?;
1075                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
1076                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
1077             }
1078
1079             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
1080         }
1081
1082         trace!("{:?}", this.dump_place(**dest));
1083         this.go_to_block(ret);
1084         Ok(())
1085     }
1086
1087     fn atomic_load(
1088         &mut self,
1089         args: &[OpTy<'tcx, Tag>],
1090         dest: &PlaceTy<'tcx, Tag>,
1091         atomic: AtomicReadOp,
1092     ) -> InterpResult<'tcx> {
1093         let this = self.eval_context_mut();
1094
1095         let [place] = check_arg_count(args)?;
1096         let place = this.deref_operand(place)?;
1097
1098         // make sure it fits into a scalar; otherwise it cannot be atomic
1099         let val = this.read_scalar_atomic(&place, atomic)?;
1100
1101         // Check alignment requirements. Atomics must always be aligned to their size,
1102         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1103         // be 8-aligned).
1104         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1105         this.check_ptr_access_align(
1106             place.ptr,
1107             place.layout.size,
1108             align,
1109             CheckInAllocMsg::MemoryAccessTest,
1110         )?;
1111         // Perform regular access.
1112         this.write_scalar(val, dest)?;
1113         Ok(())
1114     }
1115
1116     fn atomic_store(
1117         &mut self,
1118         args: &[OpTy<'tcx, Tag>],
1119         atomic: AtomicWriteOp,
1120     ) -> InterpResult<'tcx> {
1121         let this = self.eval_context_mut();
1122
1123         let [place, val] = check_arg_count(args)?;
1124         let place = this.deref_operand(place)?;
1125         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
1126
1127         // Check alignment requirements. Atomics must always be aligned to their size,
1128         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1129         // be 8-aligned).
1130         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1131         this.check_ptr_access_align(
1132             place.ptr,
1133             place.layout.size,
1134             align,
1135             CheckInAllocMsg::MemoryAccessTest,
1136         )?;
1137
1138         // Perform atomic store
1139         this.write_scalar_atomic(val, &place, atomic)?;
1140         Ok(())
1141     }
1142
1143     fn compiler_fence(
1144         &mut self,
1145         args: &[OpTy<'tcx, Tag>],
1146         atomic: AtomicFenceOp,
1147     ) -> InterpResult<'tcx> {
1148         let [] = check_arg_count(args)?;
1149         let _ = atomic;
1150         //FIXME: compiler fences are currently ignored
1151         Ok(())
1152     }
1153
1154     fn atomic_fence(
1155         &mut self,
1156         args: &[OpTy<'tcx, Tag>],
1157         atomic: AtomicFenceOp,
1158     ) -> InterpResult<'tcx> {
1159         let this = self.eval_context_mut();
1160         let [] = check_arg_count(args)?;
1161         this.validate_atomic_fence(atomic)?;
1162         Ok(())
1163     }
1164
1165     fn atomic_op(
1166         &mut self,
1167         args: &[OpTy<'tcx, Tag>],
1168         dest: &PlaceTy<'tcx, Tag>,
1169         atomic_op: AtomicOp,
1170         atomic: AtomicRwOp,
1171     ) -> InterpResult<'tcx> {
1172         let this = self.eval_context_mut();
1173
1174         let [place, rhs] = check_arg_count(args)?;
1175         let place = this.deref_operand(place)?;
1176
1177         if !place.layout.ty.is_integral() {
1178             bug!("Atomic arithmetic operations only work on integer types");
1179         }
1180         let rhs = this.read_immediate(rhs)?;
1181
1182         // Check alignment requirements. Atomics must always be aligned to their size,
1183         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1184         // be 8-aligned).
1185         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1186         this.check_ptr_access_align(
1187             place.ptr,
1188             place.layout.size,
1189             align,
1190             CheckInAllocMsg::MemoryAccessTest,
1191         )?;
1192
1193         match atomic_op {
1194             AtomicOp::Min => {
1195                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
1196                 this.write_immediate(*old, dest)?; // old value is returned
1197                 Ok(())
1198             }
1199             AtomicOp::Max => {
1200                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
1201                 this.write_immediate(*old, dest)?; // old value is returned
1202                 Ok(())
1203             }
1204             AtomicOp::MirOp(op, neg) => {
1205                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
1206                 this.write_immediate(*old, dest)?; // old value is returned
1207                 Ok(())
1208             }
1209         }
1210     }
1211
1212     fn atomic_exchange(
1213         &mut self,
1214         args: &[OpTy<'tcx, Tag>],
1215         dest: &PlaceTy<'tcx, Tag>,
1216         atomic: AtomicRwOp,
1217     ) -> InterpResult<'tcx> {
1218         let this = self.eval_context_mut();
1219
1220         let [place, new] = check_arg_count(args)?;
1221         let place = this.deref_operand(place)?;
1222         let new = this.read_scalar(new)?;
1223
1224         // Check alignment requirements. Atomics must always be aligned to their size,
1225         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1226         // be 8-aligned).
1227         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1228         this.check_ptr_access_align(
1229             place.ptr,
1230             place.layout.size,
1231             align,
1232             CheckInAllocMsg::MemoryAccessTest,
1233         )?;
1234
1235         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
1236         this.write_scalar(old, dest)?; // old value is returned
1237         Ok(())
1238     }
1239
1240     fn atomic_compare_exchange_impl(
1241         &mut self,
1242         args: &[OpTy<'tcx, Tag>],
1243         dest: &PlaceTy<'tcx, Tag>,
1244         success: AtomicRwOp,
1245         fail: AtomicReadOp,
1246         can_fail_spuriously: bool,
1247     ) -> InterpResult<'tcx> {
1248         let this = self.eval_context_mut();
1249
1250         let [place, expect_old, new] = check_arg_count(args)?;
1251         let place = this.deref_operand(place)?;
1252         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
1253         let new = this.read_scalar(new)?;
1254
1255         // Check alignment requirements. Atomics must always be aligned to their size,
1256         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1257         // be 8-aligned).
1258         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1259         this.check_ptr_access_align(
1260             place.ptr,
1261             place.layout.size,
1262             align,
1263             CheckInAllocMsg::MemoryAccessTest,
1264         )?;
1265
1266         let old = this.atomic_compare_exchange_scalar(
1267             &place,
1268             &expect_old,
1269             new,
1270             success,
1271             fail,
1272             can_fail_spuriously,
1273         )?;
1274
1275         // Return old value.
1276         this.write_immediate(old, dest)?;
1277         Ok(())
1278     }
1279
1280     fn atomic_compare_exchange(
1281         &mut self,
1282         args: &[OpTy<'tcx, Tag>],
1283         dest: &PlaceTy<'tcx, Tag>,
1284         success: AtomicRwOp,
1285         fail: AtomicReadOp,
1286     ) -> InterpResult<'tcx> {
1287         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
1288     }
1289
1290     fn atomic_compare_exchange_weak(
1291         &mut self,
1292         args: &[OpTy<'tcx, Tag>],
1293         dest: &PlaceTy<'tcx, Tag>,
1294         success: AtomicRwOp,
1295         fail: AtomicReadOp,
1296     ) -> InterpResult<'tcx> {
1297         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
1298     }
1299
1300     fn float_to_int_unchecked<F>(
1301         &self,
1302         f: F,
1303         dest_ty: ty::Ty<'tcx>,
1304     ) -> InterpResult<'tcx, Scalar<Tag>>
1305     where
1306         F: Float + Into<Scalar<Tag>>,
1307     {
1308         let this = self.eval_context_ref();
1309
1310         // Step 1: cut off the fractional part of `f`. The result of this is
1311         // guaranteed to be precisely representable in IEEE floats.
1312         let f = f.round_to_integral(Round::TowardZero).value;
1313
1314         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
1315         Ok(match dest_ty.kind() {
1316             // Unsigned
1317             ty::Uint(t) => {
1318                 let size = Integer::from_uint_ty(this, *t).size();
1319                 let res = f.to_u128(size.bits_usize());
1320                 if res.status.is_empty() {
1321                     // No status flags means there was no further rounding or other loss of precision.
1322                     Scalar::from_uint(res.value, size)
1323                 } else {
1324                     // `f` was not representable in this integer type.
1325                     throw_ub_format!(
1326                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
1327                         f,
1328                         dest_ty,
1329                     );
1330                 }
1331             }
1332             // Signed
1333             ty::Int(t) => {
1334                 let size = Integer::from_int_ty(this, *t).size();
1335                 let res = f.to_i128(size.bits_usize());
1336                 if res.status.is_empty() {
1337                     // No status flags means there was no further rounding or other loss of precision.
1338                     Scalar::from_int(res.value, size)
1339                 } else {
1340                     // `f` was not representable in this integer type.
1341                     throw_ub_format!(
1342                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
1343                         f,
1344                         dest_ty,
1345                     );
1346                 }
1347             }
1348             // Nothing else
1349             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
1350         })
1351     }
1352 }
1353
1354 fn fmax_op<'tcx>(
1355     left: &ImmTy<'tcx, Tag>,
1356     right: &ImmTy<'tcx, Tag>,
1357 ) -> InterpResult<'tcx, Scalar<Tag>> {
1358     assert_eq!(left.layout.ty, right.layout.ty);
1359     let ty::Float(float_ty) = left.layout.ty.kind() else {
1360         bug!("fmax operand is not a float")
1361     };
1362     let left = left.to_scalar()?;
1363     let right = right.to_scalar()?;
1364     Ok(match float_ty {
1365         FloatTy::F32 => Scalar::from_f32(left.to_f32()?.max(right.to_f32()?)),
1366         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.max(right.to_f64()?)),
1367     })
1368 }
1369
1370 fn fmin_op<'tcx>(
1371     left: &ImmTy<'tcx, Tag>,
1372     right: &ImmTy<'tcx, Tag>,
1373 ) -> InterpResult<'tcx, Scalar<Tag>> {
1374     assert_eq!(left.layout.ty, right.layout.ty);
1375     let ty::Float(float_ty) = left.layout.ty.kind() else {
1376         bug!("fmin operand is not a float")
1377     };
1378     let left = left.to_scalar()?;
1379     let right = right.to_scalar()?;
1380     Ok(match float_ty {
1381         FloatTy::F32 => Scalar::from_f32(left.to_f32()?.min(right.to_f32()?)),
1382         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)),
1383     })
1384 }
1385
1386 fn bool_to_simd_element(b: bool, size: Size) -> Scalar<Tag> {
1387     // SIMD uses all-1 as pattern for "true"
1388     let val = if b { -1 } else { 0 };
1389     Scalar::from_int(val, size)
1390 }
1391
1392 fn simd_element_to_bool(elem: ImmTy<'_, Tag>) -> InterpResult<'_, bool> {
1393     let val = elem.to_scalar()?.to_int(elem.layout.size)?;
1394     Ok(match val {
1395         0 => false,
1396         -1 => true,
1397         _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"),
1398     })
1399 }
1400
1401 fn simd_bitmask_index(idx: u64, vec_len: u64, endianess: Endian) -> u64 {
1402     assert!(idx < vec_len);
1403     match endianess {
1404         Endian::Little => idx,
1405         Endian::Big => vec_len - 1 - idx, // reverse order of bits
1406     }
1407 }