src/shims/intrinsics.rs

   1 use std::iter;
   2
   3 use log::trace;
   4
   5 use rustc_apfloat::{Float, Round};
   6 use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf};
   7 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
   8 use rustc_target::abi::{Align, Integer};
   9
  10 use crate::*;
  11 use helpers::{bool_to_simd_element, check_arg_count, simd_element_to_bool};
  12
  13 pub enum AtomicOp {
  14     MirOp(mir::BinOp, bool),
  15     Max,
  16     Min,
  17 }
  18
  19 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  20 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  21     fn call_intrinsic(
  22         &mut self,
  23         instance: ty::Instance<'tcx>,
  24         args: &[OpTy<'tcx, Tag>],
  25         ret: Option<(&PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  26         _unwind: StackPopUnwind,
  27     ) -> InterpResult<'tcx> {
  28         let this = self.eval_context_mut();
  29
  30         if this.emulate_intrinsic(instance, args, ret)? {
  31             return Ok(());
  32         }
  33
  34         // All supported intrinsics have a return place.
  35         let intrinsic_name = this.tcx.item_name(instance.def_id());
  36         let intrinsic_name = intrinsic_name.as_str();
  37         let (dest, ret) = match ret {
  38             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  39             Some(p) => p,
  40         };
  41
  42         // Then handle terminating intrinsics.
  43         match intrinsic_name {
  44             // Miri overwriting CTFE intrinsics.
  45             "ptr_guaranteed_eq" => {
  46                 let &[ref left, ref right] = check_arg_count(args)?;
  47                 let left = this.read_immediate(left)?;
  48                 let right = this.read_immediate(right)?;
  49                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  50             }
  51             "ptr_guaranteed_ne" => {
  52                 let &[ref left, ref right] = check_arg_count(args)?;
  53                 let left = this.read_immediate(left)?;
  54                 let right = this.read_immediate(right)?;
  55                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  56             }
  57             "const_allocate" => {
  58                 // For now, for compatibility with the run-time implementation of this, we just return null.
  59                 // See <https://github.com/rust-lang/rust/issues/93935>.
  60                 this.write_null(dest)?;
  61             }
  62             "const_deallocate" => {
  63                 // complete NOP
  64             }
  65
  66             // Raw memory accesses
  67             "volatile_load" => {
  68                 let &[ref place] = check_arg_count(args)?;
  69                 let place = this.deref_operand(place)?;
  70                 this.copy_op(&place.into(), dest)?;
  71             }
  72             "volatile_store" => {
  73                 let &[ref place, ref dest] = check_arg_count(args)?;
  74                 let place = this.deref_operand(place)?;
  75                 this.copy_op(dest, &place.into())?;
  76             }
  77
  78             "write_bytes" | "volatile_set_memory" => {
  79                 let &[ref ptr, ref val_byte, ref count] = check_arg_count(args)?;
  80                 let ty = instance.substs.type_at(0);
  81                 let ty_layout = this.layout_of(ty)?;
  82                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
  83                 let ptr = this.read_pointer(ptr)?;
  84                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  85                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
  86                     err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
  87                 })?;
  88                 this.memory
  89                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
  90             }
  91
  92             // Floating-point operations
  93             #[rustfmt::skip]
  94             | "sinf32"
  95             | "fabsf32"
  96             | "cosf32"
  97             | "sqrtf32"
  98             | "expf32"
  99             | "exp2f32"
 100             | "logf32"
 101             | "log10f32"
 102             | "log2f32"
 103             | "floorf32"
 104             | "ceilf32"
 105             | "truncf32"
 106             | "roundf32"
 107             => {
 108                 let &[ref f] = check_arg_count(args)?;
 109                 // FIXME: Using host floats.
 110                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 111                 let f = match intrinsic_name {
 112                     "sinf32" => f.sin(),
 113                     "fabsf32" => f.abs(),
 114                     "cosf32" => f.cos(),
 115                     "sqrtf32" => f.sqrt(),
 116                     "expf32" => f.exp(),
 117                     "exp2f32" => f.exp2(),
 118                     "logf32" => f.ln(),
 119                     "log10f32" => f.log10(),
 120                     "log2f32" => f.log2(),
 121                     "floorf32" => f.floor(),
 122                     "ceilf32" => f.ceil(),
 123                     "truncf32" => f.trunc(),
 124                     "roundf32" => f.round(),
 125                     _ => bug!(),
 126                 };
 127                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 128             }
 129
 130             #[rustfmt::skip]
 131             | "sinf64"
 132             | "fabsf64"
 133             | "cosf64"
 134             | "sqrtf64"
 135             | "expf64"
 136             | "exp2f64"
 137             | "logf64"
 138             | "log10f64"
 139             | "log2f64"
 140             | "floorf64"
 141             | "ceilf64"
 142             | "truncf64"
 143             | "roundf64"
 144             => {
 145                 let &[ref f] = check_arg_count(args)?;
 146                 // FIXME: Using host floats.
 147                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 148                 let f = match intrinsic_name {
 149                     "sinf64" => f.sin(),
 150                     "fabsf64" => f.abs(),
 151                     "cosf64" => f.cos(),
 152                     "sqrtf64" => f.sqrt(),
 153                     "expf64" => f.exp(),
 154                     "exp2f64" => f.exp2(),
 155                     "logf64" => f.ln(),
 156                     "log10f64" => f.log10(),
 157                     "log2f64" => f.log2(),
 158                     "floorf64" => f.floor(),
 159                     "ceilf64" => f.ceil(),
 160                     "truncf64" => f.trunc(),
 161                     "roundf64" => f.round(),
 162                     _ => bug!(),
 163                 };
 164                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 165             }
 166
 167             #[rustfmt::skip]
 168             | "fadd_fast"
 169             | "fsub_fast"
 170             | "fmul_fast"
 171             | "fdiv_fast"
 172             | "frem_fast"
 173             => {
 174                 let &[ref a, ref b] = check_arg_count(args)?;
 175                 let a = this.read_immediate(a)?;
 176                 let b = this.read_immediate(b)?;
 177                 let op = match intrinsic_name {
 178                     "fadd_fast" => mir::BinOp::Add,
 179                     "fsub_fast" => mir::BinOp::Sub,
 180                     "fmul_fast" => mir::BinOp::Mul,
 181                     "fdiv_fast" => mir::BinOp::Div,
 182                     "frem_fast" => mir::BinOp::Rem,
 183                     _ => bug!(),
 184                 };
 185                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 186                     Ok(match x.layout.ty.kind() {
 187                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 188                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 189                         _ => bug!(
 190                             "`{}` called with non-float input type {:?}",
 191                             intrinsic_name,
 192                             x.layout.ty
 193                         ),
 194                     })
 195                 };
 196                 match (float_finite(a)?, float_finite(b)?) {
 197                     (false, false) => throw_ub_format!(
 198                         "`{}` intrinsic called with non-finite value as both parameters",
 199                         intrinsic_name,
 200                     ),
 201                     (false, _) => throw_ub_format!(
 202                         "`{}` intrinsic called with non-finite value as first parameter",
 203                         intrinsic_name,
 204                     ),
 205                     (_, false) => throw_ub_format!(
 206                         "`{}` intrinsic called with non-finite value as second parameter",
 207                         intrinsic_name,
 208                     ),
 209                     _ => {}
 210                 }
 211                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 212             }
 213
 214             #[rustfmt::skip]
 215             | "minnumf32"
 216             | "maxnumf32"
 217             | "copysignf32"
 218             => {
 219                 let &[ref a, ref b] = check_arg_count(args)?;
 220                 let a = this.read_scalar(a)?.to_f32()?;
 221                 let b = this.read_scalar(b)?.to_f32()?;
 222                 let res = match intrinsic_name {
 223                     "minnumf32" => a.min(b),
 224                     "maxnumf32" => a.max(b),
 225                     "copysignf32" => a.copy_sign(b),
 226                     _ => bug!(),
 227                 };
 228                 this.write_scalar(Scalar::from_f32(res), dest)?;
 229             }
 230
 231             #[rustfmt::skip]
 232             | "minnumf64"
 233             | "maxnumf64"
 234             | "copysignf64"
 235             => {
 236                 let &[ref a, ref b] = check_arg_count(args)?;
 237                 let a = this.read_scalar(a)?.to_f64()?;
 238                 let b = this.read_scalar(b)?.to_f64()?;
 239                 let res = match intrinsic_name {
 240                     "minnumf64" => a.min(b),
 241                     "maxnumf64" => a.max(b),
 242                     "copysignf64" => a.copy_sign(b),
 243                     _ => bug!(),
 244                 };
 245                 this.write_scalar(Scalar::from_f64(res), dest)?;
 246             }
 247
 248             "powf32" => {
 249                 let &[ref f, ref f2] = check_arg_count(args)?;
 250                 // FIXME: Using host floats.
 251                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 252                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 253                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 254             }
 255
 256             "powf64" => {
 257                 let &[ref f, ref f2] = check_arg_count(args)?;
 258                 // FIXME: Using host floats.
 259                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 260                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 261                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 262             }
 263
 264             "fmaf32" => {
 265                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 266                 let a = this.read_scalar(a)?.to_f32()?;
 267                 let b = this.read_scalar(b)?.to_f32()?;
 268                 let c = this.read_scalar(c)?.to_f32()?;
 269                 let res = a.mul_add(b, c).value;
 270                 this.write_scalar(Scalar::from_f32(res), dest)?;
 271             }
 272
 273             "fmaf64" => {
 274                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 275                 let a = this.read_scalar(a)?.to_f64()?;
 276                 let b = this.read_scalar(b)?.to_f64()?;
 277                 let c = this.read_scalar(c)?.to_f64()?;
 278                 let res = a.mul_add(b, c).value;
 279                 this.write_scalar(Scalar::from_f64(res), dest)?;
 280             }
 281
 282             "powif32" => {
 283                 let &[ref f, ref i] = check_arg_count(args)?;
 284                 // FIXME: Using host floats.
 285                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 286                 let i = this.read_scalar(i)?.to_i32()?;
 287                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 288             }
 289
 290             "powif64" => {
 291                 let &[ref f, ref i] = check_arg_count(args)?;
 292                 // FIXME: Using host floats.
 293                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 294                 let i = this.read_scalar(i)?.to_i32()?;
 295                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 296             }
 297
 298             "float_to_int_unchecked" => {
 299                 let &[ref val] = check_arg_count(args)?;
 300                 let val = this.read_immediate(val)?;
 301
 302                 let res = match val.layout.ty.kind() {
 303                     ty::Float(FloatTy::F32) =>
 304                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 305                     ty::Float(FloatTy::F64) =>
 306                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 307                     _ =>
 308                         bug!(
 309                             "`float_to_int_unchecked` called with non-float input type {:?}",
 310                             val.layout.ty
 311                         ),
 312                 };
 313
 314                 this.write_scalar(res, dest)?;
 315             }
 316
 317             // SIMD operations
 318             #[rustfmt::skip]
 319             | "simd_neg"
 320             | "simd_fabs" => {
 321                 let &[ref op] = check_arg_count(args)?;
 322                 let (op, op_len) = this.operand_to_simd(op)?;
 323                 let (dest, dest_len) = this.place_to_simd(dest)?;
 324
 325                 assert_eq!(dest_len, op_len);
 326
 327                 enum Op {
 328                     MirOp(mir::UnOp),
 329                     Abs,
 330                 }
 331                 let which = match intrinsic_name {
 332                     "simd_neg" => Op::MirOp(mir::UnOp::Neg),
 333                     "simd_fabs" => Op::Abs,
 334                     _ => unreachable!(),
 335                 };
 336
 337                 for i in 0..dest_len {
 338                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 339                     let dest = this.mplace_index(&dest, i)?;
 340                     let val = match which {
 341                         Op::MirOp(mir_op) => this.unary_op(mir_op, &op)?.to_scalar()?,
 342                         Op::Abs => {
 343                             // Works for f32 and f64.
 344                             let ty::Float(float_ty) = op.layout.ty.kind() else {
 345                                 bug!("simd_fabs operand is not a float")
 346                             };
 347                             let op = op.to_scalar()?;
 348                             match float_ty {
 349                                 FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()),
 350                                 FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()),
 351                             }
 352                         }
 353                     };
 354                     this.write_scalar(val, &dest.into())?;
 355                 }
 356             }
 357             #[rustfmt::skip]
 358             | "simd_add"
 359             | "simd_sub"
 360             | "simd_mul"
 361             | "simd_div"
 362             | "simd_rem"
 363             | "simd_shl"
 364             | "simd_shr"
 365             | "simd_and"
 366             | "simd_or"
 367             | "simd_xor"
 368             | "simd_eq"
 369             | "simd_ne"
 370             | "simd_lt"
 371             | "simd_le"
 372             | "simd_gt"
 373             | "simd_ge"
 374             | "simd_fmax"
 375             | "simd_fmin"
 376             | "simd_saturating_add"
 377             | "simd_saturating_sub" => {
 378                 use mir::BinOp;
 379
 380                 let &[ref left, ref right] = check_arg_count(args)?;
 381                 let (left, left_len) = this.operand_to_simd(left)?;
 382                 let (right, right_len) = this.operand_to_simd(right)?;
 383                 let (dest, dest_len) = this.place_to_simd(dest)?;
 384
 385                 assert_eq!(dest_len, left_len);
 386                 assert_eq!(dest_len, right_len);
 387
 388                 enum Op {
 389                     MirOp(BinOp),
 390                     SaturatingOp(BinOp),
 391                     FMax,
 392                     FMin,
 393                 }
 394                 let which = match intrinsic_name {
 395                     "simd_add" => Op::MirOp(BinOp::Add),
 396                     "simd_sub" => Op::MirOp(BinOp::Sub),
 397                     "simd_mul" => Op::MirOp(BinOp::Mul),
 398                     "simd_div" => Op::MirOp(BinOp::Div),
 399                     "simd_rem" => Op::MirOp(BinOp::Rem),
 400                     "simd_shl" => Op::MirOp(BinOp::Shl),
 401                     "simd_shr" => Op::MirOp(BinOp::Shr),
 402                     "simd_and" => Op::MirOp(BinOp::BitAnd),
 403                     "simd_or" => Op::MirOp(BinOp::BitOr),
 404                     "simd_xor" => Op::MirOp(BinOp::BitXor),
 405                     "simd_eq" => Op::MirOp(BinOp::Eq),
 406                     "simd_ne" => Op::MirOp(BinOp::Ne),
 407                     "simd_lt" => Op::MirOp(BinOp::Lt),
 408                     "simd_le" => Op::MirOp(BinOp::Le),
 409                     "simd_gt" => Op::MirOp(BinOp::Gt),
 410                     "simd_ge" => Op::MirOp(BinOp::Ge),
 411                     "simd_fmax" => Op::FMax,
 412                     "simd_fmin" => Op::FMin,
 413                     "simd_saturating_add" => Op::SaturatingOp(BinOp::Add),
 414                     "simd_saturating_sub" => Op::SaturatingOp(BinOp::Sub),
 415                     _ => unreachable!(),
 416                 };
 417
 418                 for i in 0..dest_len {
 419                     let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
 420                     let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
 421                     let dest = this.mplace_index(&dest, i)?;
 422                     let val = match which {
 423                         Op::MirOp(mir_op) => {
 424                             let (val, overflowed, ty) = this.overflowing_binary_op(mir_op, &left, &right)?;
 425                             if matches!(mir_op, BinOp::Shl | BinOp::Shr) {
 426                                 // Shifts have extra UB as SIMD operations that the MIR binop does not have.
 427                                 // See <https://github.com/rust-lang/rust/issues/91237>.
 428                                 if overflowed {
 429                                     let r_val = right.to_scalar()?.to_bits(right.layout.size)?;
 430                                     throw_ub_format!("overflowing shift by {} in `{}` in SIMD lane {}", r_val, intrinsic_name, i);
 431                                 }
 432                             }
 433                             if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) {
 434                                 // Special handling for boolean-returning operations
 435                                 assert_eq!(ty, this.tcx.types.bool);
 436                                 let val = val.to_bool().unwrap();
 437                                 bool_to_simd_element(val, dest.layout.size)
 438                             } else {
 439                                 assert_ne!(ty, this.tcx.types.bool);
 440                                 assert_eq!(ty, dest.layout.ty);
 441                                 val
 442                             }
 443                         }
 444                         Op::FMax => {
 445                             fmax_op(&left, &right)?
 446                         }
 447                         Op::FMin => {
 448                             fmin_op(&left, &right)?
 449                         }
 450                         Op::SaturatingOp(mir_op) => {
 451                             this.saturating_arith(mir_op, &left, &right)?
 452                         }
 453                     };
 454                     this.write_scalar(val, &dest.into())?;
 455                 }
 456             }
 457             #[rustfmt::skip]
 458             | "simd_reduce_and"
 459             | "simd_reduce_or"
 460             | "simd_reduce_xor"
 461             | "simd_reduce_any"
 462             | "simd_reduce_all"
 463             | "simd_reduce_max"
 464             | "simd_reduce_min" => {
 465                 use mir::BinOp;
 466
 467                 let &[ref op] = check_arg_count(args)?;
 468                 let (op, op_len) = this.operand_to_simd(op)?;
 469
 470                 let imm_from_bool =
 471                     |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool);
 472
 473                 enum Op {
 474                     MirOp(BinOp),
 475                     MirOpBool(BinOp),
 476                     Max,
 477                     Min,
 478                 }
 479                 let which = match intrinsic_name {
 480                     "simd_reduce_and" => Op::MirOp(BinOp::BitAnd),
 481                     "simd_reduce_or" => Op::MirOp(BinOp::BitOr),
 482                     "simd_reduce_xor" => Op::MirOp(BinOp::BitXor),
 483                     "simd_reduce_any" => Op::MirOpBool(BinOp::BitOr),
 484                     "simd_reduce_all" => Op::MirOpBool(BinOp::BitAnd),
 485                     "simd_reduce_max" => Op::Max,
 486                     "simd_reduce_min" => Op::Min,
 487                     _ => unreachable!(),
 488                 };
 489
 490                 // Initialize with first lane, then proceed with the rest.
 491                 let mut res = this.read_immediate(&this.mplace_index(&op, 0)?.into())?;
 492                 if matches!(which, Op::MirOpBool(_)) {
 493                     // Convert to `bool` scalar.
 494                     res = imm_from_bool(simd_element_to_bool(res)?);
 495                 }
 496                 for i in 1..op_len {
 497                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 498                     res = match which {
 499                         Op::MirOp(mir_op) => {
 500                             this.binary_op(mir_op, &res, &op)?
 501                         }
 502                         Op::MirOpBool(mir_op) => {
 503                             let op = imm_from_bool(simd_element_to_bool(op)?);
 504                             this.binary_op(mir_op, &res, &op)?
 505                         }
 506                         Op::Max => {
 507                             if matches!(res.layout.ty.kind(), ty::Float(_)) {
 508                                 ImmTy::from_scalar(fmax_op(&res, &op)?, res.layout)
 509                             } else {
 510                                 // Just boring integers, so NaNs to worry about
 511                                 if this.binary_op(BinOp::Ge, &res, &op)?.to_scalar()?.to_bool()? {
 512                                     res
 513                                 } else {
 514                                     op
 515                                 }
 516                             }
 517                         }
 518                         Op::Min => {
 519                             if matches!(res.layout.ty.kind(), ty::Float(_)) {
 520                                 ImmTy::from_scalar(fmin_op(&res, &op)?, res.layout)
 521                             } else {
 522                                 // Just boring integers, so NaNs to worry about
 523                                 if this.binary_op(BinOp::Le, &res, &op)?.to_scalar()?.to_bool()? {
 524                                     res
 525                                 } else {
 526                                     op
 527                                 }
 528                             }
 529                         }
 530                     };
 531                 }
 532                 this.write_immediate(*res, dest)?;
 533             }
 534             #[rustfmt::skip]
 535             | "simd_reduce_add_ordered"
 536             | "simd_reduce_mul_ordered" => {
 537                 use mir::BinOp;
 538
 539                 let &[ref op, ref init] = check_arg_count(args)?;
 540                 let (op, op_len) = this.operand_to_simd(op)?;
 541                 let init = this.read_immediate(init)?;
 542
 543                 let mir_op = match intrinsic_name {
 544                     "simd_reduce_add_ordered" => BinOp::Add,
 545                     "simd_reduce_mul_ordered" => BinOp::Mul,
 546                     _ => unreachable!(),
 547                 };
 548
 549                 let mut res = init;
 550                 for i in 0..op_len {
 551                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 552                     res = this.binary_op(mir_op, &res, &op)?;
 553                 }
 554                 this.write_immediate(*res, dest)?;
 555             }
 556             "simd_select" => {
 557                 let &[ref mask, ref yes, ref no] = check_arg_count(args)?;
 558                 let (mask, mask_len) = this.operand_to_simd(mask)?;
 559                 let (yes, yes_len) = this.operand_to_simd(yes)?;
 560                 let (no, no_len) = this.operand_to_simd(no)?;
 561                 let (dest, dest_len) = this.place_to_simd(dest)?;
 562
 563                 assert_eq!(dest_len, mask_len);
 564                 assert_eq!(dest_len, yes_len);
 565                 assert_eq!(dest_len, no_len);
 566
 567                 for i in 0..dest_len {
 568                     let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
 569                     let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
 570                     let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
 571                     let dest = this.mplace_index(&dest, i)?;
 572
 573                     let mask = simd_element_to_bool(mask)?;
 574                     let val = if mask { yes } else { no };
 575                     this.write_immediate(*val, &dest.into())?;
 576                 }
 577             }
 578             #[rustfmt::skip]
 579             "simd_cast" | "simd_as" => {
 580                 let &[ref op] = check_arg_count(args)?;
 581                 let (op, op_len) = this.operand_to_simd(op)?;
 582                 let (dest, dest_len) = this.place_to_simd(dest)?;
 583
 584                 assert_eq!(dest_len, op_len);
 585
 586                 let safe_cast = intrinsic_name == "simd_as";
 587
 588                 for i in 0..dest_len {
 589                     let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
 590                     let dest = this.mplace_index(&dest, i)?;
 591
 592                     let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) {
 593                         // Int-to-(int|float): always safe
 594                         (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) =>
 595                             this.misc_cast(&op, dest.layout.ty)?,
 596                         // Float-to-float: always safe
 597                         (ty::Float(_), ty::Float(_)) =>
 598                             this.misc_cast(&op, dest.layout.ty)?,
 599                         // Float-to-int in safe mode
 600                         (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast =>
 601                             this.misc_cast(&op, dest.layout.ty)?,
 602                         // Float-to-int in unchecked mode
 603                         (ty::Float(FloatTy::F32), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
 604                             this.float_to_int_unchecked(op.to_scalar()?.to_f32()?, dest.layout.ty)?.into(),
 605                         (ty::Float(FloatTy::F64), ty::Int(_) | ty::Uint(_)) if !safe_cast =>
 606                             this.float_to_int_unchecked(op.to_scalar()?.to_f64()?, dest.layout.ty)?.into(),
 607                         _ =>
 608                             throw_unsup_format!(
 609                                 "Unsupported SIMD cast from element type {} to {}",
 610                                 op.layout.ty,
 611                                 dest.layout.ty
 612                             ),
 613                     };
 614                     this.write_immediate(val, &dest.into())?;
 615                 }
 616             }
 617             "simd_shuffle" => {
 618                 let &[ref left, ref right, ref index] = check_arg_count(args)?;
 619                 let (left, left_len) = this.operand_to_simd(left)?;
 620                 let (right, right_len) = this.operand_to_simd(right)?;
 621                 let (dest, dest_len) = this.place_to_simd(dest)?;
 622
 623                 // `index` is an array, not a SIMD type
 624                 let ty::Array(_, index_len) = index.layout.ty.kind() else {
 625                     bug!("simd_shuffle index argument has non-array type {}", index.layout.ty)
 626                 };
 627                 let index_len = index_len.eval_usize(*this.tcx, this.param_env());
 628
 629                 assert_eq!(left_len, right_len);
 630                 assert_eq!(index_len, dest_len);
 631
 632                 for i in 0..dest_len {
 633                     let src_index: u64 = this
 634                         .read_immediate(&this.operand_index(&index, i)?.into())?
 635                         .to_scalar()?
 636                         .to_u32()?
 637                         .into();
 638                     let dest = this.mplace_index(&dest, i)?;
 639
 640                     let val = if src_index < left_len {
 641                         this.read_immediate(&this.mplace_index(&left, src_index)?.into())?
 642                     } else if src_index < left_len.checked_add(right_len).unwrap() {
 643                         this.read_immediate(
 644                             &this.mplace_index(&right, src_index - left_len)?.into(),
 645                         )?
 646                     } else {
 647                         bug!(
 648                             "simd_shuffle index {} is out of bounds for 2 vectors of size {}",
 649                             src_index,
 650                             left_len
 651                         );
 652                     };
 653                     this.write_immediate(*val, &dest.into())?;
 654                 }
 655             }
 656
 657             // Atomic operations
 658             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
 659             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
 660             "atomic_load_acq" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
 661
 662             "atomic_store" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
 663             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
 664             "atomic_store_rel" => this.atomic_store(args, AtomicWriteOp::Release)?,
 665
 666             "atomic_fence_acq" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
 667             "atomic_fence_rel" => this.atomic_fence(args, AtomicFenceOp::Release)?,
 668             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
 669             "atomic_fence" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
 670
 671             "atomic_singlethreadfence_acq" => this.compiler_fence(args, AtomicFenceOp::Acquire)?,
 672             "atomic_singlethreadfence_rel" => this.compiler_fence(args, AtomicFenceOp::Release)?,
 673             "atomic_singlethreadfence_acqrel" =>
 674                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
 675             "atomic_singlethreadfence" => this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
 676
 677             "atomic_xchg" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
 678             "atomic_xchg_acq" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
 679             "atomic_xchg_rel" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
 680             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
 681             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
 682
 683             #[rustfmt::skip]
 684             "atomic_cxchg" =>
 685                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 686             #[rustfmt::skip]
 687             "atomic_cxchg_acq" =>
 688                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 689             #[rustfmt::skip]
 690             "atomic_cxchg_rel" =>
 691                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 692             #[rustfmt::skip]
 693             "atomic_cxchg_acqrel" =>
 694                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 695             #[rustfmt::skip]
 696             "atomic_cxchg_relaxed" =>
 697                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 698             #[rustfmt::skip]
 699             "atomic_cxchg_acq_failrelaxed" =>
 700                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 701             #[rustfmt::skip]
 702             "atomic_cxchg_acqrel_failrelaxed" =>
 703                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 704             #[rustfmt::skip]
 705             "atomic_cxchg_failrelaxed" =>
 706                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 707             #[rustfmt::skip]
 708             "atomic_cxchg_failacq" =>
 709                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 710
 711             #[rustfmt::skip]
 712             "atomic_cxchgweak" =>
 713                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 714             #[rustfmt::skip]
 715             "atomic_cxchgweak_acq" =>
 716                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 717             #[rustfmt::skip]
 718             "atomic_cxchgweak_rel" =>
 719                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 720             #[rustfmt::skip]
 721             "atomic_cxchgweak_acqrel" =>
 722                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 723             #[rustfmt::skip]
 724             "atomic_cxchgweak_relaxed" =>
 725                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 726             #[rustfmt::skip]
 727             "atomic_cxchgweak_acq_failrelaxed" =>
 728                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 729             #[rustfmt::skip]
 730             "atomic_cxchgweak_acqrel_failrelaxed" =>
 731                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 732             #[rustfmt::skip]
 733             "atomic_cxchgweak_failrelaxed" =>
 734                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 735             #[rustfmt::skip]
 736             "atomic_cxchgweak_failacq" =>
 737                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 738
 739             #[rustfmt::skip]
 740             "atomic_or" =>
 741                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::SeqCst)?,
 742             #[rustfmt::skip]
 743             "atomic_or_acq" =>
 744                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Acquire)?,
 745             #[rustfmt::skip]
 746             "atomic_or_rel" =>
 747                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Release)?,
 748             #[rustfmt::skip]
 749             "atomic_or_acqrel" =>
 750                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::AcqRel)?,
 751             #[rustfmt::skip]
 752             "atomic_or_relaxed" =>
 753                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Relaxed)?,
 754             #[rustfmt::skip]
 755             "atomic_xor" =>
 756                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::SeqCst)?,
 757             #[rustfmt::skip]
 758             "atomic_xor_acq" =>
 759                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Acquire)?,
 760             #[rustfmt::skip]
 761             "atomic_xor_rel" =>
 762                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Release)?,
 763             #[rustfmt::skip]
 764             "atomic_xor_acqrel" =>
 765                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::AcqRel)?,
 766             #[rustfmt::skip]
 767             "atomic_xor_relaxed" =>
 768                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Relaxed)?,
 769             #[rustfmt::skip]
 770             "atomic_and" =>
 771                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::SeqCst)?,
 772             #[rustfmt::skip]
 773             "atomic_and_acq" =>
 774                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Acquire)?,
 775             #[rustfmt::skip]
 776             "atomic_and_rel" =>
 777                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Release)?,
 778             #[rustfmt::skip]
 779             "atomic_and_acqrel" =>
 780                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::AcqRel)?,
 781             #[rustfmt::skip]
 782             "atomic_and_relaxed" =>
 783                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Relaxed)?,
 784             #[rustfmt::skip]
 785             "atomic_nand" =>
 786                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::SeqCst)?,
 787             #[rustfmt::skip]
 788             "atomic_nand_acq" =>
 789                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Acquire)?,
 790             #[rustfmt::skip]
 791             "atomic_nand_rel" =>
 792                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Release)?,
 793             #[rustfmt::skip]
 794             "atomic_nand_acqrel" =>
 795                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::AcqRel)?,
 796             #[rustfmt::skip]
 797             "atomic_nand_relaxed" =>
 798                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Relaxed)?,
 799             #[rustfmt::skip]
 800             "atomic_xadd" =>
 801                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::SeqCst)?,
 802             #[rustfmt::skip]
 803             "atomic_xadd_acq" =>
 804                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Acquire)?,
 805             #[rustfmt::skip]
 806             "atomic_xadd_rel" =>
 807                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Release)?,
 808             #[rustfmt::skip]
 809             "atomic_xadd_acqrel" =>
 810                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::AcqRel)?,
 811             #[rustfmt::skip]
 812             "atomic_xadd_relaxed" =>
 813                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Relaxed)?,
 814             #[rustfmt::skip]
 815             "atomic_xsub" =>
 816                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::SeqCst)?,
 817             #[rustfmt::skip]
 818             "atomic_xsub_acq" =>
 819                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Acquire)?,
 820             #[rustfmt::skip]
 821             "atomic_xsub_rel" =>
 822                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Release)?,
 823             #[rustfmt::skip]
 824             "atomic_xsub_acqrel" =>
 825                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::AcqRel)?,
 826             #[rustfmt::skip]
 827             "atomic_xsub_relaxed" =>
 828                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Relaxed)?,
 829             "atomic_min" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 830             "atomic_min_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 831             "atomic_min_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 832             "atomic_min_acqrel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 833             "atomic_min_relaxed" =>
 834                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 835             "atomic_max" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 836             "atomic_max_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 837             "atomic_max_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 838             "atomic_max_acqrel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 839             "atomic_max_relaxed" =>
 840                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 841             "atomic_umin" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 842             "atomic_umin_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 843             "atomic_umin_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 844             "atomic_umin_acqrel" =>
 845                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 846             "atomic_umin_relaxed" =>
 847                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 848             "atomic_umax" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 849             "atomic_umax_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 850             "atomic_umax_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 851             "atomic_umax_acqrel" =>
 852                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 853             "atomic_umax_relaxed" =>
 854                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 855
 856             // Other
 857             "exact_div" => {
 858                 let &[ref num, ref denom] = check_arg_count(args)?;
 859                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
 860             }
 861
 862             "try" => return this.handle_try(args, dest, ret),
 863
 864             "breakpoint" => {
 865                 let &[] = check_arg_count(args)?;
 866                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
 867                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
 868             }
 869
 870             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
 871         }
 872
 873         trace!("{:?}", this.dump_place(**dest));
 874         this.go_to_block(ret);
 875         Ok(())
 876     }
 877
 878     fn atomic_load(
 879         &mut self,
 880         args: &[OpTy<'tcx, Tag>],
 881         dest: &PlaceTy<'tcx, Tag>,
 882         atomic: AtomicReadOp,
 883     ) -> InterpResult<'tcx> {
 884         let this = self.eval_context_mut();
 885
 886         let &[ref place] = check_arg_count(args)?;
 887         let place = this.deref_operand(place)?;
 888
 889         // make sure it fits into a scalar; otherwise it cannot be atomic
 890         let val = this.read_scalar_atomic(&place, atomic)?;
 891
 892         // Check alignment requirements. Atomics must always be aligned to their size,
 893         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 894         // be 8-aligned).
 895         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 896         this.memory.check_ptr_access_align(
 897             place.ptr,
 898             place.layout.size,
 899             align,
 900             CheckInAllocMsg::MemoryAccessTest,
 901         )?;
 902         // Perform regular access.
 903         this.write_scalar(val, dest)?;
 904         Ok(())
 905     }
 906
 907     fn atomic_store(
 908         &mut self,
 909         args: &[OpTy<'tcx, Tag>],
 910         atomic: AtomicWriteOp,
 911     ) -> InterpResult<'tcx> {
 912         let this = self.eval_context_mut();
 913
 914         let &[ref place, ref val] = check_arg_count(args)?;
 915         let place = this.deref_operand(place)?;
 916         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
 917
 918         // Check alignment requirements. Atomics must always be aligned to their size,
 919         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 920         // be 8-aligned).
 921         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 922         this.memory.check_ptr_access_align(
 923             place.ptr,
 924             place.layout.size,
 925             align,
 926             CheckInAllocMsg::MemoryAccessTest,
 927         )?;
 928
 929         // Perform atomic store
 930         this.write_scalar_atomic(val, &place, atomic)?;
 931         Ok(())
 932     }
 933
 934     fn compiler_fence(
 935         &mut self,
 936         args: &[OpTy<'tcx, Tag>],
 937         atomic: AtomicFenceOp,
 938     ) -> InterpResult<'tcx> {
 939         let &[] = check_arg_count(args)?;
 940         let _ = atomic;
 941         //FIXME: compiler fences are currently ignored
 942         Ok(())
 943     }
 944
 945     fn atomic_fence(
 946         &mut self,
 947         args: &[OpTy<'tcx, Tag>],
 948         atomic: AtomicFenceOp,
 949     ) -> InterpResult<'tcx> {
 950         let this = self.eval_context_mut();
 951         let &[] = check_arg_count(args)?;
 952         this.validate_atomic_fence(atomic)?;
 953         Ok(())
 954     }
 955
 956     fn atomic_op(
 957         &mut self,
 958         args: &[OpTy<'tcx, Tag>],
 959         dest: &PlaceTy<'tcx, Tag>,
 960         atomic_op: AtomicOp,
 961         atomic: AtomicRwOp,
 962     ) -> InterpResult<'tcx> {
 963         let this = self.eval_context_mut();
 964
 965         let &[ref place, ref rhs] = check_arg_count(args)?;
 966         let place = this.deref_operand(place)?;
 967
 968         if !place.layout.ty.is_integral() {
 969             bug!("Atomic arithmetic operations only work on integer types");
 970         }
 971         let rhs = this.read_immediate(rhs)?;
 972
 973         // Check alignment requirements. Atomics must always be aligned to their size,
 974         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 975         // be 8-aligned).
 976         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 977         this.memory.check_ptr_access_align(
 978             place.ptr,
 979             place.layout.size,
 980             align,
 981             CheckInAllocMsg::MemoryAccessTest,
 982         )?;
 983
 984         match atomic_op {
 985             AtomicOp::Min => {
 986                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
 987                 this.write_immediate(*old, &dest)?; // old value is returned
 988                 Ok(())
 989             }
 990             AtomicOp::Max => {
 991                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
 992                 this.write_immediate(*old, &dest)?; // old value is returned
 993                 Ok(())
 994             }
 995             AtomicOp::MirOp(op, neg) => {
 996                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
 997                 this.write_immediate(*old, dest)?; // old value is returned
 998                 Ok(())
 999             }
1000         }
1001     }
1002
1003     fn atomic_exchange(
1004         &mut self,
1005         args: &[OpTy<'tcx, Tag>],
1006         dest: &PlaceTy<'tcx, Tag>,
1007         atomic: AtomicRwOp,
1008     ) -> InterpResult<'tcx> {
1009         let this = self.eval_context_mut();
1010
1011         let &[ref place, ref new] = check_arg_count(args)?;
1012         let place = this.deref_operand(place)?;
1013         let new = this.read_scalar(new)?;
1014
1015         // Check alignment requirements. Atomics must always be aligned to their size,
1016         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1017         // be 8-aligned).
1018         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1019         this.memory.check_ptr_access_align(
1020             place.ptr,
1021             place.layout.size,
1022             align,
1023             CheckInAllocMsg::MemoryAccessTest,
1024         )?;
1025
1026         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
1027         this.write_scalar(old, dest)?; // old value is returned
1028         Ok(())
1029     }
1030
1031     fn atomic_compare_exchange_impl(
1032         &mut self,
1033         args: &[OpTy<'tcx, Tag>],
1034         dest: &PlaceTy<'tcx, Tag>,
1035         success: AtomicRwOp,
1036         fail: AtomicReadOp,
1037         can_fail_spuriously: bool,
1038     ) -> InterpResult<'tcx> {
1039         let this = self.eval_context_mut();
1040
1041         let &[ref place, ref expect_old, ref new] = check_arg_count(args)?;
1042         let place = this.deref_operand(place)?;
1043         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
1044         let new = this.read_scalar(new)?;
1045
1046         // Check alignment requirements. Atomics must always be aligned to their size,
1047         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
1048         // be 8-aligned).
1049         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
1050         this.memory.check_ptr_access_align(
1051             place.ptr,
1052             place.layout.size,
1053             align,
1054             CheckInAllocMsg::MemoryAccessTest,
1055         )?;
1056
1057         let old = this.atomic_compare_exchange_scalar(
1058             &place,
1059             &expect_old,
1060             new,
1061             success,
1062             fail,
1063             can_fail_spuriously,
1064         )?;
1065
1066         // Return old value.
1067         this.write_immediate(old, dest)?;
1068         Ok(())
1069     }
1070
1071     fn atomic_compare_exchange(
1072         &mut self,
1073         args: &[OpTy<'tcx, Tag>],
1074         dest: &PlaceTy<'tcx, Tag>,
1075         success: AtomicRwOp,
1076         fail: AtomicReadOp,
1077     ) -> InterpResult<'tcx> {
1078         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
1079     }
1080
1081     fn atomic_compare_exchange_weak(
1082         &mut self,
1083         args: &[OpTy<'tcx, Tag>],
1084         dest: &PlaceTy<'tcx, Tag>,
1085         success: AtomicRwOp,
1086         fail: AtomicReadOp,
1087     ) -> InterpResult<'tcx> {
1088         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
1089     }
1090
1091     fn float_to_int_unchecked<F>(
1092         &self,
1093         f: F,
1094         dest_ty: ty::Ty<'tcx>,
1095     ) -> InterpResult<'tcx, Scalar<Tag>>
1096     where
1097         F: Float + Into<Scalar<Tag>>,
1098     {
1099         let this = self.eval_context_ref();
1100
1101         // Step 1: cut off the fractional part of `f`. The result of this is
1102         // guaranteed to be precisely representable in IEEE floats.
1103         let f = f.round_to_integral(Round::TowardZero).value;
1104
1105         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
1106         Ok(match dest_ty.kind() {
1107             // Unsigned
1108             ty::Uint(t) => {
1109                 let size = Integer::from_uint_ty(this, *t).size();
1110                 let res = f.to_u128(size.bits_usize());
1111                 if res.status.is_empty() {
1112                     // No status flags means there was no further rounding or other loss of precision.
1113                     Scalar::from_uint(res.value, size)
1114                 } else {
1115                     // `f` was not representable in this integer type.
1116                     throw_ub_format!(
1117                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
1118                         f,
1119                         dest_ty,
1120                     );
1121                 }
1122             }
1123             // Signed
1124             ty::Int(t) => {
1125                 let size = Integer::from_int_ty(this, *t).size();
1126                 let res = f.to_i128(size.bits_usize());
1127                 if res.status.is_empty() {
1128                     // No status flags means there was no further rounding or other loss of precision.
1129                     Scalar::from_int(res.value, size)
1130                 } else {
1131                     // `f` was not representable in this integer type.
1132                     throw_ub_format!(
1133                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
1134                         f,
1135                         dest_ty,
1136                     );
1137                 }
1138             }
1139             // Nothing else
1140             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
1141         })
1142     }
1143 }
1144
1145 fn fmax_op<'tcx>(
1146     left: &ImmTy<'tcx, Tag>,
1147     right: &ImmTy<'tcx, Tag>,
1148 ) -> InterpResult<'tcx, Scalar<Tag>> {
1149     assert_eq!(left.layout.ty, right.layout.ty);
1150     let ty::Float(float_ty) = left.layout.ty.kind() else {
1151         bug!("fmax operand is not a float")
1152     };
1153     let left = left.to_scalar()?;
1154     let right = right.to_scalar()?;
1155     Ok(match float_ty {
1156         FloatTy::F32 => Scalar::from_f32(left.to_f32()?.max(right.to_f32()?)),
1157         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.max(right.to_f64()?)),
1158     })
1159 }
1160
1161 fn fmin_op<'tcx>(
1162     left: &ImmTy<'tcx, Tag>,
1163     right: &ImmTy<'tcx, Tag>,
1164 ) -> InterpResult<'tcx, Scalar<Tag>> {
1165     assert_eq!(left.layout.ty, right.layout.ty);
1166     let ty::Float(float_ty) = left.layout.ty.kind() else {
1167         bug!("fmin operand is not a float")
1168     };
1169     let left = left.to_scalar()?;
1170     let right = right.to_scalar()?;
1171     Ok(match float_ty {
1172         FloatTy::F32 => Scalar::from_f32(left.to_f32()?.min(right.to_f32()?)),
1173         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)),
1174     })
1175 }