src/shims/intrinsics.rs

   1 use std::iter;
   2
   3 use log::trace;
   4
   5 use rustc_apfloat::{Float, Round};
   6 use rustc_middle::ty::layout::{IntegerExt, LayoutOf};
   7 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
   8 use rustc_target::abi::{Align, Integer};
   9
  10 use crate::*;
  11 use helpers::check_arg_count;
  12
  13 pub enum AtomicOp {
  14     MirOp(mir::BinOp, bool),
  15     Max,
  16     Min,
  17 }
  18
  19 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  20 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  21     fn call_intrinsic(
  22         &mut self,
  23         instance: ty::Instance<'tcx>,
  24         args: &[OpTy<'tcx, Tag>],
  25         ret: Option<(&PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  26         _unwind: StackPopUnwind,
  27     ) -> InterpResult<'tcx> {
  28         let this = self.eval_context_mut();
  29
  30         if this.emulate_intrinsic(instance, args, ret)? {
  31             return Ok(());
  32         }
  33
  34         // All supported intrinsics have a return place.
  35         let intrinsic_name = this.tcx.item_name(instance.def_id());
  36         let intrinsic_name = intrinsic_name.as_str();
  37         let (dest, ret) = match ret {
  38             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  39             Some(p) => p,
  40         };
  41
  42         // Then handle terminating intrinsics.
  43         match intrinsic_name {
  44             // Miri overwriting CTFE intrinsics.
  45             "ptr_guaranteed_eq" => {
  46                 let &[ref left, ref right] = check_arg_count(args)?;
  47                 let left = this.read_immediate(left)?;
  48                 let right = this.read_immediate(right)?;
  49                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  50             }
  51             "ptr_guaranteed_ne" => {
  52                 let &[ref left, ref right] = check_arg_count(args)?;
  53                 let left = this.read_immediate(left)?;
  54                 let right = this.read_immediate(right)?;
  55                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  56             }
  57             "const_allocate" => {
  58                 // For now, for compatibility with the run-time implementation of this, we just return null.
  59                 // See <https://github.com/rust-lang/rust/issues/93935>.
  60                 this.write_null(dest)?;
  61             }
  62             "const_deallocate" => {
  63                 // complete NOP
  64             }
  65
  66             // Raw memory accesses
  67             "volatile_load" => {
  68                 let &[ref place] = check_arg_count(args)?;
  69                 let place = this.deref_operand(place)?;
  70                 this.copy_op(&place.into(), dest)?;
  71             }
  72             "volatile_store" => {
  73                 let &[ref place, ref dest] = check_arg_count(args)?;
  74                 let place = this.deref_operand(place)?;
  75                 this.copy_op(dest, &place.into())?;
  76             }
  77
  78             "write_bytes" | "volatile_set_memory" => {
  79                 let &[ref ptr, ref val_byte, ref count] = check_arg_count(args)?;
  80                 let ty = instance.substs.type_at(0);
  81                 let ty_layout = this.layout_of(ty)?;
  82                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
  83                 let ptr = this.read_pointer(ptr)?;
  84                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  85                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
  86                     err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
  87                 })?;
  88                 this.memory
  89                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
  90             }
  91
  92             // Floating-point operations
  93             #[rustfmt::skip]
  94             | "sinf32"
  95             | "fabsf32"
  96             | "cosf32"
  97             | "sqrtf32"
  98             | "expf32"
  99             | "exp2f32"
 100             | "logf32"
 101             | "log10f32"
 102             | "log2f32"
 103             | "floorf32"
 104             | "ceilf32"
 105             | "truncf32"
 106             | "roundf32"
 107             => {
 108                 let &[ref f] = check_arg_count(args)?;
 109                 // FIXME: Using host floats.
 110                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 111                 let f = match intrinsic_name {
 112                     "sinf32" => f.sin(),
 113                     "fabsf32" => f.abs(),
 114                     "cosf32" => f.cos(),
 115                     "sqrtf32" => f.sqrt(),
 116                     "expf32" => f.exp(),
 117                     "exp2f32" => f.exp2(),
 118                     "logf32" => f.ln(),
 119                     "log10f32" => f.log10(),
 120                     "log2f32" => f.log2(),
 121                     "floorf32" => f.floor(),
 122                     "ceilf32" => f.ceil(),
 123                     "truncf32" => f.trunc(),
 124                     "roundf32" => f.round(),
 125                     _ => bug!(),
 126                 };
 127                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 128             }
 129
 130             #[rustfmt::skip]
 131             | "sinf64"
 132             | "fabsf64"
 133             | "cosf64"
 134             | "sqrtf64"
 135             | "expf64"
 136             | "exp2f64"
 137             | "logf64"
 138             | "log10f64"
 139             | "log2f64"
 140             | "floorf64"
 141             | "ceilf64"
 142             | "truncf64"
 143             | "roundf64"
 144             => {
 145                 let &[ref f] = check_arg_count(args)?;
 146                 // FIXME: Using host floats.
 147                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 148                 let f = match intrinsic_name {
 149                     "sinf64" => f.sin(),
 150                     "fabsf64" => f.abs(),
 151                     "cosf64" => f.cos(),
 152                     "sqrtf64" => f.sqrt(),
 153                     "expf64" => f.exp(),
 154                     "exp2f64" => f.exp2(),
 155                     "logf64" => f.ln(),
 156                     "log10f64" => f.log10(),
 157                     "log2f64" => f.log2(),
 158                     "floorf64" => f.floor(),
 159                     "ceilf64" => f.ceil(),
 160                     "truncf64" => f.trunc(),
 161                     "roundf64" => f.round(),
 162                     _ => bug!(),
 163                 };
 164                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 165             }
 166
 167             #[rustfmt::skip]
 168             | "fadd_fast"
 169             | "fsub_fast"
 170             | "fmul_fast"
 171             | "fdiv_fast"
 172             | "frem_fast"
 173             => {
 174                 let &[ref a, ref b] = check_arg_count(args)?;
 175                 let a = this.read_immediate(a)?;
 176                 let b = this.read_immediate(b)?;
 177                 let op = match intrinsic_name {
 178                     "fadd_fast" => mir::BinOp::Add,
 179                     "fsub_fast" => mir::BinOp::Sub,
 180                     "fmul_fast" => mir::BinOp::Mul,
 181                     "fdiv_fast" => mir::BinOp::Div,
 182                     "frem_fast" => mir::BinOp::Rem,
 183                     _ => bug!(),
 184                 };
 185                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 186                     Ok(match x.layout.ty.kind() {
 187                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 188                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 189                         _ => bug!(
 190                             "`{}` called with non-float input type {:?}",
 191                             intrinsic_name,
 192                             x.layout.ty
 193                         ),
 194                     })
 195                 };
 196                 match (float_finite(a)?, float_finite(b)?) {
 197                     (false, false) => throw_ub_format!(
 198                         "`{}` intrinsic called with non-finite value as both parameters",
 199                         intrinsic_name,
 200                     ),
 201                     (false, _) => throw_ub_format!(
 202                         "`{}` intrinsic called with non-finite value as first parameter",
 203                         intrinsic_name,
 204                     ),
 205                     (_, false) => throw_ub_format!(
 206                         "`{}` intrinsic called with non-finite value as second parameter",
 207                         intrinsic_name,
 208                     ),
 209                     _ => {}
 210                 }
 211                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 212             }
 213
 214             #[rustfmt::skip]
 215             | "minnumf32"
 216             | "maxnumf32"
 217             | "copysignf32"
 218             => {
 219                 let &[ref a, ref b] = check_arg_count(args)?;
 220                 let a = this.read_scalar(a)?.to_f32()?;
 221                 let b = this.read_scalar(b)?.to_f32()?;
 222                 let res = match intrinsic_name {
 223                     "minnumf32" => a.min(b),
 224                     "maxnumf32" => a.max(b),
 225                     "copysignf32" => a.copy_sign(b),
 226                     _ => bug!(),
 227                 };
 228                 this.write_scalar(Scalar::from_f32(res), dest)?;
 229             }
 230
 231             #[rustfmt::skip]
 232             | "minnumf64"
 233             | "maxnumf64"
 234             | "copysignf64"
 235             => {
 236                 let &[ref a, ref b] = check_arg_count(args)?;
 237                 let a = this.read_scalar(a)?.to_f64()?;
 238                 let b = this.read_scalar(b)?.to_f64()?;
 239                 let res = match intrinsic_name {
 240                     "minnumf64" => a.min(b),
 241                     "maxnumf64" => a.max(b),
 242                     "copysignf64" => a.copy_sign(b),
 243                     _ => bug!(),
 244                 };
 245                 this.write_scalar(Scalar::from_f64(res), dest)?;
 246             }
 247
 248             "powf32" => {
 249                 let &[ref f, ref f2] = check_arg_count(args)?;
 250                 // FIXME: Using host floats.
 251                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 252                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 253                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 254             }
 255
 256             "powf64" => {
 257                 let &[ref f, ref f2] = check_arg_count(args)?;
 258                 // FIXME: Using host floats.
 259                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 260                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 261                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 262             }
 263
 264             "fmaf32" => {
 265                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 266                 let a = this.read_scalar(a)?.to_f32()?;
 267                 let b = this.read_scalar(b)?.to_f32()?;
 268                 let c = this.read_scalar(c)?.to_f32()?;
 269                 let res = a.mul_add(b, c).value;
 270                 this.write_scalar(Scalar::from_f32(res), dest)?;
 271             }
 272
 273             "fmaf64" => {
 274                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 275                 let a = this.read_scalar(a)?.to_f64()?;
 276                 let b = this.read_scalar(b)?.to_f64()?;
 277                 let c = this.read_scalar(c)?.to_f64()?;
 278                 let res = a.mul_add(b, c).value;
 279                 this.write_scalar(Scalar::from_f64(res), dest)?;
 280             }
 281
 282             "powif32" => {
 283                 let &[ref f, ref i] = check_arg_count(args)?;
 284                 // FIXME: Using host floats.
 285                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 286                 let i = this.read_scalar(i)?.to_i32()?;
 287                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 288             }
 289
 290             "powif64" => {
 291                 let &[ref f, ref i] = check_arg_count(args)?;
 292                 // FIXME: Using host floats.
 293                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 294                 let i = this.read_scalar(i)?.to_i32()?;
 295                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 296             }
 297
 298             "float_to_int_unchecked" => {
 299                 let &[ref val] = check_arg_count(args)?;
 300                 let val = this.read_immediate(val)?;
 301
 302                 let res = match val.layout.ty.kind() {
 303                     ty::Float(FloatTy::F32) =>
 304                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 305                     ty::Float(FloatTy::F64) =>
 306                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 307                     _ =>
 308                         bug!(
 309                             "`float_to_int_unchecked` called with non-float input type {:?}",
 310                             val.layout.ty
 311                         ),
 312                 };
 313
 314                 this.write_scalar(res, dest)?;
 315             }
 316
 317             // SIMD operations
 318             #[rustfmt::skip]
 319             | "simd_add"
 320             | "simd_sub"
 321             | "simd_mul"
 322             | "simd_div"
 323             | "simd_rem"
 324             | "simd_shl"
 325             | "simd_shr"
 326             | "simd_and"
 327             | "simd_or"
 328             | "simd_eq" => {
 329                 let &[ref left, ref right] = check_arg_count(args)?;
 330                 let (left, left_len) = this.operand_to_simd(left)?;
 331                 let (right, right_len) = this.operand_to_simd(right)?;
 332                 let (dest, dest_len) = this.place_to_simd(dest)?;
 333
 334                 assert_eq!(dest_len, left_len);
 335                 assert_eq!(dest_len, right_len);
 336
 337                 let op = match intrinsic_name {
 338                     "simd_add" => mir::BinOp::Add,
 339                     "simd_sub" => mir::BinOp::Sub,
 340                     "simd_mul" => mir::BinOp::Mul,
 341                     "simd_div" => mir::BinOp::Div,
 342                     "simd_rem" => mir::BinOp::Rem,
 343                     "simd_shl" => mir::BinOp::Shl,
 344                     "simd_shr" => mir::BinOp::Shr,
 345                     "simd_and" => mir::BinOp::BitAnd,
 346                     "simd_or" => mir::BinOp::BitOr,
 347                     "simd_eq" => mir::BinOp::Eq,
 348                     _ => unreachable!(),
 349                 };
 350
 351                 for i in 0..dest_len {
 352                     let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
 353                     let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
 354                     let dest = this.mplace_index(&dest, i)?;
 355                     let (val, overflowed, ty) = this.overflowing_binary_op(op, &left, &right)?;
 356                     if matches!(op, mir::BinOp::Shl | mir::BinOp::Shr) {
 357                         // Shifts have extra UB as SIMD operations that the MIR binop does not have.
 358                         // See <https://github.com/rust-lang/rust/issues/91237>.
 359                         if overflowed {
 360                             let r_val = right.to_scalar()?.to_bits(right.layout.size)?;
 361                             throw_ub_format!("overflowing shift by {} in `{}` in SIMD lane {}", r_val, intrinsic_name, i);
 362                         }
 363                     }
 364                     if matches!(op, mir::BinOp::Eq) {
 365                         // Special handling for boolean-returning operations
 366                         assert_eq!(ty, this.tcx.types.bool);
 367                         let val = val.to_bool().unwrap();
 368                         let val = if val { -1 } else { 0 }; // SIMD uses all-1 as pattern for "true"
 369                         let val = Scalar::from_int(val, dest.layout.size);
 370                         this.write_scalar(val, &dest.into())?;
 371                     } else {
 372                         assert_eq!(ty, dest.layout.ty);
 373                         this.write_scalar(val, &dest.into())?;
 374                     }
 375                 }
 376             }
 377             "simd_reduce_any" => {
 378                 let &[ref arg] = check_arg_count(args)?;
 379                 let (arg, arg_len) = this.operand_to_simd(arg)?;
 380
 381                 let mut res = false; // the neutral element
 382                 for i in 0..arg_len {
 383                     let op = this.read_immediate(&this.mplace_index(&arg, i)?.into())?;
 384                     // We convert it to a *signed* integer and expect either 0 or -1 (the latter means all bits were set).
 385                     let val = op.to_scalar()?.to_int(op.layout.size)?;
 386                     let val = match val {
 387                         0 => false,
 388                         -1 => true,
 389                         _ =>
 390                             throw_ub_format!(
 391                                 "each element of a simd_reduce_any operand must be all-0-bits or all-1-bits"
 392                             ),
 393                     };
 394                     res = res | val;
 395                 }
 396
 397                 this.write_scalar(Scalar::from_bool(res), dest)?;
 398             }
 399
 400             // Atomic operations
 401             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
 402             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
 403             "atomic_load_acq" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
 404
 405             "atomic_store" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
 406             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
 407             "atomic_store_rel" => this.atomic_store(args, AtomicWriteOp::Release)?,
 408
 409             "atomic_fence_acq" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
 410             "atomic_fence_rel" => this.atomic_fence(args, AtomicFenceOp::Release)?,
 411             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
 412             "atomic_fence" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
 413
 414             "atomic_singlethreadfence_acq" => this.compiler_fence(args, AtomicFenceOp::Acquire)?,
 415             "atomic_singlethreadfence_rel" => this.compiler_fence(args, AtomicFenceOp::Release)?,
 416             "atomic_singlethreadfence_acqrel" =>
 417                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
 418             "atomic_singlethreadfence" => this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
 419
 420             "atomic_xchg" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
 421             "atomic_xchg_acq" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
 422             "atomic_xchg_rel" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
 423             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
 424             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
 425
 426             #[rustfmt::skip]
 427             "atomic_cxchg" =>
 428                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 429             #[rustfmt::skip]
 430             "atomic_cxchg_acq" =>
 431                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 432             #[rustfmt::skip]
 433             "atomic_cxchg_rel" =>
 434                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 435             #[rustfmt::skip]
 436             "atomic_cxchg_acqrel" =>
 437                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 438             #[rustfmt::skip]
 439             "atomic_cxchg_relaxed" =>
 440                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 441             #[rustfmt::skip]
 442             "atomic_cxchg_acq_failrelaxed" =>
 443                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 444             #[rustfmt::skip]
 445             "atomic_cxchg_acqrel_failrelaxed" =>
 446                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 447             #[rustfmt::skip]
 448             "atomic_cxchg_failrelaxed" =>
 449                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 450             #[rustfmt::skip]
 451             "atomic_cxchg_failacq" =>
 452                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 453
 454             #[rustfmt::skip]
 455             "atomic_cxchgweak" =>
 456                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 457             #[rustfmt::skip]
 458             "atomic_cxchgweak_acq" =>
 459                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 460             #[rustfmt::skip]
 461             "atomic_cxchgweak_rel" =>
 462                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 463             #[rustfmt::skip]
 464             "atomic_cxchgweak_acqrel" =>
 465                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 466             #[rustfmt::skip]
 467             "atomic_cxchgweak_relaxed" =>
 468                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 469             #[rustfmt::skip]
 470             "atomic_cxchgweak_acq_failrelaxed" =>
 471                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 472             #[rustfmt::skip]
 473             "atomic_cxchgweak_acqrel_failrelaxed" =>
 474                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 475             #[rustfmt::skip]
 476             "atomic_cxchgweak_failrelaxed" =>
 477                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 478             #[rustfmt::skip]
 479             "atomic_cxchgweak_failacq" =>
 480                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 481
 482             #[rustfmt::skip]
 483             "atomic_or" =>
 484                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::SeqCst)?,
 485             #[rustfmt::skip]
 486             "atomic_or_acq" =>
 487                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Acquire)?,
 488             #[rustfmt::skip]
 489             "atomic_or_rel" =>
 490                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Release)?,
 491             #[rustfmt::skip]
 492             "atomic_or_acqrel" =>
 493                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::AcqRel)?,
 494             #[rustfmt::skip]
 495             "atomic_or_relaxed" =>
 496                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Relaxed)?,
 497             #[rustfmt::skip]
 498             "atomic_xor" =>
 499                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::SeqCst)?,
 500             #[rustfmt::skip]
 501             "atomic_xor_acq" =>
 502                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Acquire)?,
 503             #[rustfmt::skip]
 504             "atomic_xor_rel" =>
 505                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Release)?,
 506             #[rustfmt::skip]
 507             "atomic_xor_acqrel" =>
 508                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::AcqRel)?,
 509             #[rustfmt::skip]
 510             "atomic_xor_relaxed" =>
 511                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Relaxed)?,
 512             #[rustfmt::skip]
 513             "atomic_and" =>
 514                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::SeqCst)?,
 515             #[rustfmt::skip]
 516             "atomic_and_acq" =>
 517                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Acquire)?,
 518             #[rustfmt::skip]
 519             "atomic_and_rel" =>
 520                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Release)?,
 521             #[rustfmt::skip]
 522             "atomic_and_acqrel" =>
 523                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::AcqRel)?,
 524             #[rustfmt::skip]
 525             "atomic_and_relaxed" =>
 526                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Relaxed)?,
 527             #[rustfmt::skip]
 528             "atomic_nand" =>
 529                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::SeqCst)?,
 530             #[rustfmt::skip]
 531             "atomic_nand_acq" =>
 532                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Acquire)?,
 533             #[rustfmt::skip]
 534             "atomic_nand_rel" =>
 535                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Release)?,
 536             #[rustfmt::skip]
 537             "atomic_nand_acqrel" =>
 538                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::AcqRel)?,
 539             #[rustfmt::skip]
 540             "atomic_nand_relaxed" =>
 541                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Relaxed)?,
 542             #[rustfmt::skip]
 543             "atomic_xadd" =>
 544                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::SeqCst)?,
 545             #[rustfmt::skip]
 546             "atomic_xadd_acq" =>
 547                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Acquire)?,
 548             #[rustfmt::skip]
 549             "atomic_xadd_rel" =>
 550                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Release)?,
 551             #[rustfmt::skip]
 552             "atomic_xadd_acqrel" =>
 553                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::AcqRel)?,
 554             #[rustfmt::skip]
 555             "atomic_xadd_relaxed" =>
 556                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Relaxed)?,
 557             #[rustfmt::skip]
 558             "atomic_xsub" =>
 559                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::SeqCst)?,
 560             #[rustfmt::skip]
 561             "atomic_xsub_acq" =>
 562                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Acquire)?,
 563             #[rustfmt::skip]
 564             "atomic_xsub_rel" =>
 565                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Release)?,
 566             #[rustfmt::skip]
 567             "atomic_xsub_acqrel" =>
 568                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::AcqRel)?,
 569             #[rustfmt::skip]
 570             "atomic_xsub_relaxed" =>
 571                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Relaxed)?,
 572             "atomic_min" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 573             "atomic_min_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 574             "atomic_min_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 575             "atomic_min_acqrel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 576             "atomic_min_relaxed" =>
 577                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 578             "atomic_max" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 579             "atomic_max_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 580             "atomic_max_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 581             "atomic_max_acqrel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 582             "atomic_max_relaxed" =>
 583                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 584             "atomic_umin" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 585             "atomic_umin_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 586             "atomic_umin_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 587             "atomic_umin_acqrel" =>
 588                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 589             "atomic_umin_relaxed" =>
 590                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 591             "atomic_umax" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 592             "atomic_umax_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 593             "atomic_umax_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 594             "atomic_umax_acqrel" =>
 595                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 596             "atomic_umax_relaxed" =>
 597                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 598
 599             // Other
 600             "exact_div" => {
 601                 let &[ref num, ref denom] = check_arg_count(args)?;
 602                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
 603             }
 604
 605             "try" => return this.handle_try(args, dest, ret),
 606
 607             "breakpoint" => {
 608                 let &[] = check_arg_count(args)?;
 609                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
 610                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
 611             }
 612
 613             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
 614         }
 615
 616         trace!("{:?}", this.dump_place(**dest));
 617         this.go_to_block(ret);
 618         Ok(())
 619     }
 620
 621     fn atomic_load(
 622         &mut self,
 623         args: &[OpTy<'tcx, Tag>],
 624         dest: &PlaceTy<'tcx, Tag>,
 625         atomic: AtomicReadOp,
 626     ) -> InterpResult<'tcx> {
 627         let this = self.eval_context_mut();
 628
 629         let &[ref place] = check_arg_count(args)?;
 630         let place = this.deref_operand(place)?;
 631
 632         // make sure it fits into a scalar; otherwise it cannot be atomic
 633         let val = this.read_scalar_atomic(&place, atomic)?;
 634
 635         // Check alignment requirements. Atomics must always be aligned to their size,
 636         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 637         // be 8-aligned).
 638         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 639         this.memory.check_ptr_access_align(
 640             place.ptr,
 641             place.layout.size,
 642             align,
 643             CheckInAllocMsg::MemoryAccessTest,
 644         )?;
 645         // Perform regular access.
 646         this.write_scalar(val, dest)?;
 647         Ok(())
 648     }
 649
 650     fn atomic_store(
 651         &mut self,
 652         args: &[OpTy<'tcx, Tag>],
 653         atomic: AtomicWriteOp,
 654     ) -> InterpResult<'tcx> {
 655         let this = self.eval_context_mut();
 656
 657         let &[ref place, ref val] = check_arg_count(args)?;
 658         let place = this.deref_operand(place)?;
 659         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
 660
 661         // Check alignment requirements. Atomics must always be aligned to their size,
 662         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 663         // be 8-aligned).
 664         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 665         this.memory.check_ptr_access_align(
 666             place.ptr,
 667             place.layout.size,
 668             align,
 669             CheckInAllocMsg::MemoryAccessTest,
 670         )?;
 671
 672         // Perform atomic store
 673         this.write_scalar_atomic(val, &place, atomic)?;
 674         Ok(())
 675     }
 676
 677     fn compiler_fence(
 678         &mut self,
 679         args: &[OpTy<'tcx, Tag>],
 680         atomic: AtomicFenceOp,
 681     ) -> InterpResult<'tcx> {
 682         let &[] = check_arg_count(args)?;
 683         let _ = atomic;
 684         //FIXME: compiler fences are currently ignored
 685         Ok(())
 686     }
 687
 688     fn atomic_fence(
 689         &mut self,
 690         args: &[OpTy<'tcx, Tag>],
 691         atomic: AtomicFenceOp,
 692     ) -> InterpResult<'tcx> {
 693         let this = self.eval_context_mut();
 694         let &[] = check_arg_count(args)?;
 695         this.validate_atomic_fence(atomic)?;
 696         Ok(())
 697     }
 698
 699     fn atomic_op(
 700         &mut self,
 701         args: &[OpTy<'tcx, Tag>],
 702         dest: &PlaceTy<'tcx, Tag>,
 703         atomic_op: AtomicOp,
 704         atomic: AtomicRwOp,
 705     ) -> InterpResult<'tcx> {
 706         let this = self.eval_context_mut();
 707
 708         let &[ref place, ref rhs] = check_arg_count(args)?;
 709         let place = this.deref_operand(place)?;
 710
 711         if !place.layout.ty.is_integral() {
 712             bug!("Atomic arithmetic operations only work on integer types");
 713         }
 714         let rhs = this.read_immediate(rhs)?;
 715
 716         // Check alignment requirements. Atomics must always be aligned to their size,
 717         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 718         // be 8-aligned).
 719         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 720         this.memory.check_ptr_access_align(
 721             place.ptr,
 722             place.layout.size,
 723             align,
 724             CheckInAllocMsg::MemoryAccessTest,
 725         )?;
 726
 727         match atomic_op {
 728             AtomicOp::Min => {
 729                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
 730                 this.write_immediate(*old, &dest)?; // old value is returned
 731                 Ok(())
 732             }
 733             AtomicOp::Max => {
 734                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
 735                 this.write_immediate(*old, &dest)?; // old value is returned
 736                 Ok(())
 737             }
 738             AtomicOp::MirOp(op, neg) => {
 739                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
 740                 this.write_immediate(*old, dest)?; // old value is returned
 741                 Ok(())
 742             }
 743         }
 744     }
 745
 746     fn atomic_exchange(
 747         &mut self,
 748         args: &[OpTy<'tcx, Tag>],
 749         dest: &PlaceTy<'tcx, Tag>,
 750         atomic: AtomicRwOp,
 751     ) -> InterpResult<'tcx> {
 752         let this = self.eval_context_mut();
 753
 754         let &[ref place, ref new] = check_arg_count(args)?;
 755         let place = this.deref_operand(place)?;
 756         let new = this.read_scalar(new)?;
 757
 758         // Check alignment requirements. Atomics must always be aligned to their size,
 759         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 760         // be 8-aligned).
 761         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 762         this.memory.check_ptr_access_align(
 763             place.ptr,
 764             place.layout.size,
 765             align,
 766             CheckInAllocMsg::MemoryAccessTest,
 767         )?;
 768
 769         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
 770         this.write_scalar(old, dest)?; // old value is returned
 771         Ok(())
 772     }
 773
 774     fn atomic_compare_exchange_impl(
 775         &mut self,
 776         args: &[OpTy<'tcx, Tag>],
 777         dest: &PlaceTy<'tcx, Tag>,
 778         success: AtomicRwOp,
 779         fail: AtomicReadOp,
 780         can_fail_spuriously: bool,
 781     ) -> InterpResult<'tcx> {
 782         let this = self.eval_context_mut();
 783
 784         let &[ref place, ref expect_old, ref new] = check_arg_count(args)?;
 785         let place = this.deref_operand(place)?;
 786         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
 787         let new = this.read_scalar(new)?;
 788
 789         // Check alignment requirements. Atomics must always be aligned to their size,
 790         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 791         // be 8-aligned).
 792         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 793         this.memory.check_ptr_access_align(
 794             place.ptr,
 795             place.layout.size,
 796             align,
 797             CheckInAllocMsg::MemoryAccessTest,
 798         )?;
 799
 800         let old = this.atomic_compare_exchange_scalar(
 801             &place,
 802             &expect_old,
 803             new,
 804             success,
 805             fail,
 806             can_fail_spuriously,
 807         )?;
 808
 809         // Return old value.
 810         this.write_immediate(old, dest)?;
 811         Ok(())
 812     }
 813
 814     fn atomic_compare_exchange(
 815         &mut self,
 816         args: &[OpTy<'tcx, Tag>],
 817         dest: &PlaceTy<'tcx, Tag>,
 818         success: AtomicRwOp,
 819         fail: AtomicReadOp,
 820     ) -> InterpResult<'tcx> {
 821         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
 822     }
 823
 824     fn atomic_compare_exchange_weak(
 825         &mut self,
 826         args: &[OpTy<'tcx, Tag>],
 827         dest: &PlaceTy<'tcx, Tag>,
 828         success: AtomicRwOp,
 829         fail: AtomicReadOp,
 830     ) -> InterpResult<'tcx> {
 831         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
 832     }
 833
 834     fn float_to_int_unchecked<F>(
 835         &self,
 836         f: F,
 837         dest_ty: ty::Ty<'tcx>,
 838     ) -> InterpResult<'tcx, Scalar<Tag>>
 839     where
 840         F: Float + Into<Scalar<Tag>>,
 841     {
 842         let this = self.eval_context_ref();
 843
 844         // Step 1: cut off the fractional part of `f`. The result of this is
 845         // guaranteed to be precisely representable in IEEE floats.
 846         let f = f.round_to_integral(Round::TowardZero).value;
 847
 848         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
 849         Ok(match dest_ty.kind() {
 850             // Unsigned
 851             ty::Uint(t) => {
 852                 let size = Integer::from_uint_ty(this, *t).size();
 853                 let res = f.to_u128(size.bits_usize());
 854                 if res.status.is_empty() {
 855                     // No status flags means there was no further rounding or other loss of precision.
 856                     Scalar::from_uint(res.value, size)
 857                 } else {
 858                     // `f` was not representable in this integer type.
 859                     throw_ub_format!(
 860                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 861                         f,
 862                         dest_ty,
 863                     );
 864                 }
 865             }
 866             // Signed
 867             ty::Int(t) => {
 868                 let size = Integer::from_int_ty(this, *t).size();
 869                 let res = f.to_i128(size.bits_usize());
 870                 if res.status.is_empty() {
 871                     // No status flags means there was no further rounding or other loss of precision.
 872                     Scalar::from_int(res.value, size)
 873                 } else {
 874                     // `f` was not representable in this integer type.
 875                     throw_ub_format!(
 876                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 877                         f,
 878                         dest_ty,
 879                     );
 880                 }
 881             }
 882             // Nothing else
 883             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
 884         })
 885     }
 886 }