src/shims/intrinsics.rs

   1 use std::iter;
   2
   3 use log::trace;
   4
   5 use rustc_apfloat::{Float, Round};
   6 use rustc_middle::ty::layout::{IntegerExt, LayoutOf};
   7 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
   8 use rustc_target::abi::{Align, Integer};
   9
  10 use crate::*;
  11 use helpers::check_arg_count;
  12
  13 pub enum AtomicOp {
  14     MirOp(mir::BinOp, bool),
  15     Max,
  16     Min,
  17 }
  18
  19 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  20 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  21     fn call_intrinsic(
  22         &mut self,
  23         instance: ty::Instance<'tcx>,
  24         args: &[OpTy<'tcx, Tag>],
  25         ret: Option<(&PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  26         _unwind: StackPopUnwind,
  27     ) -> InterpResult<'tcx> {
  28         let this = self.eval_context_mut();
  29
  30         if this.emulate_intrinsic(instance, args, ret)? {
  31             return Ok(());
  32         }
  33
  34         // All supported intrinsics have a return place.
  35         let intrinsic_name = &*this.tcx.item_name(instance.def_id()).as_str();
  36         let (dest, ret) = match ret {
  37             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  38             Some(p) => p,
  39         };
  40
  41         // Then handle terminating intrinsics.
  42         match intrinsic_name {
  43             // Miri overwriting CTFE intrinsics.
  44             "ptr_guaranteed_eq" => {
  45                 let &[ref left, ref right] = check_arg_count(args)?;
  46                 let left = this.read_immediate(left)?;
  47                 let right = this.read_immediate(right)?;
  48                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  49             }
  50             "ptr_guaranteed_ne" => {
  51                 let &[ref left, ref right] = check_arg_count(args)?;
  52                 let left = this.read_immediate(left)?;
  53                 let right = this.read_immediate(right)?;
  54                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  55             }
  56
  57             // Raw memory accesses
  58             "volatile_load" => {
  59                 let &[ref place] = check_arg_count(args)?;
  60                 let place = this.deref_operand(place)?;
  61                 this.copy_op(&place.into(), dest)?;
  62             }
  63             "volatile_store" => {
  64                 let &[ref place, ref dest] = check_arg_count(args)?;
  65                 let place = this.deref_operand(place)?;
  66                 this.copy_op(dest, &place.into())?;
  67             }
  68
  69             "write_bytes" | "volatile_set_memory" => {
  70                 let &[ref ptr, ref val_byte, ref count] = check_arg_count(args)?;
  71                 let ty = instance.substs.type_at(0);
  72                 let ty_layout = this.layout_of(ty)?;
  73                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
  74                 let ptr = this.read_pointer(ptr)?;
  75                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  76                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
  77                     err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
  78                 })?;
  79                 this.memory
  80                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
  81             }
  82
  83             // Floating-point operations
  84             #[rustfmt::skip]
  85             | "sinf32"
  86             | "fabsf32"
  87             | "cosf32"
  88             | "sqrtf32"
  89             | "expf32"
  90             | "exp2f32"
  91             | "logf32"
  92             | "log10f32"
  93             | "log2f32"
  94             | "floorf32"
  95             | "ceilf32"
  96             | "truncf32"
  97             | "roundf32"
  98             => {
  99                 let &[ref f] = check_arg_count(args)?;
 100                 // FIXME: Using host floats.
 101                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 102                 let f = match intrinsic_name {
 103                     "sinf32" => f.sin(),
 104                     "fabsf32" => f.abs(),
 105                     "cosf32" => f.cos(),
 106                     "sqrtf32" => f.sqrt(),
 107                     "expf32" => f.exp(),
 108                     "exp2f32" => f.exp2(),
 109                     "logf32" => f.ln(),
 110                     "log10f32" => f.log10(),
 111                     "log2f32" => f.log2(),
 112                     "floorf32" => f.floor(),
 113                     "ceilf32" => f.ceil(),
 114                     "truncf32" => f.trunc(),
 115                     "roundf32" => f.round(),
 116                     _ => bug!(),
 117                 };
 118                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 119             }
 120
 121             #[rustfmt::skip]
 122             | "sinf64"
 123             | "fabsf64"
 124             | "cosf64"
 125             | "sqrtf64"
 126             | "expf64"
 127             | "exp2f64"
 128             | "logf64"
 129             | "log10f64"
 130             | "log2f64"
 131             | "floorf64"
 132             | "ceilf64"
 133             | "truncf64"
 134             | "roundf64"
 135             => {
 136                 let &[ref f] = check_arg_count(args)?;
 137                 // FIXME: Using host floats.
 138                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 139                 let f = match intrinsic_name {
 140                     "sinf64" => f.sin(),
 141                     "fabsf64" => f.abs(),
 142                     "cosf64" => f.cos(),
 143                     "sqrtf64" => f.sqrt(),
 144                     "expf64" => f.exp(),
 145                     "exp2f64" => f.exp2(),
 146                     "logf64" => f.ln(),
 147                     "log10f64" => f.log10(),
 148                     "log2f64" => f.log2(),
 149                     "floorf64" => f.floor(),
 150                     "ceilf64" => f.ceil(),
 151                     "truncf64" => f.trunc(),
 152                     "roundf64" => f.round(),
 153                     _ => bug!(),
 154                 };
 155                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 156             }
 157
 158             #[rustfmt::skip]
 159             | "fadd_fast"
 160             | "fsub_fast"
 161             | "fmul_fast"
 162             | "fdiv_fast"
 163             | "frem_fast"
 164             => {
 165                 let &[ref a, ref b] = check_arg_count(args)?;
 166                 let a = this.read_immediate(a)?;
 167                 let b = this.read_immediate(b)?;
 168                 let op = match intrinsic_name {
 169                     "fadd_fast" => mir::BinOp::Add,
 170                     "fsub_fast" => mir::BinOp::Sub,
 171                     "fmul_fast" => mir::BinOp::Mul,
 172                     "fdiv_fast" => mir::BinOp::Div,
 173                     "frem_fast" => mir::BinOp::Rem,
 174                     _ => bug!(),
 175                 };
 176                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 177                     Ok(match x.layout.ty.kind() {
 178                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 179                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 180                         _ => bug!(
 181                             "`{}` called with non-float input type {:?}",
 182                             intrinsic_name,
 183                             x.layout.ty
 184                         ),
 185                     })
 186                 };
 187                 match (float_finite(a)?, float_finite(b)?) {
 188                     (false, false) => throw_ub_format!(
 189                         "`{}` intrinsic called with non-finite value as both parameters",
 190                         intrinsic_name,
 191                     ),
 192                     (false, _) => throw_ub_format!(
 193                         "`{}` intrinsic called with non-finite value as first parameter",
 194                         intrinsic_name,
 195                     ),
 196                     (_, false) => throw_ub_format!(
 197                         "`{}` intrinsic called with non-finite value as second parameter",
 198                         intrinsic_name,
 199                     ),
 200                     _ => {}
 201                 }
 202                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 203             }
 204
 205             #[rustfmt::skip]
 206             | "minnumf32"
 207             | "maxnumf32"
 208             | "copysignf32"
 209             => {
 210                 let &[ref a, ref b] = check_arg_count(args)?;
 211                 let a = this.read_scalar(a)?.to_f32()?;
 212                 let b = this.read_scalar(b)?.to_f32()?;
 213                 let res = match intrinsic_name {
 214                     "minnumf32" => a.min(b),
 215                     "maxnumf32" => a.max(b),
 216                     "copysignf32" => a.copy_sign(b),
 217                     _ => bug!(),
 218                 };
 219                 this.write_scalar(Scalar::from_f32(res), dest)?;
 220             }
 221
 222             #[rustfmt::skip]
 223             | "minnumf64"
 224             | "maxnumf64"
 225             | "copysignf64"
 226             => {
 227                 let &[ref a, ref b] = check_arg_count(args)?;
 228                 let a = this.read_scalar(a)?.to_f64()?;
 229                 let b = this.read_scalar(b)?.to_f64()?;
 230                 let res = match intrinsic_name {
 231                     "minnumf64" => a.min(b),
 232                     "maxnumf64" => a.max(b),
 233                     "copysignf64" => a.copy_sign(b),
 234                     _ => bug!(),
 235                 };
 236                 this.write_scalar(Scalar::from_f64(res), dest)?;
 237             }
 238
 239             "powf32" => {
 240                 let &[ref f, ref f2] = check_arg_count(args)?;
 241                 // FIXME: Using host floats.
 242                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 243                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 244                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 245             }
 246
 247             "powf64" => {
 248                 let &[ref f, ref f2] = check_arg_count(args)?;
 249                 // FIXME: Using host floats.
 250                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 251                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 252                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 253             }
 254
 255             "fmaf32" => {
 256                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 257                 let a = this.read_scalar(a)?.to_f32()?;
 258                 let b = this.read_scalar(b)?.to_f32()?;
 259                 let c = this.read_scalar(c)?.to_f32()?;
 260                 let res = a.mul_add(b, c).value;
 261                 this.write_scalar(Scalar::from_f32(res), dest)?;
 262             }
 263
 264             "fmaf64" => {
 265                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 266                 let a = this.read_scalar(a)?.to_f64()?;
 267                 let b = this.read_scalar(b)?.to_f64()?;
 268                 let c = this.read_scalar(c)?.to_f64()?;
 269                 let res = a.mul_add(b, c).value;
 270                 this.write_scalar(Scalar::from_f64(res), dest)?;
 271             }
 272
 273             "powif32" => {
 274                 let &[ref f, ref i] = check_arg_count(args)?;
 275                 // FIXME: Using host floats.
 276                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 277                 let i = this.read_scalar(i)?.to_i32()?;
 278                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 279             }
 280
 281             "powif64" => {
 282                 let &[ref f, ref i] = check_arg_count(args)?;
 283                 // FIXME: Using host floats.
 284                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 285                 let i = this.read_scalar(i)?.to_i32()?;
 286                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 287             }
 288
 289             "float_to_int_unchecked" => {
 290                 let &[ref val] = check_arg_count(args)?;
 291                 let val = this.read_immediate(val)?;
 292
 293                 let res = match val.layout.ty.kind() {
 294                     ty::Float(FloatTy::F32) =>
 295                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 296                     ty::Float(FloatTy::F64) =>
 297                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 298                     _ =>
 299                         bug!(
 300                             "`float_to_int_unchecked` called with non-float input type {:?}",
 301                             val.layout.ty
 302                         ),
 303                 };
 304
 305                 this.write_scalar(res, dest)?;
 306             }
 307
 308             // SIMD operations
 309             #[rustfmt::skip]
 310             | "simd_add"
 311             | "simd_sub"
 312             | "simd_mul"
 313             | "simd_div"
 314             | "simd_rem"
 315             | "simd_shl"
 316             | "simd_shr" => {
 317                 let &[ref left, ref right] = check_arg_count(args)?;
 318                 let (left, left_len) = this.operand_to_simd(left)?;
 319                 let (right, right_len) = this.operand_to_simd(right)?;
 320                 let (dest, dest_len) = this.place_to_simd(dest)?;
 321
 322                 assert_eq!(dest_len, left_len);
 323                 assert_eq!(dest_len, right_len);
 324
 325                 let op = match intrinsic_name {
 326                     "simd_add" => mir::BinOp::Add,
 327                     "simd_sub" => mir::BinOp::Sub,
 328                     "simd_mul" => mir::BinOp::Mul,
 329                     "simd_div" => mir::BinOp::Div,
 330                     "simd_rem" => mir::BinOp::Rem,
 331                     "simd_shl" => mir::BinOp::Shl,
 332                     "simd_shr" => mir::BinOp::Shr,
 333                     _ => unreachable!(),
 334                 };
 335
 336                 for i in 0..dest_len {
 337                     let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
 338                     let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
 339                     let dest = this.mplace_index(&dest, i)?;
 340                     let (val, overflowed, ty) = this.overflowing_binary_op(op, &left, &right)?;
 341                     assert_eq!(ty, dest.layout.ty);
 342                     if matches!(op, mir::BinOp::Shl | mir::BinOp::Shr) {
 343                         // Shifts have extra UB as SIMD operations that the MIR binop does not have.
 344                         // See <https://github.com/rust-lang/rust/issues/91237>.
 345                         if overflowed {
 346                             let r_val = right.to_scalar()?.to_bits(right.layout.size)?;
 347                             throw_ub_format!("overflowing shift by {} in `{}` in SIMD lane {}", r_val, intrinsic_name, i);
 348                         }
 349                     }
 350                     this.write_scalar(val, &dest.into())?;
 351                 }
 352             }
 353
 354             // Atomic operations
 355             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
 356             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
 357             "atomic_load_acq" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
 358
 359             "atomic_store" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
 360             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
 361             "atomic_store_rel" => this.atomic_store(args, AtomicWriteOp::Release)?,
 362
 363             "atomic_fence_acq" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
 364             "atomic_fence_rel" => this.atomic_fence(args, AtomicFenceOp::Release)?,
 365             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
 366             "atomic_fence" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
 367
 368             "atomic_singlethreadfence_acq" => this.compiler_fence(args, AtomicFenceOp::Acquire)?,
 369             "atomic_singlethreadfence_rel" => this.compiler_fence(args, AtomicFenceOp::Release)?,
 370             "atomic_singlethreadfence_acqrel" =>
 371                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
 372             "atomic_singlethreadfence" => this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
 373
 374             "atomic_xchg" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
 375             "atomic_xchg_acq" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
 376             "atomic_xchg_rel" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
 377             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
 378             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
 379
 380             #[rustfmt::skip]
 381             "atomic_cxchg" =>
 382                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 383             #[rustfmt::skip]
 384             "atomic_cxchg_acq" =>
 385                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 386             #[rustfmt::skip]
 387             "atomic_cxchg_rel" =>
 388                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 389             #[rustfmt::skip]
 390             "atomic_cxchg_acqrel" =>
 391                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 392             #[rustfmt::skip]
 393             "atomic_cxchg_relaxed" =>
 394                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 395             #[rustfmt::skip]
 396             "atomic_cxchg_acq_failrelaxed" =>
 397                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 398             #[rustfmt::skip]
 399             "atomic_cxchg_acqrel_failrelaxed" =>
 400                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 401             #[rustfmt::skip]
 402             "atomic_cxchg_failrelaxed" =>
 403                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 404             #[rustfmt::skip]
 405             "atomic_cxchg_failacq" =>
 406                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 407
 408             #[rustfmt::skip]
 409             "atomic_cxchgweak" =>
 410                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 411             #[rustfmt::skip]
 412             "atomic_cxchgweak_acq" =>
 413                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 414             #[rustfmt::skip]
 415             "atomic_cxchgweak_rel" =>
 416                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 417             #[rustfmt::skip]
 418             "atomic_cxchgweak_acqrel" =>
 419                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 420             #[rustfmt::skip]
 421             "atomic_cxchgweak_relaxed" =>
 422                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 423             #[rustfmt::skip]
 424             "atomic_cxchgweak_acq_failrelaxed" =>
 425                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 426             #[rustfmt::skip]
 427             "atomic_cxchgweak_acqrel_failrelaxed" =>
 428                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 429             #[rustfmt::skip]
 430             "atomic_cxchgweak_failrelaxed" =>
 431                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 432             #[rustfmt::skip]
 433             "atomic_cxchgweak_failacq" =>
 434                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 435
 436             #[rustfmt::skip]
 437             "atomic_or" =>
 438                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::SeqCst)?,
 439             #[rustfmt::skip]
 440             "atomic_or_acq" =>
 441                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Acquire)?,
 442             #[rustfmt::skip]
 443             "atomic_or_rel" =>
 444                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Release)?,
 445             #[rustfmt::skip]
 446             "atomic_or_acqrel" =>
 447                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::AcqRel)?,
 448             #[rustfmt::skip]
 449             "atomic_or_relaxed" =>
 450                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Relaxed)?,
 451             #[rustfmt::skip]
 452             "atomic_xor" =>
 453                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::SeqCst)?,
 454             #[rustfmt::skip]
 455             "atomic_xor_acq" =>
 456                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Acquire)?,
 457             #[rustfmt::skip]
 458             "atomic_xor_rel" =>
 459                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Release)?,
 460             #[rustfmt::skip]
 461             "atomic_xor_acqrel" =>
 462                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::AcqRel)?,
 463             #[rustfmt::skip]
 464             "atomic_xor_relaxed" =>
 465                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Relaxed)?,
 466             #[rustfmt::skip]
 467             "atomic_and" =>
 468                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::SeqCst)?,
 469             #[rustfmt::skip]
 470             "atomic_and_acq" =>
 471                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Acquire)?,
 472             #[rustfmt::skip]
 473             "atomic_and_rel" =>
 474                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Release)?,
 475             #[rustfmt::skip]
 476             "atomic_and_acqrel" =>
 477                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::AcqRel)?,
 478             #[rustfmt::skip]
 479             "atomic_and_relaxed" =>
 480                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Relaxed)?,
 481             #[rustfmt::skip]
 482             "atomic_nand" =>
 483                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::SeqCst)?,
 484             #[rustfmt::skip]
 485             "atomic_nand_acq" =>
 486                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Acquire)?,
 487             #[rustfmt::skip]
 488             "atomic_nand_rel" =>
 489                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Release)?,
 490             #[rustfmt::skip]
 491             "atomic_nand_acqrel" =>
 492                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::AcqRel)?,
 493             #[rustfmt::skip]
 494             "atomic_nand_relaxed" =>
 495                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Relaxed)?,
 496             #[rustfmt::skip]
 497             "atomic_xadd" =>
 498                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::SeqCst)?,
 499             #[rustfmt::skip]
 500             "atomic_xadd_acq" =>
 501                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Acquire)?,
 502             #[rustfmt::skip]
 503             "atomic_xadd_rel" =>
 504                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Release)?,
 505             #[rustfmt::skip]
 506             "atomic_xadd_acqrel" =>
 507                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::AcqRel)?,
 508             #[rustfmt::skip]
 509             "atomic_xadd_relaxed" =>
 510                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Relaxed)?,
 511             #[rustfmt::skip]
 512             "atomic_xsub" =>
 513                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::SeqCst)?,
 514             #[rustfmt::skip]
 515             "atomic_xsub_acq" =>
 516                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Acquire)?,
 517             #[rustfmt::skip]
 518             "atomic_xsub_rel" =>
 519                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Release)?,
 520             #[rustfmt::skip]
 521             "atomic_xsub_acqrel" =>
 522                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::AcqRel)?,
 523             #[rustfmt::skip]
 524             "atomic_xsub_relaxed" =>
 525                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Relaxed)?,
 526             "atomic_min" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 527             "atomic_min_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 528             "atomic_min_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 529             "atomic_min_acqrel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 530             "atomic_min_relaxed" =>
 531                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 532             "atomic_max" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 533             "atomic_max_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 534             "atomic_max_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 535             "atomic_max_acqrel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 536             "atomic_max_relaxed" =>
 537                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 538             "atomic_umin" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 539             "atomic_umin_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 540             "atomic_umin_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 541             "atomic_umin_acqrel" =>
 542                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 543             "atomic_umin_relaxed" =>
 544                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 545             "atomic_umax" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 546             "atomic_umax_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 547             "atomic_umax_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 548             "atomic_umax_acqrel" =>
 549                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 550             "atomic_umax_relaxed" =>
 551                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 552
 553             // Query type information
 554             "assert_zero_valid" | "assert_uninit_valid" => {
 555                 let &[] = check_arg_count(args)?;
 556                 let ty = instance.substs.type_at(0);
 557                 let layout = this.layout_of(ty)?;
 558                 // Abort here because the caller might not be panic safe.
 559                 if layout.abi.is_uninhabited() {
 560                     // Use this message even for the other intrinsics, as that's what codegen does
 561                     throw_machine_stop!(TerminationInfo::Abort(format!(
 562                         "aborted execution: attempted to instantiate uninhabited type `{}`",
 563                         ty
 564                     )))
 565                 }
 566                 if intrinsic_name == "assert_zero_valid"
 567                     && !layout.might_permit_raw_init(this, /*zero:*/ true)
 568                 {
 569                     throw_machine_stop!(TerminationInfo::Abort(format!(
 570                         "aborted execution: attempted to zero-initialize type `{}`, which is invalid",
 571                         ty
 572                     )))
 573                 }
 574                 if intrinsic_name == "assert_uninit_valid"
 575                     && !layout.might_permit_raw_init(this, /*zero:*/ false)
 576                 {
 577                     throw_machine_stop!(TerminationInfo::Abort(format!(
 578                         "aborted execution: attempted to leave type `{}` uninitialized, which is invalid",
 579                         ty
 580                     )))
 581                 }
 582             }
 583
 584             // Other
 585             "exact_div" => {
 586                 let &[ref num, ref denom] = check_arg_count(args)?;
 587                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
 588             }
 589
 590             "try" => return this.handle_try(args, dest, ret),
 591
 592             "breakpoint" => {
 593                 let &[] = check_arg_count(args)?;
 594                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
 595                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
 596             }
 597
 598             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
 599         }
 600
 601         trace!("{:?}", this.dump_place(**dest));
 602         this.go_to_block(ret);
 603         Ok(())
 604     }
 605
 606     fn atomic_load(
 607         &mut self,
 608         args: &[OpTy<'tcx, Tag>],
 609         dest: &PlaceTy<'tcx, Tag>,
 610         atomic: AtomicReadOp,
 611     ) -> InterpResult<'tcx> {
 612         let this = self.eval_context_mut();
 613
 614         let &[ref place] = check_arg_count(args)?;
 615         let place = this.deref_operand(place)?;
 616
 617         // make sure it fits into a scalar; otherwise it cannot be atomic
 618         let val = this.read_scalar_atomic(&place, atomic)?;
 619
 620         // Check alignment requirements. Atomics must always be aligned to their size,
 621         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 622         // be 8-aligned).
 623         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 624         this.memory.check_ptr_access_align(
 625             place.ptr,
 626             place.layout.size,
 627             align,
 628             CheckInAllocMsg::MemoryAccessTest,
 629         )?;
 630         // Perform regular access.
 631         this.write_scalar(val, dest)?;
 632         Ok(())
 633     }
 634
 635     fn atomic_store(
 636         &mut self,
 637         args: &[OpTy<'tcx, Tag>],
 638         atomic: AtomicWriteOp,
 639     ) -> InterpResult<'tcx> {
 640         let this = self.eval_context_mut();
 641
 642         let &[ref place, ref val] = check_arg_count(args)?;
 643         let place = this.deref_operand(place)?;
 644         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
 645
 646         // Check alignment requirements. Atomics must always be aligned to their size,
 647         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 648         // be 8-aligned).
 649         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 650         this.memory.check_ptr_access_align(
 651             place.ptr,
 652             place.layout.size,
 653             align,
 654             CheckInAllocMsg::MemoryAccessTest,
 655         )?;
 656
 657         // Perform atomic store
 658         this.write_scalar_atomic(val, &place, atomic)?;
 659         Ok(())
 660     }
 661
 662     fn compiler_fence(
 663         &mut self,
 664         args: &[OpTy<'tcx, Tag>],
 665         atomic: AtomicFenceOp,
 666     ) -> InterpResult<'tcx> {
 667         let &[] = check_arg_count(args)?;
 668         let _ = atomic;
 669         //FIXME: compiler fences are currently ignored
 670         Ok(())
 671     }
 672
 673     fn atomic_fence(
 674         &mut self,
 675         args: &[OpTy<'tcx, Tag>],
 676         atomic: AtomicFenceOp,
 677     ) -> InterpResult<'tcx> {
 678         let this = self.eval_context_mut();
 679         let &[] = check_arg_count(args)?;
 680         this.validate_atomic_fence(atomic)?;
 681         Ok(())
 682     }
 683
 684     fn atomic_op(
 685         &mut self,
 686         args: &[OpTy<'tcx, Tag>],
 687         dest: &PlaceTy<'tcx, Tag>,
 688         atomic_op: AtomicOp,
 689         atomic: AtomicRwOp,
 690     ) -> InterpResult<'tcx> {
 691         let this = self.eval_context_mut();
 692
 693         let &[ref place, ref rhs] = check_arg_count(args)?;
 694         let place = this.deref_operand(place)?;
 695
 696         if !place.layout.ty.is_integral() {
 697             bug!("Atomic arithmetic operations only work on integer types");
 698         }
 699         let rhs = this.read_immediate(rhs)?;
 700
 701         // Check alignment requirements. Atomics must always be aligned to their size,
 702         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 703         // be 8-aligned).
 704         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 705         this.memory.check_ptr_access_align(
 706             place.ptr,
 707             place.layout.size,
 708             align,
 709             CheckInAllocMsg::MemoryAccessTest,
 710         )?;
 711
 712         match atomic_op {
 713             AtomicOp::Min => {
 714                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
 715                 this.write_immediate(*old, &dest)?; // old value is returned
 716                 Ok(())
 717             }
 718             AtomicOp::Max => {
 719                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
 720                 this.write_immediate(*old, &dest)?; // old value is returned
 721                 Ok(())
 722             }
 723             AtomicOp::MirOp(op, neg) => {
 724                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
 725                 this.write_immediate(*old, dest)?; // old value is returned
 726                 Ok(())
 727             }
 728         }
 729     }
 730
 731     fn atomic_exchange(
 732         &mut self,
 733         args: &[OpTy<'tcx, Tag>],
 734         dest: &PlaceTy<'tcx, Tag>,
 735         atomic: AtomicRwOp,
 736     ) -> InterpResult<'tcx> {
 737         let this = self.eval_context_mut();
 738
 739         let &[ref place, ref new] = check_arg_count(args)?;
 740         let place = this.deref_operand(place)?;
 741         let new = this.read_scalar(new)?;
 742
 743         // Check alignment requirements. Atomics must always be aligned to their size,
 744         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 745         // be 8-aligned).
 746         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 747         this.memory.check_ptr_access_align(
 748             place.ptr,
 749             place.layout.size,
 750             align,
 751             CheckInAllocMsg::MemoryAccessTest,
 752         )?;
 753
 754         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
 755         this.write_scalar(old, dest)?; // old value is returned
 756         Ok(())
 757     }
 758
 759     fn atomic_compare_exchange_impl(
 760         &mut self,
 761         args: &[OpTy<'tcx, Tag>],
 762         dest: &PlaceTy<'tcx, Tag>,
 763         success: AtomicRwOp,
 764         fail: AtomicReadOp,
 765         can_fail_spuriously: bool,
 766     ) -> InterpResult<'tcx> {
 767         let this = self.eval_context_mut();
 768
 769         let &[ref place, ref expect_old, ref new] = check_arg_count(args)?;
 770         let place = this.deref_operand(place)?;
 771         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
 772         let new = this.read_scalar(new)?;
 773
 774         // Check alignment requirements. Atomics must always be aligned to their size,
 775         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 776         // be 8-aligned).
 777         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 778         this.memory.check_ptr_access_align(
 779             place.ptr,
 780             place.layout.size,
 781             align,
 782             CheckInAllocMsg::MemoryAccessTest,
 783         )?;
 784
 785         let old = this.atomic_compare_exchange_scalar(
 786             &place,
 787             &expect_old,
 788             new,
 789             success,
 790             fail,
 791             can_fail_spuriously,
 792         )?;
 793
 794         // Return old value.
 795         this.write_immediate(old, dest)?;
 796         Ok(())
 797     }
 798
 799     fn atomic_compare_exchange(
 800         &mut self,
 801         args: &[OpTy<'tcx, Tag>],
 802         dest: &PlaceTy<'tcx, Tag>,
 803         success: AtomicRwOp,
 804         fail: AtomicReadOp,
 805     ) -> InterpResult<'tcx> {
 806         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
 807     }
 808
 809     fn atomic_compare_exchange_weak(
 810         &mut self,
 811         args: &[OpTy<'tcx, Tag>],
 812         dest: &PlaceTy<'tcx, Tag>,
 813         success: AtomicRwOp,
 814         fail: AtomicReadOp,
 815     ) -> InterpResult<'tcx> {
 816         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
 817     }
 818
 819     fn float_to_int_unchecked<F>(
 820         &self,
 821         f: F,
 822         dest_ty: ty::Ty<'tcx>,
 823     ) -> InterpResult<'tcx, Scalar<Tag>>
 824     where
 825         F: Float + Into<Scalar<Tag>>,
 826     {
 827         let this = self.eval_context_ref();
 828
 829         // Step 1: cut off the fractional part of `f`. The result of this is
 830         // guaranteed to be precisely representable in IEEE floats.
 831         let f = f.round_to_integral(Round::TowardZero).value;
 832
 833         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
 834         Ok(match dest_ty.kind() {
 835             // Unsigned
 836             ty::Uint(t) => {
 837                 let size = Integer::from_uint_ty(this, *t).size();
 838                 let res = f.to_u128(size.bits_usize());
 839                 if res.status.is_empty() {
 840                     // No status flags means there was no further rounding or other loss of precision.
 841                     Scalar::from_uint(res.value, size)
 842                 } else {
 843                     // `f` was not representable in this integer type.
 844                     throw_ub_format!(
 845                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 846                         f,
 847                         dest_ty,
 848                     );
 849                 }
 850             }
 851             // Signed
 852             ty::Int(t) => {
 853                 let size = Integer::from_int_ty(this, *t).size();
 854                 let res = f.to_i128(size.bits_usize());
 855                 if res.status.is_empty() {
 856                     // No status flags means there was no further rounding or other loss of precision.
 857                     Scalar::from_int(res.value, size)
 858                 } else {
 859                     // `f` was not representable in this integer type.
 860                     throw_ub_format!(
 861                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 862                         f,
 863                         dest_ty,
 864                     );
 865                 }
 866             }
 867             // Nothing else
 868             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
 869         })
 870     }
 871 }