src/shims/intrinsics.rs

   1 use std::iter;
   2
   3 use log::trace;
   4
   5 use rustc_apfloat::{Float, Round};
   6 use rustc_middle::ty::layout::IntegerExt;
   7 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
   8 use rustc_target::abi::{Align, Integer, LayoutOf};
   9
  10 use crate::*;
  11 use helpers::check_arg_count;
  12
  13 pub enum AtomicOp {
  14     MirOp(mir::BinOp, bool),
  15     Max,
  16     Min,
  17 }
  18
  19 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  20 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  21     fn call_intrinsic(
  22         &mut self,
  23         instance: ty::Instance<'tcx>,
  24         args: &[OpTy<'tcx, Tag>],
  25         ret: Option<(&PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  26         _unwind: StackPopUnwind,
  27     ) -> InterpResult<'tcx> {
  28         let this = self.eval_context_mut();
  29
  30         if this.emulate_intrinsic(instance, args, ret)? {
  31             return Ok(());
  32         }
  33
  34         // All supported intrinsics have a return place.
  35         let intrinsic_name = &*this.tcx.item_name(instance.def_id()).as_str();
  36         let (dest, ret) = match ret {
  37             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  38             Some(p) => p,
  39         };
  40
  41         // Then handle terminating intrinsics.
  42         match intrinsic_name {
  43             // Miri overwriting CTFE intrinsics.
  44             "ptr_guaranteed_eq" => {
  45                 let &[ref left, ref right] = check_arg_count(args)?;
  46                 let left = this.read_immediate(left)?;
  47                 let right = this.read_immediate(right)?;
  48                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  49             }
  50             "ptr_guaranteed_ne" => {
  51                 let &[ref left, ref right] = check_arg_count(args)?;
  52                 let left = this.read_immediate(left)?;
  53                 let right = this.read_immediate(right)?;
  54                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  55             }
  56
  57             // Raw memory accesses
  58             "volatile_load" => {
  59                 let &[ref place] = check_arg_count(args)?;
  60                 let place = this.deref_operand(place)?;
  61                 this.copy_op(&place.into(), dest)?;
  62             }
  63             "volatile_store" => {
  64                 let &[ref place, ref dest] = check_arg_count(args)?;
  65                 let place = this.deref_operand(place)?;
  66                 this.copy_op(dest, &place.into())?;
  67             }
  68
  69             "write_bytes" => {
  70                 let &[ref ptr, ref val_byte, ref count] = check_arg_count(args)?;
  71                 let ty = instance.substs.type_at(0);
  72                 let ty_layout = this.layout_of(ty)?;
  73                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
  74                 let ptr = this.read_scalar(ptr)?.check_init()?;
  75                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  76                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
  77                     err_ub_format!("overflow computing total size of `write_bytes`")
  78                 })?;
  79                 this.memory
  80                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
  81             }
  82
  83             // Floating-point operations
  84             #[rustfmt::skip]
  85             | "sinf32"
  86             | "fabsf32"
  87             | "cosf32"
  88             | "sqrtf32"
  89             | "expf32"
  90             | "exp2f32"
  91             | "logf32"
  92             | "log10f32"
  93             | "log2f32"
  94             | "floorf32"
  95             | "ceilf32"
  96             | "truncf32"
  97             | "roundf32"
  98             => {
  99                 let &[ref f] = check_arg_count(args)?;
 100                 // FIXME: Using host floats.
 101                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 102                 let f = match intrinsic_name {
 103                     "sinf32" => f.sin(),
 104                     "fabsf32" => f.abs(),
 105                     "cosf32" => f.cos(),
 106                     "sqrtf32" => f.sqrt(),
 107                     "expf32" => f.exp(),
 108                     "exp2f32" => f.exp2(),
 109                     "logf32" => f.ln(),
 110                     "log10f32" => f.log10(),
 111                     "log2f32" => f.log2(),
 112                     "floorf32" => f.floor(),
 113                     "ceilf32" => f.ceil(),
 114                     "truncf32" => f.trunc(),
 115                     "roundf32" => f.round(),
 116                     _ => bug!(),
 117                 };
 118                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 119             }
 120
 121             #[rustfmt::skip]
 122             | "sinf64"
 123             | "fabsf64"
 124             | "cosf64"
 125             | "sqrtf64"
 126             | "expf64"
 127             | "exp2f64"
 128             | "logf64"
 129             | "log10f64"
 130             | "log2f64"
 131             | "floorf64"
 132             | "ceilf64"
 133             | "truncf64"
 134             | "roundf64"
 135             => {
 136                 let &[ref f] = check_arg_count(args)?;
 137                 // FIXME: Using host floats.
 138                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 139                 let f = match intrinsic_name {
 140                     "sinf64" => f.sin(),
 141                     "fabsf64" => f.abs(),
 142                     "cosf64" => f.cos(),
 143                     "sqrtf64" => f.sqrt(),
 144                     "expf64" => f.exp(),
 145                     "exp2f64" => f.exp2(),
 146                     "logf64" => f.ln(),
 147                     "log10f64" => f.log10(),
 148                     "log2f64" => f.log2(),
 149                     "floorf64" => f.floor(),
 150                     "ceilf64" => f.ceil(),
 151                     "truncf64" => f.trunc(),
 152                     "roundf64" => f.round(),
 153                     _ => bug!(),
 154                 };
 155                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 156             }
 157
 158             #[rustfmt::skip]
 159             | "fadd_fast"
 160             | "fsub_fast"
 161             | "fmul_fast"
 162             | "fdiv_fast"
 163             | "frem_fast"
 164             => {
 165                 let &[ref a, ref b] = check_arg_count(args)?;
 166                 let a = this.read_immediate(a)?;
 167                 let b = this.read_immediate(b)?;
 168                 let op = match intrinsic_name {
 169                     "fadd_fast" => mir::BinOp::Add,
 170                     "fsub_fast" => mir::BinOp::Sub,
 171                     "fmul_fast" => mir::BinOp::Mul,
 172                     "fdiv_fast" => mir::BinOp::Div,
 173                     "frem_fast" => mir::BinOp::Rem,
 174                     _ => bug!(),
 175                 };
 176                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 177                     Ok(match x.layout.ty.kind() {
 178                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 179                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 180                         _ => bug!(
 181                             "`{}` called with non-float input type {:?}",
 182                             intrinsic_name,
 183                             x.layout.ty
 184                         ),
 185                     })
 186                 };
 187                 match (float_finite(a)?, float_finite(b)?) {
 188                     (false, false) => throw_ub_format!(
 189                         "`{}` intrinsic called with non-finite value as both parameters",
 190                         intrinsic_name,
 191                     ),
 192                     (false, _) => throw_ub_format!(
 193                         "`{}` intrinsic called with non-finite value as first parameter",
 194                         intrinsic_name,
 195                     ),
 196                     (_, false) => throw_ub_format!(
 197                         "`{}` intrinsic called with non-finite value as second parameter",
 198                         intrinsic_name,
 199                     ),
 200                     _ => {}
 201                 }
 202                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 203             }
 204
 205             #[rustfmt::skip]
 206             | "minnumf32"
 207             | "maxnumf32"
 208             | "copysignf32"
 209             => {
 210                 let &[ref a, ref b] = check_arg_count(args)?;
 211                 let a = this.read_scalar(a)?.to_f32()?;
 212                 let b = this.read_scalar(b)?.to_f32()?;
 213                 let res = match intrinsic_name {
 214                     "minnumf32" => a.min(b),
 215                     "maxnumf32" => a.max(b),
 216                     "copysignf32" => a.copy_sign(b),
 217                     _ => bug!(),
 218                 };
 219                 this.write_scalar(Scalar::from_f32(res), dest)?;
 220             }
 221
 222             #[rustfmt::skip]
 223             | "minnumf64"
 224             | "maxnumf64"
 225             | "copysignf64"
 226             => {
 227                 let &[ref a, ref b] = check_arg_count(args)?;
 228                 let a = this.read_scalar(a)?.to_f64()?;
 229                 let b = this.read_scalar(b)?.to_f64()?;
 230                 let res = match intrinsic_name {
 231                     "minnumf64" => a.min(b),
 232                     "maxnumf64" => a.max(b),
 233                     "copysignf64" => a.copy_sign(b),
 234                     _ => bug!(),
 235                 };
 236                 this.write_scalar(Scalar::from_f64(res), dest)?;
 237             }
 238
 239             "powf32" => {
 240                 let &[ref f, ref f2] = check_arg_count(args)?;
 241                 // FIXME: Using host floats.
 242                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 243                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 244                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 245             }
 246
 247             "powf64" => {
 248                 let &[ref f, ref f2] = check_arg_count(args)?;
 249                 // FIXME: Using host floats.
 250                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 251                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 252                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 253             }
 254
 255             "fmaf32" => {
 256                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 257                 let a = this.read_scalar(a)?.to_f32()?;
 258                 let b = this.read_scalar(b)?.to_f32()?;
 259                 let c = this.read_scalar(c)?.to_f32()?;
 260                 let res = a.mul_add(b, c).value;
 261                 this.write_scalar(Scalar::from_f32(res), dest)?;
 262             }
 263
 264             "fmaf64" => {
 265                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 266                 let a = this.read_scalar(a)?.to_f64()?;
 267                 let b = this.read_scalar(b)?.to_f64()?;
 268                 let c = this.read_scalar(c)?.to_f64()?;
 269                 let res = a.mul_add(b, c).value;
 270                 this.write_scalar(Scalar::from_f64(res), dest)?;
 271             }
 272
 273             "powif32" => {
 274                 let &[ref f, ref i] = check_arg_count(args)?;
 275                 // FIXME: Using host floats.
 276                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 277                 let i = this.read_scalar(i)?.to_i32()?;
 278                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 279             }
 280
 281             "powif64" => {
 282                 let &[ref f, ref i] = check_arg_count(args)?;
 283                 // FIXME: Using host floats.
 284                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 285                 let i = this.read_scalar(i)?.to_i32()?;
 286                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 287             }
 288
 289             "float_to_int_unchecked" => {
 290                 let &[ref val] = check_arg_count(args)?;
 291                 let val = this.read_immediate(val)?;
 292
 293                 let res = match val.layout.ty.kind() {
 294                     ty::Float(FloatTy::F32) =>
 295                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 296                     ty::Float(FloatTy::F64) =>
 297                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 298                     _ => bug!(
 299                         "`float_to_int_unchecked` called with non-float input type {:?}",
 300                         val.layout.ty
 301                     ),
 302                 };
 303
 304                 this.write_scalar(res, dest)?;
 305             }
 306
 307             // Atomic operations
 308             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
 309             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
 310             "atomic_load_acq" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
 311
 312             "atomic_store" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
 313             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
 314             "atomic_store_rel" => this.atomic_store(args, AtomicWriteOp::Release)?,
 315
 316             "atomic_fence_acq" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
 317             "atomic_fence_rel" => this.atomic_fence(args, AtomicFenceOp::Release)?,
 318             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
 319             "atomic_fence" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
 320
 321             "atomic_singlethreadfence_acq" => this.compiler_fence(args, AtomicFenceOp::Acquire)?,
 322             "atomic_singlethreadfence_rel" => this.compiler_fence(args, AtomicFenceOp::Release)?,
 323             "atomic_singlethreadfence_acqrel" =>
 324                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
 325             "atomic_singlethreadfence" => this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
 326
 327             "atomic_xchg" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
 328             "atomic_xchg_acq" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
 329             "atomic_xchg_rel" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
 330             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
 331             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
 332
 333             #[rustfmt::skip]
 334             "atomic_cxchg" =>
 335                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 336             #[rustfmt::skip]
 337             "atomic_cxchg_acq" =>
 338                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 339             #[rustfmt::skip]
 340             "atomic_cxchg_rel" =>
 341                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 342             #[rustfmt::skip]
 343             "atomic_cxchg_acqrel" =>
 344                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 345             #[rustfmt::skip]
 346             "atomic_cxchg_relaxed" =>
 347                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 348             #[rustfmt::skip]
 349             "atomic_cxchg_acq_failrelaxed" =>
 350                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 351             #[rustfmt::skip]
 352             "atomic_cxchg_acqrel_failrelaxed" =>
 353                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 354             #[rustfmt::skip]
 355             "atomic_cxchg_failrelaxed" =>
 356                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 357             #[rustfmt::skip]
 358             "atomic_cxchg_failacq" =>
 359                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 360
 361             #[rustfmt::skip]
 362             "atomic_cxchgweak" =>
 363                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 364             #[rustfmt::skip]
 365             "atomic_cxchgweak_acq" =>
 366                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 367             #[rustfmt::skip]
 368             "atomic_cxchgweak_rel" =>
 369                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 370             #[rustfmt::skip]
 371             "atomic_cxchgweak_acqrel" =>
 372                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 373             #[rustfmt::skip]
 374             "atomic_cxchgweak_relaxed" =>
 375                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 376             #[rustfmt::skip]
 377             "atomic_cxchgweak_acq_failrelaxed" =>
 378                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 379             #[rustfmt::skip]
 380             "atomic_cxchgweak_acqrel_failrelaxed" =>
 381                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 382             #[rustfmt::skip]
 383             "atomic_cxchgweak_failrelaxed" =>
 384                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 385             #[rustfmt::skip]
 386             "atomic_cxchgweak_failacq" =>
 387                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 388
 389             #[rustfmt::skip]
 390             "atomic_or" =>
 391                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::SeqCst)?,
 392             #[rustfmt::skip]
 393             "atomic_or_acq" =>
 394                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Acquire)?,
 395             #[rustfmt::skip]
 396             "atomic_or_rel" =>
 397                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Release)?,
 398             #[rustfmt::skip]
 399             "atomic_or_acqrel" =>
 400                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::AcqRel)?,
 401             #[rustfmt::skip]
 402             "atomic_or_relaxed" =>
 403                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Relaxed)?,
 404             #[rustfmt::skip]
 405             "atomic_xor" =>
 406                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::SeqCst)?,
 407             #[rustfmt::skip]
 408             "atomic_xor_acq" =>
 409                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Acquire)?,
 410             #[rustfmt::skip]
 411             "atomic_xor_rel" =>
 412                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Release)?,
 413             #[rustfmt::skip]
 414             "atomic_xor_acqrel" =>
 415                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::AcqRel)?,
 416             #[rustfmt::skip]
 417             "atomic_xor_relaxed" =>
 418                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Relaxed)?,
 419             #[rustfmt::skip]
 420             "atomic_and" =>
 421                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::SeqCst)?,
 422             #[rustfmt::skip]
 423             "atomic_and_acq" =>
 424                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Acquire)?,
 425             #[rustfmt::skip]
 426             "atomic_and_rel" =>
 427                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Release)?,
 428             #[rustfmt::skip]
 429             "atomic_and_acqrel" =>
 430                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::AcqRel)?,
 431             #[rustfmt::skip]
 432             "atomic_and_relaxed" =>
 433                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Relaxed)?,
 434             #[rustfmt::skip]
 435             "atomic_nand" =>
 436                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::SeqCst)?,
 437             #[rustfmt::skip]
 438             "atomic_nand_acq" =>
 439                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Acquire)?,
 440             #[rustfmt::skip]
 441             "atomic_nand_rel" =>
 442                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Release)?,
 443             #[rustfmt::skip]
 444             "atomic_nand_acqrel" =>
 445                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::AcqRel)?,
 446             #[rustfmt::skip]
 447             "atomic_nand_relaxed" =>
 448                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Relaxed)?,
 449             #[rustfmt::skip]
 450             "atomic_xadd" =>
 451                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::SeqCst)?,
 452             #[rustfmt::skip]
 453             "atomic_xadd_acq" =>
 454                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Acquire)?,
 455             #[rustfmt::skip]
 456             "atomic_xadd_rel" =>
 457                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Release)?,
 458             #[rustfmt::skip]
 459             "atomic_xadd_acqrel" =>
 460                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::AcqRel)?,
 461             #[rustfmt::skip]
 462             "atomic_xadd_relaxed" =>
 463                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Relaxed)?,
 464             #[rustfmt::skip]
 465             "atomic_xsub" =>
 466                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::SeqCst)?,
 467             #[rustfmt::skip]
 468             "atomic_xsub_acq" =>
 469                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Acquire)?,
 470             #[rustfmt::skip]
 471             "atomic_xsub_rel" =>
 472                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Release)?,
 473             #[rustfmt::skip]
 474             "atomic_xsub_acqrel" =>
 475                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::AcqRel)?,
 476             #[rustfmt::skip]
 477             "atomic_xsub_relaxed" =>
 478                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Relaxed)?,
 479             "atomic_min" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 480             "atomic_min_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 481             "atomic_min_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 482             "atomic_min_acqrel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 483             "atomic_min_relaxed" =>
 484                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 485             "atomic_max" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 486             "atomic_max_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 487             "atomic_max_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 488             "atomic_max_acqrel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 489             "atomic_max_relaxed" =>
 490                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 491             "atomic_umin" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 492             "atomic_umin_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 493             "atomic_umin_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 494             "atomic_umin_acqrel" =>
 495                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 496             "atomic_umin_relaxed" =>
 497                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 498             "atomic_umax" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 499             "atomic_umax_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 500             "atomic_umax_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 501             "atomic_umax_acqrel" =>
 502                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 503             "atomic_umax_relaxed" =>
 504                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 505
 506             // Query type information
 507             "assert_zero_valid" | "assert_uninit_valid" => {
 508                 let &[] = check_arg_count(args)?;
 509                 let ty = instance.substs.type_at(0);
 510                 let layout = this.layout_of(ty)?;
 511                 // Abort here because the caller might not be panic safe.
 512                 if layout.abi.is_uninhabited() {
 513                     // Use this message even for the other intrinsics, as that's what codegen does
 514                     throw_machine_stop!(TerminationInfo::Abort(format!(
 515                         "aborted execution: attempted to instantiate uninhabited type `{}`",
 516                         ty
 517                     )))
 518                 }
 519                 if intrinsic_name == "assert_zero_valid"
 520                     && !layout.might_permit_raw_init(this, /*zero:*/ true).unwrap()
 521                 {
 522                     throw_machine_stop!(TerminationInfo::Abort(format!(
 523                         "aborted execution: attempted to zero-initialize type `{}`, which is invalid",
 524                         ty
 525                     )))
 526                 }
 527                 if intrinsic_name == "assert_uninit_valid"
 528                     && !layout.might_permit_raw_init(this, /*zero:*/ false).unwrap()
 529                 {
 530                     throw_machine_stop!(TerminationInfo::Abort(format!(
 531                         "aborted execution: attempted to leave type `{}` uninitialized, which is invalid",
 532                         ty
 533                     )))
 534                 }
 535             }
 536
 537             // Other
 538             "exact_div" => {
 539                 let &[ref num, ref denom] = check_arg_count(args)?;
 540                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
 541             }
 542
 543             "try" => return this.handle_try(args, dest, ret),
 544
 545             "breakpoint" => {
 546                 let &[] = check_arg_count(args)?;
 547                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
 548                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
 549             }
 550
 551             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
 552         }
 553
 554         trace!("{:?}", this.dump_place(**dest));
 555         this.go_to_block(ret);
 556         Ok(())
 557     }
 558
 559     fn atomic_load(
 560         &mut self,
 561         args: &[OpTy<'tcx, Tag>],
 562         dest: &PlaceTy<'tcx, Tag>,
 563         atomic: AtomicReadOp,
 564     ) -> InterpResult<'tcx> {
 565         let this = self.eval_context_mut();
 566
 567         let &[ref place] = check_arg_count(args)?;
 568         let place = this.deref_operand(place)?;
 569
 570         // make sure it fits into a scalar; otherwise it cannot be atomic
 571         let val = this.read_scalar_atomic(&place, atomic)?;
 572
 573         // Check alignment requirements. Atomics must always be aligned to their size,
 574         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 575         // be 8-aligned).
 576         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 577         this.memory.check_ptr_access_align(
 578             place.ptr,
 579             place.layout.size,
 580             align,
 581             CheckInAllocMsg::MemoryAccessTest,
 582         )?;
 583         // Perform regular access.
 584         this.write_scalar(val, dest)?;
 585         Ok(())
 586     }
 587
 588     fn atomic_store(
 589         &mut self,
 590         args: &[OpTy<'tcx, Tag>],
 591         atomic: AtomicWriteOp,
 592     ) -> InterpResult<'tcx> {
 593         let this = self.eval_context_mut();
 594
 595         let &[ref place, ref val] = check_arg_count(args)?;
 596         let place = this.deref_operand(place)?;
 597         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
 598
 599         // Check alignment requirements. Atomics must always be aligned to their size,
 600         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 601         // be 8-aligned).
 602         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 603         this.memory.check_ptr_access_align(
 604             place.ptr,
 605             place.layout.size,
 606             align,
 607             CheckInAllocMsg::MemoryAccessTest,
 608         )?;
 609
 610         // Perform atomic store
 611         this.write_scalar_atomic(val, &place, atomic)?;
 612         Ok(())
 613     }
 614
 615     fn compiler_fence(
 616         &mut self,
 617         args: &[OpTy<'tcx, Tag>],
 618         atomic: AtomicFenceOp,
 619     ) -> InterpResult<'tcx> {
 620         let &[] = check_arg_count(args)?;
 621         let _ = atomic;
 622         //FIXME: compiler fences are currently ignored
 623         Ok(())
 624     }
 625
 626     fn atomic_fence(
 627         &mut self,
 628         args: &[OpTy<'tcx, Tag>],
 629         atomic: AtomicFenceOp,
 630     ) -> InterpResult<'tcx> {
 631         let this = self.eval_context_mut();
 632         let &[] = check_arg_count(args)?;
 633         this.validate_atomic_fence(atomic)?;
 634         Ok(())
 635     }
 636
 637     fn atomic_op(
 638         &mut self,
 639         args: &[OpTy<'tcx, Tag>],
 640         dest: &PlaceTy<'tcx, Tag>,
 641         atomic_op: AtomicOp,
 642         atomic: AtomicRwOp,
 643     ) -> InterpResult<'tcx> {
 644         let this = self.eval_context_mut();
 645
 646         let &[ref place, ref rhs] = check_arg_count(args)?;
 647         let place = this.deref_operand(place)?;
 648
 649         if !place.layout.ty.is_integral() {
 650             bug!("Atomic arithmetic operations only work on integer types");
 651         }
 652         let rhs = this.read_immediate(rhs)?;
 653
 654         // Check alignment requirements. Atomics must always be aligned to their size,
 655         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 656         // be 8-aligned).
 657         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 658         this.memory.check_ptr_access_align(
 659             place.ptr,
 660             place.layout.size,
 661             align,
 662             CheckInAllocMsg::MemoryAccessTest,
 663         )?;
 664
 665         match atomic_op {
 666             AtomicOp::Min => {
 667                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
 668                 this.write_immediate(*old, &dest)?; // old value is returned
 669                 Ok(())
 670             }
 671             AtomicOp::Max => {
 672                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
 673                 this.write_immediate(*old, &dest)?; // old value is returned
 674                 Ok(())
 675             }
 676             AtomicOp::MirOp(op, neg) => {
 677                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
 678                 this.write_immediate(*old, dest)?; // old value is returned
 679                 Ok(())
 680             }
 681         }
 682     }
 683
 684     fn atomic_exchange(
 685         &mut self,
 686         args: &[OpTy<'tcx, Tag>],
 687         dest: &PlaceTy<'tcx, Tag>,
 688         atomic: AtomicRwOp,
 689     ) -> InterpResult<'tcx> {
 690         let this = self.eval_context_mut();
 691
 692         let &[ref place, ref new] = check_arg_count(args)?;
 693         let place = this.deref_operand(place)?;
 694         let new = this.read_scalar(new)?;
 695
 696         // Check alignment requirements. Atomics must always be aligned to their size,
 697         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 698         // be 8-aligned).
 699         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 700         this.memory.check_ptr_access_align(
 701             place.ptr,
 702             place.layout.size,
 703             align,
 704             CheckInAllocMsg::MemoryAccessTest,
 705         )?;
 706
 707         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
 708         this.write_scalar(old, dest)?; // old value is returned
 709         Ok(())
 710     }
 711
 712     fn atomic_compare_exchange_impl(
 713         &mut self,
 714         args: &[OpTy<'tcx, Tag>],
 715         dest: &PlaceTy<'tcx, Tag>,
 716         success: AtomicRwOp,
 717         fail: AtomicReadOp,
 718         can_fail_spuriously: bool,
 719     ) -> InterpResult<'tcx> {
 720         let this = self.eval_context_mut();
 721
 722         let &[ref place, ref expect_old, ref new] = check_arg_count(args)?;
 723         let place = this.deref_operand(place)?;
 724         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
 725         let new = this.read_scalar(new)?;
 726
 727         // Check alignment requirements. Atomics must always be aligned to their size,
 728         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 729         // be 8-aligned).
 730         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 731         this.memory.check_ptr_access_align(
 732             place.ptr,
 733             place.layout.size,
 734             align,
 735             CheckInAllocMsg::MemoryAccessTest,
 736         )?;
 737
 738         let old = this.atomic_compare_exchange_scalar(
 739             &place,
 740             &expect_old,
 741             new,
 742             success,
 743             fail,
 744             can_fail_spuriously,
 745         )?;
 746
 747         // Return old value.
 748         this.write_immediate(old, dest)?;
 749         Ok(())
 750     }
 751
 752     fn atomic_compare_exchange(
 753         &mut self,
 754         args: &[OpTy<'tcx, Tag>],
 755         dest: &PlaceTy<'tcx, Tag>,
 756         success: AtomicRwOp,
 757         fail: AtomicReadOp,
 758     ) -> InterpResult<'tcx> {
 759         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
 760     }
 761
 762     fn atomic_compare_exchange_weak(
 763         &mut self,
 764         args: &[OpTy<'tcx, Tag>],
 765         dest: &PlaceTy<'tcx, Tag>,
 766         success: AtomicRwOp,
 767         fail: AtomicReadOp,
 768     ) -> InterpResult<'tcx> {
 769         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
 770     }
 771
 772     fn float_to_int_unchecked<F>(
 773         &self,
 774         f: F,
 775         dest_ty: ty::Ty<'tcx>,
 776     ) -> InterpResult<'tcx, Scalar<Tag>>
 777     where
 778         F: Float + Into<Scalar<Tag>>,
 779     {
 780         let this = self.eval_context_ref();
 781
 782         // Step 1: cut off the fractional part of `f`. The result of this is
 783         // guaranteed to be precisely representable in IEEE floats.
 784         let f = f.round_to_integral(Round::TowardZero).value;
 785
 786         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
 787         Ok(match dest_ty.kind() {
 788             // Unsigned
 789             ty::Uint(t) => {
 790                 let size = Integer::from_uint_ty(this, *t).size();
 791                 let res = f.to_u128(size.bits_usize());
 792                 if res.status.is_empty() {
 793                     // No status flags means there was no further rounding or other loss of precision.
 794                     Scalar::from_uint(res.value, size)
 795                 } else {
 796                     // `f` was not representable in this integer type.
 797                     throw_ub_format!(
 798                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 799                         f,
 800                         dest_ty,
 801                     );
 802                 }
 803             }
 804             // Signed
 805             ty::Int(t) => {
 806                 let size = Integer::from_int_ty(this, *t).size();
 807                 let res = f.to_i128(size.bits_usize());
 808                 if res.status.is_empty() {
 809                     // No status flags means there was no further rounding or other loss of precision.
 810                     Scalar::from_int(res.value, size)
 811                 } else {
 812                     // `f` was not representable in this integer type.
 813                     throw_ub_format!(
 814                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 815                         f,
 816                         dest_ty,
 817                     );
 818                 }
 819             }
 820             // Nothing else
 821             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
 822         })
 823     }
 824 }