src/shims/intrinsics.rs

   1 use std::iter;
   2
   3 use log::trace;
   4
   5 use rustc_apfloat::{Float, Round};
   6 use rustc_middle::ty::layout::{IntegerExt, LayoutOf};
   7 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
   8 use rustc_target::abi::{Align, Integer};
   9
  10 use crate::*;
  11 use helpers::check_arg_count;
  12
  13 pub enum AtomicOp {
  14     MirOp(mir::BinOp, bool),
  15     Max,
  16     Min,
  17 }
  18
  19 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  20 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  21     fn call_intrinsic(
  22         &mut self,
  23         instance: ty::Instance<'tcx>,
  24         args: &[OpTy<'tcx, Tag>],
  25         ret: Option<(&PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  26         _unwind: StackPopUnwind,
  27     ) -> InterpResult<'tcx> {
  28         let this = self.eval_context_mut();
  29
  30         if this.emulate_intrinsic(instance, args, ret)? {
  31             return Ok(());
  32         }
  33
  34         // All supported intrinsics have a return place.
  35         let intrinsic_name = &*this.tcx.item_name(instance.def_id()).as_str();
  36         let (dest, ret) = match ret {
  37             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  38             Some(p) => p,
  39         };
  40
  41         // Then handle terminating intrinsics.
  42         match intrinsic_name {
  43             // Miri overwriting CTFE intrinsics.
  44             "ptr_guaranteed_eq" => {
  45                 let &[ref left, ref right] = check_arg_count(args)?;
  46                 let left = this.read_immediate(left)?;
  47                 let right = this.read_immediate(right)?;
  48                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  49             }
  50             "ptr_guaranteed_ne" => {
  51                 let &[ref left, ref right] = check_arg_count(args)?;
  52                 let left = this.read_immediate(left)?;
  53                 let right = this.read_immediate(right)?;
  54                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  55             }
  56
  57             // Raw memory accesses
  58             "volatile_load" => {
  59                 let &[ref place] = check_arg_count(args)?;
  60                 let place = this.deref_operand(place)?;
  61                 this.copy_op(&place.into(), dest)?;
  62             }
  63             "volatile_store" => {
  64                 let &[ref place, ref dest] = check_arg_count(args)?;
  65                 let place = this.deref_operand(place)?;
  66                 this.copy_op(dest, &place.into())?;
  67             }
  68
  69             "write_bytes" | "volatile_set_memory" => {
  70                 let &[ref ptr, ref val_byte, ref count] = check_arg_count(args)?;
  71                 let ty = instance.substs.type_at(0);
  72                 let ty_layout = this.layout_of(ty)?;
  73                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
  74                 let ptr = this.read_pointer(ptr)?;
  75                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  76                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
  77                     err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
  78                 })?;
  79                 this.memory
  80                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
  81             }
  82
  83             // Floating-point operations
  84             #[rustfmt::skip]
  85             | "sinf32"
  86             | "fabsf32"
  87             | "cosf32"
  88             | "sqrtf32"
  89             | "expf32"
  90             | "exp2f32"
  91             | "logf32"
  92             | "log10f32"
  93             | "log2f32"
  94             | "floorf32"
  95             | "ceilf32"
  96             | "truncf32"
  97             | "roundf32"
  98             => {
  99                 let &[ref f] = check_arg_count(args)?;
 100                 // FIXME: Using host floats.
 101                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 102                 let f = match intrinsic_name {
 103                     "sinf32" => f.sin(),
 104                     "fabsf32" => f.abs(),
 105                     "cosf32" => f.cos(),
 106                     "sqrtf32" => f.sqrt(),
 107                     "expf32" => f.exp(),
 108                     "exp2f32" => f.exp2(),
 109                     "logf32" => f.ln(),
 110                     "log10f32" => f.log10(),
 111                     "log2f32" => f.log2(),
 112                     "floorf32" => f.floor(),
 113                     "ceilf32" => f.ceil(),
 114                     "truncf32" => f.trunc(),
 115                     "roundf32" => f.round(),
 116                     _ => bug!(),
 117                 };
 118                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 119             }
 120
 121             #[rustfmt::skip]
 122             | "sinf64"
 123             | "fabsf64"
 124             | "cosf64"
 125             | "sqrtf64"
 126             | "expf64"
 127             | "exp2f64"
 128             | "logf64"
 129             | "log10f64"
 130             | "log2f64"
 131             | "floorf64"
 132             | "ceilf64"
 133             | "truncf64"
 134             | "roundf64"
 135             => {
 136                 let &[ref f] = check_arg_count(args)?;
 137                 // FIXME: Using host floats.
 138                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 139                 let f = match intrinsic_name {
 140                     "sinf64" => f.sin(),
 141                     "fabsf64" => f.abs(),
 142                     "cosf64" => f.cos(),
 143                     "sqrtf64" => f.sqrt(),
 144                     "expf64" => f.exp(),
 145                     "exp2f64" => f.exp2(),
 146                     "logf64" => f.ln(),
 147                     "log10f64" => f.log10(),
 148                     "log2f64" => f.log2(),
 149                     "floorf64" => f.floor(),
 150                     "ceilf64" => f.ceil(),
 151                     "truncf64" => f.trunc(),
 152                     "roundf64" => f.round(),
 153                     _ => bug!(),
 154                 };
 155                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 156             }
 157
 158             #[rustfmt::skip]
 159             | "fadd_fast"
 160             | "fsub_fast"
 161             | "fmul_fast"
 162             | "fdiv_fast"
 163             | "frem_fast"
 164             => {
 165                 let &[ref a, ref b] = check_arg_count(args)?;
 166                 let a = this.read_immediate(a)?;
 167                 let b = this.read_immediate(b)?;
 168                 let op = match intrinsic_name {
 169                     "fadd_fast" => mir::BinOp::Add,
 170                     "fsub_fast" => mir::BinOp::Sub,
 171                     "fmul_fast" => mir::BinOp::Mul,
 172                     "fdiv_fast" => mir::BinOp::Div,
 173                     "frem_fast" => mir::BinOp::Rem,
 174                     _ => bug!(),
 175                 };
 176                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 177                     Ok(match x.layout.ty.kind() {
 178                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 179                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 180                         _ => bug!(
 181                             "`{}` called with non-float input type {:?}",
 182                             intrinsic_name,
 183                             x.layout.ty
 184                         ),
 185                     })
 186                 };
 187                 match (float_finite(a)?, float_finite(b)?) {
 188                     (false, false) => throw_ub_format!(
 189                         "`{}` intrinsic called with non-finite value as both parameters",
 190                         intrinsic_name,
 191                     ),
 192                     (false, _) => throw_ub_format!(
 193                         "`{}` intrinsic called with non-finite value as first parameter",
 194                         intrinsic_name,
 195                     ),
 196                     (_, false) => throw_ub_format!(
 197                         "`{}` intrinsic called with non-finite value as second parameter",
 198                         intrinsic_name,
 199                     ),
 200                     _ => {}
 201                 }
 202                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 203             }
 204
 205             #[rustfmt::skip]
 206             | "minnumf32"
 207             | "maxnumf32"
 208             | "copysignf32"
 209             => {
 210                 let &[ref a, ref b] = check_arg_count(args)?;
 211                 let a = this.read_scalar(a)?.to_f32()?;
 212                 let b = this.read_scalar(b)?.to_f32()?;
 213                 let res = match intrinsic_name {
 214                     "minnumf32" => a.min(b),
 215                     "maxnumf32" => a.max(b),
 216                     "copysignf32" => a.copy_sign(b),
 217                     _ => bug!(),
 218                 };
 219                 this.write_scalar(Scalar::from_f32(res), dest)?;
 220             }
 221
 222             #[rustfmt::skip]
 223             | "minnumf64"
 224             | "maxnumf64"
 225             | "copysignf64"
 226             => {
 227                 let &[ref a, ref b] = check_arg_count(args)?;
 228                 let a = this.read_scalar(a)?.to_f64()?;
 229                 let b = this.read_scalar(b)?.to_f64()?;
 230                 let res = match intrinsic_name {
 231                     "minnumf64" => a.min(b),
 232                     "maxnumf64" => a.max(b),
 233                     "copysignf64" => a.copy_sign(b),
 234                     _ => bug!(),
 235                 };
 236                 this.write_scalar(Scalar::from_f64(res), dest)?;
 237             }
 238
 239             "powf32" => {
 240                 let &[ref f, ref f2] = check_arg_count(args)?;
 241                 // FIXME: Using host floats.
 242                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 243                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 244                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 245             }
 246
 247             "powf64" => {
 248                 let &[ref f, ref f2] = check_arg_count(args)?;
 249                 // FIXME: Using host floats.
 250                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 251                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 252                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 253             }
 254
 255             "fmaf32" => {
 256                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 257                 let a = this.read_scalar(a)?.to_f32()?;
 258                 let b = this.read_scalar(b)?.to_f32()?;
 259                 let c = this.read_scalar(c)?.to_f32()?;
 260                 let res = a.mul_add(b, c).value;
 261                 this.write_scalar(Scalar::from_f32(res), dest)?;
 262             }
 263
 264             "fmaf64" => {
 265                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 266                 let a = this.read_scalar(a)?.to_f64()?;
 267                 let b = this.read_scalar(b)?.to_f64()?;
 268                 let c = this.read_scalar(c)?.to_f64()?;
 269                 let res = a.mul_add(b, c).value;
 270                 this.write_scalar(Scalar::from_f64(res), dest)?;
 271             }
 272
 273             "powif32" => {
 274                 let &[ref f, ref i] = check_arg_count(args)?;
 275                 // FIXME: Using host floats.
 276                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 277                 let i = this.read_scalar(i)?.to_i32()?;
 278                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 279             }
 280
 281             "powif64" => {
 282                 let &[ref f, ref i] = check_arg_count(args)?;
 283                 // FIXME: Using host floats.
 284                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 285                 let i = this.read_scalar(i)?.to_i32()?;
 286                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 287             }
 288
 289             "float_to_int_unchecked" => {
 290                 let &[ref val] = check_arg_count(args)?;
 291                 let val = this.read_immediate(val)?;
 292
 293                 let res = match val.layout.ty.kind() {
 294                     ty::Float(FloatTy::F32) =>
 295                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 296                     ty::Float(FloatTy::F64) =>
 297                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 298                     _ =>
 299                         bug!(
 300                             "`float_to_int_unchecked` called with non-float input type {:?}",
 301                             val.layout.ty
 302                         ),
 303                 };
 304
 305                 this.write_scalar(res, dest)?;
 306             }
 307
 308             // SIMD operations
 309             "simd_add" | "simd_sub" | "simd_mul" | "simd_div" => {
 310                 let &[ref left, ref right] = check_arg_count(args)?;
 311                 let (left, left_len) = this.operand_to_simd(left)?;
 312                 let (right, right_len) = this.operand_to_simd(right)?;
 313                 let (dest, dest_len) = this.place_to_simd(dest)?;
 314
 315                 assert_eq!(dest_len, left_len);
 316                 assert_eq!(dest_len, right_len);
 317
 318                 let op = match intrinsic_name {
 319                     "simd_add" => mir::BinOp::Add,
 320                     "simd_sub" => mir::BinOp::Sub,
 321                     "simd_mul" => mir::BinOp::Mul,
 322                     "simd_div" => mir::BinOp::Div,
 323                     _ => unreachable!(),
 324                 };
 325
 326                 for i in 0..dest_len {
 327                     let left = this.read_immediate(&this.mplace_index(&left, i)?.into())?;
 328                     let right = this.read_immediate(&this.mplace_index(&right, i)?.into())?;
 329                     let dest = this.mplace_index(&dest, i)?.into();
 330                     this.binop_ignore_overflow(op, &left, &right, &dest)?;
 331                 }
 332             }
 333
 334             // Atomic operations
 335             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
 336             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
 337             "atomic_load_acq" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
 338
 339             "atomic_store" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
 340             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
 341             "atomic_store_rel" => this.atomic_store(args, AtomicWriteOp::Release)?,
 342
 343             "atomic_fence_acq" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
 344             "atomic_fence_rel" => this.atomic_fence(args, AtomicFenceOp::Release)?,
 345             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
 346             "atomic_fence" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
 347
 348             "atomic_singlethreadfence_acq" => this.compiler_fence(args, AtomicFenceOp::Acquire)?,
 349             "atomic_singlethreadfence_rel" => this.compiler_fence(args, AtomicFenceOp::Release)?,
 350             "atomic_singlethreadfence_acqrel" =>
 351                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
 352             "atomic_singlethreadfence" => this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
 353
 354             "atomic_xchg" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
 355             "atomic_xchg_acq" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
 356             "atomic_xchg_rel" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
 357             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
 358             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
 359
 360             #[rustfmt::skip]
 361             "atomic_cxchg" =>
 362                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 363             #[rustfmt::skip]
 364             "atomic_cxchg_acq" =>
 365                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 366             #[rustfmt::skip]
 367             "atomic_cxchg_rel" =>
 368                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 369             #[rustfmt::skip]
 370             "atomic_cxchg_acqrel" =>
 371                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 372             #[rustfmt::skip]
 373             "atomic_cxchg_relaxed" =>
 374                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 375             #[rustfmt::skip]
 376             "atomic_cxchg_acq_failrelaxed" =>
 377                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 378             #[rustfmt::skip]
 379             "atomic_cxchg_acqrel_failrelaxed" =>
 380                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 381             #[rustfmt::skip]
 382             "atomic_cxchg_failrelaxed" =>
 383                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 384             #[rustfmt::skip]
 385             "atomic_cxchg_failacq" =>
 386                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 387
 388             #[rustfmt::skip]
 389             "atomic_cxchgweak" =>
 390                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 391             #[rustfmt::skip]
 392             "atomic_cxchgweak_acq" =>
 393                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 394             #[rustfmt::skip]
 395             "atomic_cxchgweak_rel" =>
 396                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 397             #[rustfmt::skip]
 398             "atomic_cxchgweak_acqrel" =>
 399                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 400             #[rustfmt::skip]
 401             "atomic_cxchgweak_relaxed" =>
 402                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 403             #[rustfmt::skip]
 404             "atomic_cxchgweak_acq_failrelaxed" =>
 405                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 406             #[rustfmt::skip]
 407             "atomic_cxchgweak_acqrel_failrelaxed" =>
 408                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 409             #[rustfmt::skip]
 410             "atomic_cxchgweak_failrelaxed" =>
 411                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 412             #[rustfmt::skip]
 413             "atomic_cxchgweak_failacq" =>
 414                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 415
 416             #[rustfmt::skip]
 417             "atomic_or" =>
 418                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::SeqCst)?,
 419             #[rustfmt::skip]
 420             "atomic_or_acq" =>
 421                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Acquire)?,
 422             #[rustfmt::skip]
 423             "atomic_or_rel" =>
 424                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Release)?,
 425             #[rustfmt::skip]
 426             "atomic_or_acqrel" =>
 427                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::AcqRel)?,
 428             #[rustfmt::skip]
 429             "atomic_or_relaxed" =>
 430                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Relaxed)?,
 431             #[rustfmt::skip]
 432             "atomic_xor" =>
 433                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::SeqCst)?,
 434             #[rustfmt::skip]
 435             "atomic_xor_acq" =>
 436                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Acquire)?,
 437             #[rustfmt::skip]
 438             "atomic_xor_rel" =>
 439                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Release)?,
 440             #[rustfmt::skip]
 441             "atomic_xor_acqrel" =>
 442                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::AcqRel)?,
 443             #[rustfmt::skip]
 444             "atomic_xor_relaxed" =>
 445                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Relaxed)?,
 446             #[rustfmt::skip]
 447             "atomic_and" =>
 448                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::SeqCst)?,
 449             #[rustfmt::skip]
 450             "atomic_and_acq" =>
 451                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Acquire)?,
 452             #[rustfmt::skip]
 453             "atomic_and_rel" =>
 454                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Release)?,
 455             #[rustfmt::skip]
 456             "atomic_and_acqrel" =>
 457                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::AcqRel)?,
 458             #[rustfmt::skip]
 459             "atomic_and_relaxed" =>
 460                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Relaxed)?,
 461             #[rustfmt::skip]
 462             "atomic_nand" =>
 463                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::SeqCst)?,
 464             #[rustfmt::skip]
 465             "atomic_nand_acq" =>
 466                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Acquire)?,
 467             #[rustfmt::skip]
 468             "atomic_nand_rel" =>
 469                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Release)?,
 470             #[rustfmt::skip]
 471             "atomic_nand_acqrel" =>
 472                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::AcqRel)?,
 473             #[rustfmt::skip]
 474             "atomic_nand_relaxed" =>
 475                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Relaxed)?,
 476             #[rustfmt::skip]
 477             "atomic_xadd" =>
 478                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::SeqCst)?,
 479             #[rustfmt::skip]
 480             "atomic_xadd_acq" =>
 481                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Acquire)?,
 482             #[rustfmt::skip]
 483             "atomic_xadd_rel" =>
 484                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Release)?,
 485             #[rustfmt::skip]
 486             "atomic_xadd_acqrel" =>
 487                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::AcqRel)?,
 488             #[rustfmt::skip]
 489             "atomic_xadd_relaxed" =>
 490                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Relaxed)?,
 491             #[rustfmt::skip]
 492             "atomic_xsub" =>
 493                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::SeqCst)?,
 494             #[rustfmt::skip]
 495             "atomic_xsub_acq" =>
 496                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Acquire)?,
 497             #[rustfmt::skip]
 498             "atomic_xsub_rel" =>
 499                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Release)?,
 500             #[rustfmt::skip]
 501             "atomic_xsub_acqrel" =>
 502                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::AcqRel)?,
 503             #[rustfmt::skip]
 504             "atomic_xsub_relaxed" =>
 505                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Relaxed)?,
 506             "atomic_min" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 507             "atomic_min_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 508             "atomic_min_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 509             "atomic_min_acqrel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 510             "atomic_min_relaxed" =>
 511                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 512             "atomic_max" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 513             "atomic_max_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 514             "atomic_max_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 515             "atomic_max_acqrel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 516             "atomic_max_relaxed" =>
 517                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 518             "atomic_umin" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 519             "atomic_umin_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 520             "atomic_umin_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 521             "atomic_umin_acqrel" =>
 522                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 523             "atomic_umin_relaxed" =>
 524                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 525             "atomic_umax" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 526             "atomic_umax_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 527             "atomic_umax_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 528             "atomic_umax_acqrel" =>
 529                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 530             "atomic_umax_relaxed" =>
 531                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 532
 533             // Query type information
 534             "assert_zero_valid" | "assert_uninit_valid" => {
 535                 let &[] = check_arg_count(args)?;
 536                 let ty = instance.substs.type_at(0);
 537                 let layout = this.layout_of(ty)?;
 538                 // Abort here because the caller might not be panic safe.
 539                 if layout.abi.is_uninhabited() {
 540                     // Use this message even for the other intrinsics, as that's what codegen does
 541                     throw_machine_stop!(TerminationInfo::Abort(format!(
 542                         "aborted execution: attempted to instantiate uninhabited type `{}`",
 543                         ty
 544                     )))
 545                 }
 546                 if intrinsic_name == "assert_zero_valid"
 547                     && !layout.might_permit_raw_init(this, /*zero:*/ true)
 548                 {
 549                     throw_machine_stop!(TerminationInfo::Abort(format!(
 550                         "aborted execution: attempted to zero-initialize type `{}`, which is invalid",
 551                         ty
 552                     )))
 553                 }
 554                 if intrinsic_name == "assert_uninit_valid"
 555                     && !layout.might_permit_raw_init(this, /*zero:*/ false)
 556                 {
 557                     throw_machine_stop!(TerminationInfo::Abort(format!(
 558                         "aborted execution: attempted to leave type `{}` uninitialized, which is invalid",
 559                         ty
 560                     )))
 561                 }
 562             }
 563
 564             // Other
 565             "exact_div" => {
 566                 let &[ref num, ref denom] = check_arg_count(args)?;
 567                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
 568             }
 569
 570             "try" => return this.handle_try(args, dest, ret),
 571
 572             "breakpoint" => {
 573                 let &[] = check_arg_count(args)?;
 574                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
 575                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
 576             }
 577
 578             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
 579         }
 580
 581         trace!("{:?}", this.dump_place(**dest));
 582         this.go_to_block(ret);
 583         Ok(())
 584     }
 585
 586     fn atomic_load(
 587         &mut self,
 588         args: &[OpTy<'tcx, Tag>],
 589         dest: &PlaceTy<'tcx, Tag>,
 590         atomic: AtomicReadOp,
 591     ) -> InterpResult<'tcx> {
 592         let this = self.eval_context_mut();
 593
 594         let &[ref place] = check_arg_count(args)?;
 595         let place = this.deref_operand(place)?;
 596
 597         // make sure it fits into a scalar; otherwise it cannot be atomic
 598         let val = this.read_scalar_atomic(&place, atomic)?;
 599
 600         // Check alignment requirements. Atomics must always be aligned to their size,
 601         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 602         // be 8-aligned).
 603         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 604         this.memory.check_ptr_access_align(
 605             place.ptr,
 606             place.layout.size,
 607             align,
 608             CheckInAllocMsg::MemoryAccessTest,
 609         )?;
 610         // Perform regular access.
 611         this.write_scalar(val, dest)?;
 612         Ok(())
 613     }
 614
 615     fn atomic_store(
 616         &mut self,
 617         args: &[OpTy<'tcx, Tag>],
 618         atomic: AtomicWriteOp,
 619     ) -> InterpResult<'tcx> {
 620         let this = self.eval_context_mut();
 621
 622         let &[ref place, ref val] = check_arg_count(args)?;
 623         let place = this.deref_operand(place)?;
 624         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
 625
 626         // Check alignment requirements. Atomics must always be aligned to their size,
 627         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 628         // be 8-aligned).
 629         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 630         this.memory.check_ptr_access_align(
 631             place.ptr,
 632             place.layout.size,
 633             align,
 634             CheckInAllocMsg::MemoryAccessTest,
 635         )?;
 636
 637         // Perform atomic store
 638         this.write_scalar_atomic(val, &place, atomic)?;
 639         Ok(())
 640     }
 641
 642     fn compiler_fence(
 643         &mut self,
 644         args: &[OpTy<'tcx, Tag>],
 645         atomic: AtomicFenceOp,
 646     ) -> InterpResult<'tcx> {
 647         let &[] = check_arg_count(args)?;
 648         let _ = atomic;
 649         //FIXME: compiler fences are currently ignored
 650         Ok(())
 651     }
 652
 653     fn atomic_fence(
 654         &mut self,
 655         args: &[OpTy<'tcx, Tag>],
 656         atomic: AtomicFenceOp,
 657     ) -> InterpResult<'tcx> {
 658         let this = self.eval_context_mut();
 659         let &[] = check_arg_count(args)?;
 660         this.validate_atomic_fence(atomic)?;
 661         Ok(())
 662     }
 663
 664     fn atomic_op(
 665         &mut self,
 666         args: &[OpTy<'tcx, Tag>],
 667         dest: &PlaceTy<'tcx, Tag>,
 668         atomic_op: AtomicOp,
 669         atomic: AtomicRwOp,
 670     ) -> InterpResult<'tcx> {
 671         let this = self.eval_context_mut();
 672
 673         let &[ref place, ref rhs] = check_arg_count(args)?;
 674         let place = this.deref_operand(place)?;
 675
 676         if !place.layout.ty.is_integral() {
 677             bug!("Atomic arithmetic operations only work on integer types");
 678         }
 679         let rhs = this.read_immediate(rhs)?;
 680
 681         // Check alignment requirements. Atomics must always be aligned to their size,
 682         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 683         // be 8-aligned).
 684         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 685         this.memory.check_ptr_access_align(
 686             place.ptr,
 687             place.layout.size,
 688             align,
 689             CheckInAllocMsg::MemoryAccessTest,
 690         )?;
 691
 692         match atomic_op {
 693             AtomicOp::Min => {
 694                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
 695                 this.write_immediate(*old, &dest)?; // old value is returned
 696                 Ok(())
 697             }
 698             AtomicOp::Max => {
 699                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
 700                 this.write_immediate(*old, &dest)?; // old value is returned
 701                 Ok(())
 702             }
 703             AtomicOp::MirOp(op, neg) => {
 704                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
 705                 this.write_immediate(*old, dest)?; // old value is returned
 706                 Ok(())
 707             }
 708         }
 709     }
 710
 711     fn atomic_exchange(
 712         &mut self,
 713         args: &[OpTy<'tcx, Tag>],
 714         dest: &PlaceTy<'tcx, Tag>,
 715         atomic: AtomicRwOp,
 716     ) -> InterpResult<'tcx> {
 717         let this = self.eval_context_mut();
 718
 719         let &[ref place, ref new] = check_arg_count(args)?;
 720         let place = this.deref_operand(place)?;
 721         let new = this.read_scalar(new)?;
 722
 723         // Check alignment requirements. Atomics must always be aligned to their size,
 724         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 725         // be 8-aligned).
 726         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 727         this.memory.check_ptr_access_align(
 728             place.ptr,
 729             place.layout.size,
 730             align,
 731             CheckInAllocMsg::MemoryAccessTest,
 732         )?;
 733
 734         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
 735         this.write_scalar(old, dest)?; // old value is returned
 736         Ok(())
 737     }
 738
 739     fn atomic_compare_exchange_impl(
 740         &mut self,
 741         args: &[OpTy<'tcx, Tag>],
 742         dest: &PlaceTy<'tcx, Tag>,
 743         success: AtomicRwOp,
 744         fail: AtomicReadOp,
 745         can_fail_spuriously: bool,
 746     ) -> InterpResult<'tcx> {
 747         let this = self.eval_context_mut();
 748
 749         let &[ref place, ref expect_old, ref new] = check_arg_count(args)?;
 750         let place = this.deref_operand(place)?;
 751         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
 752         let new = this.read_scalar(new)?;
 753
 754         // Check alignment requirements. Atomics must always be aligned to their size,
 755         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 756         // be 8-aligned).
 757         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 758         this.memory.check_ptr_access_align(
 759             place.ptr,
 760             place.layout.size,
 761             align,
 762             CheckInAllocMsg::MemoryAccessTest,
 763         )?;
 764
 765         let old = this.atomic_compare_exchange_scalar(
 766             &place,
 767             &expect_old,
 768             new,
 769             success,
 770             fail,
 771             can_fail_spuriously,
 772         )?;
 773
 774         // Return old value.
 775         this.write_immediate(old, dest)?;
 776         Ok(())
 777     }
 778
 779     fn atomic_compare_exchange(
 780         &mut self,
 781         args: &[OpTy<'tcx, Tag>],
 782         dest: &PlaceTy<'tcx, Tag>,
 783         success: AtomicRwOp,
 784         fail: AtomicReadOp,
 785     ) -> InterpResult<'tcx> {
 786         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
 787     }
 788
 789     fn atomic_compare_exchange_weak(
 790         &mut self,
 791         args: &[OpTy<'tcx, Tag>],
 792         dest: &PlaceTy<'tcx, Tag>,
 793         success: AtomicRwOp,
 794         fail: AtomicReadOp,
 795     ) -> InterpResult<'tcx> {
 796         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
 797     }
 798
 799     fn float_to_int_unchecked<F>(
 800         &self,
 801         f: F,
 802         dest_ty: ty::Ty<'tcx>,
 803     ) -> InterpResult<'tcx, Scalar<Tag>>
 804     where
 805         F: Float + Into<Scalar<Tag>>,
 806     {
 807         let this = self.eval_context_ref();
 808
 809         // Step 1: cut off the fractional part of `f`. The result of this is
 810         // guaranteed to be precisely representable in IEEE floats.
 811         let f = f.round_to_integral(Round::TowardZero).value;
 812
 813         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
 814         Ok(match dest_ty.kind() {
 815             // Unsigned
 816             ty::Uint(t) => {
 817                 let size = Integer::from_uint_ty(this, *t).size();
 818                 let res = f.to_u128(size.bits_usize());
 819                 if res.status.is_empty() {
 820                     // No status flags means there was no further rounding or other loss of precision.
 821                     Scalar::from_uint(res.value, size)
 822                 } else {
 823                     // `f` was not representable in this integer type.
 824                     throw_ub_format!(
 825                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 826                         f,
 827                         dest_ty,
 828                     );
 829                 }
 830             }
 831             // Signed
 832             ty::Int(t) => {
 833                 let size = Integer::from_int_ty(this, *t).size();
 834                 let res = f.to_i128(size.bits_usize());
 835                 if res.status.is_empty() {
 836                     // No status flags means there was no further rounding or other loss of precision.
 837                     Scalar::from_int(res.value, size)
 838                 } else {
 839                     // `f` was not representable in this integer type.
 840                     throw_ub_format!(
 841                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 842                         f,
 843                         dest_ty,
 844                     );
 845                 }
 846             }
 847             // Nothing else
 848             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
 849         })
 850     }
 851 }