src/shims/intrinsics.rs

   1 use std::iter;
   2
   3 use log::trace;
   4
   5 use rustc_apfloat::{Float, Round};
   6 use rustc_middle::ty::layout::{IntegerExt, LayoutOf};
   7 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
   8 use rustc_target::abi::{Align, Integer};
   9
  10 use crate::*;
  11 use helpers::check_arg_count;
  12
  13 pub enum AtomicOp {
  14     MirOp(mir::BinOp, bool),
  15     Max,
  16     Min,
  17 }
  18
  19 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  20 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  21     fn call_intrinsic(
  22         &mut self,
  23         instance: ty::Instance<'tcx>,
  24         args: &[OpTy<'tcx, Tag>],
  25         ret: Option<(&PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  26         _unwind: StackPopUnwind,
  27     ) -> InterpResult<'tcx> {
  28         let this = self.eval_context_mut();
  29
  30         if this.emulate_intrinsic(instance, args, ret)? {
  31             return Ok(());
  32         }
  33
  34         // All supported intrinsics have a return place.
  35         let intrinsic_name = &*this.tcx.item_name(instance.def_id()).as_str();
  36         let (dest, ret) = match ret {
  37             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  38             Some(p) => p,
  39         };
  40
  41         // Then handle terminating intrinsics.
  42         match intrinsic_name {
  43             // Miri overwriting CTFE intrinsics.
  44             "ptr_guaranteed_eq" => {
  45                 let &[ref left, ref right] = check_arg_count(args)?;
  46                 let left = this.read_immediate(left)?;
  47                 let right = this.read_immediate(right)?;
  48                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  49             }
  50             "ptr_guaranteed_ne" => {
  51                 let &[ref left, ref right] = check_arg_count(args)?;
  52                 let left = this.read_immediate(left)?;
  53                 let right = this.read_immediate(right)?;
  54                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  55             }
  56
  57             // Raw memory accesses
  58             "volatile_load" => {
  59                 let &[ref place] = check_arg_count(args)?;
  60                 let place = this.deref_operand(place)?;
  61                 this.copy_op(&place.into(), dest)?;
  62             }
  63             "volatile_store" => {
  64                 let &[ref place, ref dest] = check_arg_count(args)?;
  65                 let place = this.deref_operand(place)?;
  66                 this.copy_op(dest, &place.into())?;
  67             }
  68
  69             "write_bytes" | "volatile_set_memory" => {
  70                 let &[ref ptr, ref val_byte, ref count] = check_arg_count(args)?;
  71                 let ty = instance.substs.type_at(0);
  72                 let ty_layout = this.layout_of(ty)?;
  73                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
  74                 let ptr = this.read_pointer(ptr)?;
  75                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  76                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
  77                     err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
  78                 })?;
  79                 this.memory
  80                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
  81             }
  82
  83             // Floating-point operations
  84             #[rustfmt::skip]
  85             | "sinf32"
  86             | "fabsf32"
  87             | "cosf32"
  88             | "sqrtf32"
  89             | "expf32"
  90             | "exp2f32"
  91             | "logf32"
  92             | "log10f32"
  93             | "log2f32"
  94             | "floorf32"
  95             | "ceilf32"
  96             | "truncf32"
  97             | "roundf32"
  98             => {
  99                 let &[ref f] = check_arg_count(args)?;
 100                 // FIXME: Using host floats.
 101                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 102                 let f = match intrinsic_name {
 103                     "sinf32" => f.sin(),
 104                     "fabsf32" => f.abs(),
 105                     "cosf32" => f.cos(),
 106                     "sqrtf32" => f.sqrt(),
 107                     "expf32" => f.exp(),
 108                     "exp2f32" => f.exp2(),
 109                     "logf32" => f.ln(),
 110                     "log10f32" => f.log10(),
 111                     "log2f32" => f.log2(),
 112                     "floorf32" => f.floor(),
 113                     "ceilf32" => f.ceil(),
 114                     "truncf32" => f.trunc(),
 115                     "roundf32" => f.round(),
 116                     _ => bug!(),
 117                 };
 118                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 119             }
 120
 121             #[rustfmt::skip]
 122             | "sinf64"
 123             | "fabsf64"
 124             | "cosf64"
 125             | "sqrtf64"
 126             | "expf64"
 127             | "exp2f64"
 128             | "logf64"
 129             | "log10f64"
 130             | "log2f64"
 131             | "floorf64"
 132             | "ceilf64"
 133             | "truncf64"
 134             | "roundf64"
 135             => {
 136                 let &[ref f] = check_arg_count(args)?;
 137                 // FIXME: Using host floats.
 138                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 139                 let f = match intrinsic_name {
 140                     "sinf64" => f.sin(),
 141                     "fabsf64" => f.abs(),
 142                     "cosf64" => f.cos(),
 143                     "sqrtf64" => f.sqrt(),
 144                     "expf64" => f.exp(),
 145                     "exp2f64" => f.exp2(),
 146                     "logf64" => f.ln(),
 147                     "log10f64" => f.log10(),
 148                     "log2f64" => f.log2(),
 149                     "floorf64" => f.floor(),
 150                     "ceilf64" => f.ceil(),
 151                     "truncf64" => f.trunc(),
 152                     "roundf64" => f.round(),
 153                     _ => bug!(),
 154                 };
 155                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 156             }
 157
 158             #[rustfmt::skip]
 159             | "fadd_fast"
 160             | "fsub_fast"
 161             | "fmul_fast"
 162             | "fdiv_fast"
 163             | "frem_fast"
 164             => {
 165                 let &[ref a, ref b] = check_arg_count(args)?;
 166                 let a = this.read_immediate(a)?;
 167                 let b = this.read_immediate(b)?;
 168                 let op = match intrinsic_name {
 169                     "fadd_fast" => mir::BinOp::Add,
 170                     "fsub_fast" => mir::BinOp::Sub,
 171                     "fmul_fast" => mir::BinOp::Mul,
 172                     "fdiv_fast" => mir::BinOp::Div,
 173                     "frem_fast" => mir::BinOp::Rem,
 174                     _ => bug!(),
 175                 };
 176                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 177                     Ok(match x.layout.ty.kind() {
 178                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 179                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 180                         _ => bug!(
 181                             "`{}` called with non-float input type {:?}",
 182                             intrinsic_name,
 183                             x.layout.ty
 184                         ),
 185                     })
 186                 };
 187                 match (float_finite(a)?, float_finite(b)?) {
 188                     (false, false) => throw_ub_format!(
 189                         "`{}` intrinsic called with non-finite value as both parameters",
 190                         intrinsic_name,
 191                     ),
 192                     (false, _) => throw_ub_format!(
 193                         "`{}` intrinsic called with non-finite value as first parameter",
 194                         intrinsic_name,
 195                     ),
 196                     (_, false) => throw_ub_format!(
 197                         "`{}` intrinsic called with non-finite value as second parameter",
 198                         intrinsic_name,
 199                     ),
 200                     _ => {}
 201                 }
 202                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 203             }
 204
 205             #[rustfmt::skip]
 206             | "minnumf32"
 207             | "maxnumf32"
 208             | "copysignf32"
 209             => {
 210                 let &[ref a, ref b] = check_arg_count(args)?;
 211                 let a = this.read_scalar(a)?.to_f32()?;
 212                 let b = this.read_scalar(b)?.to_f32()?;
 213                 let res = match intrinsic_name {
 214                     "minnumf32" => a.min(b),
 215                     "maxnumf32" => a.max(b),
 216                     "copysignf32" => a.copy_sign(b),
 217                     _ => bug!(),
 218                 };
 219                 this.write_scalar(Scalar::from_f32(res), dest)?;
 220             }
 221
 222             #[rustfmt::skip]
 223             | "minnumf64"
 224             | "maxnumf64"
 225             | "copysignf64"
 226             => {
 227                 let &[ref a, ref b] = check_arg_count(args)?;
 228                 let a = this.read_scalar(a)?.to_f64()?;
 229                 let b = this.read_scalar(b)?.to_f64()?;
 230                 let res = match intrinsic_name {
 231                     "minnumf64" => a.min(b),
 232                     "maxnumf64" => a.max(b),
 233                     "copysignf64" => a.copy_sign(b),
 234                     _ => bug!(),
 235                 };
 236                 this.write_scalar(Scalar::from_f64(res), dest)?;
 237             }
 238
 239             "powf32" => {
 240                 let &[ref f, ref f2] = check_arg_count(args)?;
 241                 // FIXME: Using host floats.
 242                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 243                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 244                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 245             }
 246
 247             "powf64" => {
 248                 let &[ref f, ref f2] = check_arg_count(args)?;
 249                 // FIXME: Using host floats.
 250                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 251                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 252                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 253             }
 254
 255             "fmaf32" => {
 256                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 257                 let a = this.read_scalar(a)?.to_f32()?;
 258                 let b = this.read_scalar(b)?.to_f32()?;
 259                 let c = this.read_scalar(c)?.to_f32()?;
 260                 let res = a.mul_add(b, c).value;
 261                 this.write_scalar(Scalar::from_f32(res), dest)?;
 262             }
 263
 264             "fmaf64" => {
 265                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 266                 let a = this.read_scalar(a)?.to_f64()?;
 267                 let b = this.read_scalar(b)?.to_f64()?;
 268                 let c = this.read_scalar(c)?.to_f64()?;
 269                 let res = a.mul_add(b, c).value;
 270                 this.write_scalar(Scalar::from_f64(res), dest)?;
 271             }
 272
 273             "powif32" => {
 274                 let &[ref f, ref i] = check_arg_count(args)?;
 275                 // FIXME: Using host floats.
 276                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 277                 let i = this.read_scalar(i)?.to_i32()?;
 278                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 279             }
 280
 281             "powif64" => {
 282                 let &[ref f, ref i] = check_arg_count(args)?;
 283                 // FIXME: Using host floats.
 284                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 285                 let i = this.read_scalar(i)?.to_i32()?;
 286                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 287             }
 288
 289             "float_to_int_unchecked" => {
 290                 let &[ref val] = check_arg_count(args)?;
 291                 let val = this.read_immediate(val)?;
 292
 293                 let res = match val.layout.ty.kind() {
 294                     ty::Float(FloatTy::F32) =>
 295                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 296                     ty::Float(FloatTy::F64) =>
 297                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 298                     _ =>
 299                         bug!(
 300                             "`float_to_int_unchecked` called with non-float input type {:?}",
 301                             val.layout.ty
 302                         ),
 303                 };
 304
 305                 this.write_scalar(res, dest)?;
 306             }
 307
 308             // Atomic operations
 309             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
 310             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
 311             "atomic_load_acq" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
 312
 313             "atomic_store" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
 314             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
 315             "atomic_store_rel" => this.atomic_store(args, AtomicWriteOp::Release)?,
 316
 317             "atomic_fence_acq" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
 318             "atomic_fence_rel" => this.atomic_fence(args, AtomicFenceOp::Release)?,
 319             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
 320             "atomic_fence" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
 321
 322             "atomic_singlethreadfence_acq" => this.compiler_fence(args, AtomicFenceOp::Acquire)?,
 323             "atomic_singlethreadfence_rel" => this.compiler_fence(args, AtomicFenceOp::Release)?,
 324             "atomic_singlethreadfence_acqrel" =>
 325                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
 326             "atomic_singlethreadfence" => this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
 327
 328             "atomic_xchg" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
 329             "atomic_xchg_acq" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
 330             "atomic_xchg_rel" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
 331             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
 332             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
 333
 334             #[rustfmt::skip]
 335             "atomic_cxchg" =>
 336                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 337             #[rustfmt::skip]
 338             "atomic_cxchg_acq" =>
 339                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 340             #[rustfmt::skip]
 341             "atomic_cxchg_rel" =>
 342                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 343             #[rustfmt::skip]
 344             "atomic_cxchg_acqrel" =>
 345                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 346             #[rustfmt::skip]
 347             "atomic_cxchg_relaxed" =>
 348                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 349             #[rustfmt::skip]
 350             "atomic_cxchg_acq_failrelaxed" =>
 351                 this.atomic_compare_exchange(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 352             #[rustfmt::skip]
 353             "atomic_cxchg_acqrel_failrelaxed" =>
 354                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 355             #[rustfmt::skip]
 356             "atomic_cxchg_failrelaxed" =>
 357                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 358             #[rustfmt::skip]
 359             "atomic_cxchg_failacq" =>
 360                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 361
 362             #[rustfmt::skip]
 363             "atomic_cxchgweak" =>
 364                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 365             #[rustfmt::skip]
 366             "atomic_cxchgweak_acq" =>
 367                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Acquire)?,
 368             #[rustfmt::skip]
 369             "atomic_cxchgweak_rel" =>
 370                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Release, AtomicReadOp::Relaxed)?,
 371             #[rustfmt::skip]
 372             "atomic_cxchgweak_acqrel" =>
 373                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 374             #[rustfmt::skip]
 375             "atomic_cxchgweak_relaxed" =>
 376                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Relaxed, AtomicReadOp::Relaxed)?,
 377             #[rustfmt::skip]
 378             "atomic_cxchgweak_acq_failrelaxed" =>
 379                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::Acquire, AtomicReadOp::Relaxed)?,
 380             #[rustfmt::skip]
 381             "atomic_cxchgweak_acqrel_failrelaxed" =>
 382                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 383             #[rustfmt::skip]
 384             "atomic_cxchgweak_failrelaxed" =>
 385                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 386             #[rustfmt::skip]
 387             "atomic_cxchgweak_failacq" =>
 388                 this.atomic_compare_exchange_weak(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 389
 390             #[rustfmt::skip]
 391             "atomic_or" =>
 392                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::SeqCst)?,
 393             #[rustfmt::skip]
 394             "atomic_or_acq" =>
 395                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Acquire)?,
 396             #[rustfmt::skip]
 397             "atomic_or_rel" =>
 398                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Release)?,
 399             #[rustfmt::skip]
 400             "atomic_or_acqrel" =>
 401                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::AcqRel)?,
 402             #[rustfmt::skip]
 403             "atomic_or_relaxed" =>
 404                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), AtomicRwOp::Relaxed)?,
 405             #[rustfmt::skip]
 406             "atomic_xor" =>
 407                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::SeqCst)?,
 408             #[rustfmt::skip]
 409             "atomic_xor_acq" =>
 410                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Acquire)?,
 411             #[rustfmt::skip]
 412             "atomic_xor_rel" =>
 413                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Release)?,
 414             #[rustfmt::skip]
 415             "atomic_xor_acqrel" =>
 416                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::AcqRel)?,
 417             #[rustfmt::skip]
 418             "atomic_xor_relaxed" =>
 419                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), AtomicRwOp::Relaxed)?,
 420             #[rustfmt::skip]
 421             "atomic_and" =>
 422                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::SeqCst)?,
 423             #[rustfmt::skip]
 424             "atomic_and_acq" =>
 425                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Acquire)?,
 426             #[rustfmt::skip]
 427             "atomic_and_rel" =>
 428                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Release)?,
 429             #[rustfmt::skip]
 430             "atomic_and_acqrel" =>
 431                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::AcqRel)?,
 432             #[rustfmt::skip]
 433             "atomic_and_relaxed" =>
 434                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), AtomicRwOp::Relaxed)?,
 435             #[rustfmt::skip]
 436             "atomic_nand" =>
 437                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::SeqCst)?,
 438             #[rustfmt::skip]
 439             "atomic_nand_acq" =>
 440                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Acquire)?,
 441             #[rustfmt::skip]
 442             "atomic_nand_rel" =>
 443                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Release)?,
 444             #[rustfmt::skip]
 445             "atomic_nand_acqrel" =>
 446                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::AcqRel)?,
 447             #[rustfmt::skip]
 448             "atomic_nand_relaxed" =>
 449                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), AtomicRwOp::Relaxed)?,
 450             #[rustfmt::skip]
 451             "atomic_xadd" =>
 452                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::SeqCst)?,
 453             #[rustfmt::skip]
 454             "atomic_xadd_acq" =>
 455                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Acquire)?,
 456             #[rustfmt::skip]
 457             "atomic_xadd_rel" =>
 458                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Release)?,
 459             #[rustfmt::skip]
 460             "atomic_xadd_acqrel" =>
 461                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::AcqRel)?,
 462             #[rustfmt::skip]
 463             "atomic_xadd_relaxed" =>
 464                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), AtomicRwOp::Relaxed)?,
 465             #[rustfmt::skip]
 466             "atomic_xsub" =>
 467                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::SeqCst)?,
 468             #[rustfmt::skip]
 469             "atomic_xsub_acq" =>
 470                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Acquire)?,
 471             #[rustfmt::skip]
 472             "atomic_xsub_rel" =>
 473                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Release)?,
 474             #[rustfmt::skip]
 475             "atomic_xsub_acqrel" =>
 476                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::AcqRel)?,
 477             #[rustfmt::skip]
 478             "atomic_xsub_relaxed" =>
 479                 this.atomic_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), AtomicRwOp::Relaxed)?,
 480             "atomic_min" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 481             "atomic_min_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 482             "atomic_min_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 483             "atomic_min_acqrel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 484             "atomic_min_relaxed" =>
 485                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 486             "atomic_max" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 487             "atomic_max_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 488             "atomic_max_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 489             "atomic_max_acqrel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 490             "atomic_max_relaxed" =>
 491                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 492             "atomic_umin" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 493             "atomic_umin_acq" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 494             "atomic_umin_rel" => this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 495             "atomic_umin_acqrel" =>
 496                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 497             "atomic_umin_relaxed" =>
 498                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 499             "atomic_umax" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 500             "atomic_umax_acq" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 501             "atomic_umax_rel" => this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 502             "atomic_umax_acqrel" =>
 503                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 504             "atomic_umax_relaxed" =>
 505                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 506
 507             // Query type information
 508             "assert_zero_valid" | "assert_uninit_valid" => {
 509                 let &[] = check_arg_count(args)?;
 510                 let ty = instance.substs.type_at(0);
 511                 let layout = this.layout_of(ty)?;
 512                 // Abort here because the caller might not be panic safe.
 513                 if layout.abi.is_uninhabited() {
 514                     // Use this message even for the other intrinsics, as that's what codegen does
 515                     throw_machine_stop!(TerminationInfo::Abort(format!(
 516                         "aborted execution: attempted to instantiate uninhabited type `{}`",
 517                         ty
 518                     )))
 519                 }
 520                 if intrinsic_name == "assert_zero_valid"
 521                     && !layout.might_permit_raw_init(this, /*zero:*/ true)
 522                 {
 523                     throw_machine_stop!(TerminationInfo::Abort(format!(
 524                         "aborted execution: attempted to zero-initialize type `{}`, which is invalid",
 525                         ty
 526                     )))
 527                 }
 528                 if intrinsic_name == "assert_uninit_valid"
 529                     && !layout.might_permit_raw_init(this, /*zero:*/ false)
 530                 {
 531                     throw_machine_stop!(TerminationInfo::Abort(format!(
 532                         "aborted execution: attempted to leave type `{}` uninitialized, which is invalid",
 533                         ty
 534                     )))
 535                 }
 536             }
 537
 538             // Other
 539             "exact_div" => {
 540                 let &[ref num, ref denom] = check_arg_count(args)?;
 541                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
 542             }
 543
 544             "try" => return this.handle_try(args, dest, ret),
 545
 546             "breakpoint" => {
 547                 let &[] = check_arg_count(args)?;
 548                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
 549                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
 550             }
 551
 552             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
 553         }
 554
 555         trace!("{:?}", this.dump_place(**dest));
 556         this.go_to_block(ret);
 557         Ok(())
 558     }
 559
 560     fn atomic_load(
 561         &mut self,
 562         args: &[OpTy<'tcx, Tag>],
 563         dest: &PlaceTy<'tcx, Tag>,
 564         atomic: AtomicReadOp,
 565     ) -> InterpResult<'tcx> {
 566         let this = self.eval_context_mut();
 567
 568         let &[ref place] = check_arg_count(args)?;
 569         let place = this.deref_operand(place)?;
 570
 571         // make sure it fits into a scalar; otherwise it cannot be atomic
 572         let val = this.read_scalar_atomic(&place, atomic)?;
 573
 574         // Check alignment requirements. Atomics must always be aligned to their size,
 575         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 576         // be 8-aligned).
 577         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 578         this.memory.check_ptr_access_align(
 579             place.ptr,
 580             place.layout.size,
 581             align,
 582             CheckInAllocMsg::MemoryAccessTest,
 583         )?;
 584         // Perform regular access.
 585         this.write_scalar(val, dest)?;
 586         Ok(())
 587     }
 588
 589     fn atomic_store(
 590         &mut self,
 591         args: &[OpTy<'tcx, Tag>],
 592         atomic: AtomicWriteOp,
 593     ) -> InterpResult<'tcx> {
 594         let this = self.eval_context_mut();
 595
 596         let &[ref place, ref val] = check_arg_count(args)?;
 597         let place = this.deref_operand(place)?;
 598         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
 599
 600         // Check alignment requirements. Atomics must always be aligned to their size,
 601         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 602         // be 8-aligned).
 603         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 604         this.memory.check_ptr_access_align(
 605             place.ptr,
 606             place.layout.size,
 607             align,
 608             CheckInAllocMsg::MemoryAccessTest,
 609         )?;
 610
 611         // Perform atomic store
 612         this.write_scalar_atomic(val, &place, atomic)?;
 613         Ok(())
 614     }
 615
 616     fn compiler_fence(
 617         &mut self,
 618         args: &[OpTy<'tcx, Tag>],
 619         atomic: AtomicFenceOp,
 620     ) -> InterpResult<'tcx> {
 621         let &[] = check_arg_count(args)?;
 622         let _ = atomic;
 623         //FIXME: compiler fences are currently ignored
 624         Ok(())
 625     }
 626
 627     fn atomic_fence(
 628         &mut self,
 629         args: &[OpTy<'tcx, Tag>],
 630         atomic: AtomicFenceOp,
 631     ) -> InterpResult<'tcx> {
 632         let this = self.eval_context_mut();
 633         let &[] = check_arg_count(args)?;
 634         this.validate_atomic_fence(atomic)?;
 635         Ok(())
 636     }
 637
 638     fn atomic_op(
 639         &mut self,
 640         args: &[OpTy<'tcx, Tag>],
 641         dest: &PlaceTy<'tcx, Tag>,
 642         atomic_op: AtomicOp,
 643         atomic: AtomicRwOp,
 644     ) -> InterpResult<'tcx> {
 645         let this = self.eval_context_mut();
 646
 647         let &[ref place, ref rhs] = check_arg_count(args)?;
 648         let place = this.deref_operand(place)?;
 649
 650         if !place.layout.ty.is_integral() {
 651             bug!("Atomic arithmetic operations only work on integer types");
 652         }
 653         let rhs = this.read_immediate(rhs)?;
 654
 655         // Check alignment requirements. Atomics must always be aligned to their size,
 656         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 657         // be 8-aligned).
 658         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 659         this.memory.check_ptr_access_align(
 660             place.ptr,
 661             place.layout.size,
 662             align,
 663             CheckInAllocMsg::MemoryAccessTest,
 664         )?;
 665
 666         match atomic_op {
 667             AtomicOp::Min => {
 668                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
 669                 this.write_immediate(*old, &dest)?; // old value is returned
 670                 Ok(())
 671             }
 672             AtomicOp::Max => {
 673                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
 674                 this.write_immediate(*old, &dest)?; // old value is returned
 675                 Ok(())
 676             }
 677             AtomicOp::MirOp(op, neg) => {
 678                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
 679                 this.write_immediate(*old, dest)?; // old value is returned
 680                 Ok(())
 681             }
 682         }
 683     }
 684
 685     fn atomic_exchange(
 686         &mut self,
 687         args: &[OpTy<'tcx, Tag>],
 688         dest: &PlaceTy<'tcx, Tag>,
 689         atomic: AtomicRwOp,
 690     ) -> InterpResult<'tcx> {
 691         let this = self.eval_context_mut();
 692
 693         let &[ref place, ref new] = check_arg_count(args)?;
 694         let place = this.deref_operand(place)?;
 695         let new = this.read_scalar(new)?;
 696
 697         // Check alignment requirements. Atomics must always be aligned to their size,
 698         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 699         // be 8-aligned).
 700         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 701         this.memory.check_ptr_access_align(
 702             place.ptr,
 703             place.layout.size,
 704             align,
 705             CheckInAllocMsg::MemoryAccessTest,
 706         )?;
 707
 708         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
 709         this.write_scalar(old, dest)?; // old value is returned
 710         Ok(())
 711     }
 712
 713     fn atomic_compare_exchange_impl(
 714         &mut self,
 715         args: &[OpTy<'tcx, Tag>],
 716         dest: &PlaceTy<'tcx, Tag>,
 717         success: AtomicRwOp,
 718         fail: AtomicReadOp,
 719         can_fail_spuriously: bool,
 720     ) -> InterpResult<'tcx> {
 721         let this = self.eval_context_mut();
 722
 723         let &[ref place, ref expect_old, ref new] = check_arg_count(args)?;
 724         let place = this.deref_operand(place)?;
 725         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
 726         let new = this.read_scalar(new)?;
 727
 728         // Check alignment requirements. Atomics must always be aligned to their size,
 729         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 730         // be 8-aligned).
 731         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 732         this.memory.check_ptr_access_align(
 733             place.ptr,
 734             place.layout.size,
 735             align,
 736             CheckInAllocMsg::MemoryAccessTest,
 737         )?;
 738
 739         let old = this.atomic_compare_exchange_scalar(
 740             &place,
 741             &expect_old,
 742             new,
 743             success,
 744             fail,
 745             can_fail_spuriously,
 746         )?;
 747
 748         // Return old value.
 749         this.write_immediate(old, dest)?;
 750         Ok(())
 751     }
 752
 753     fn atomic_compare_exchange(
 754         &mut self,
 755         args: &[OpTy<'tcx, Tag>],
 756         dest: &PlaceTy<'tcx, Tag>,
 757         success: AtomicRwOp,
 758         fail: AtomicReadOp,
 759     ) -> InterpResult<'tcx> {
 760         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
 761     }
 762
 763     fn atomic_compare_exchange_weak(
 764         &mut self,
 765         args: &[OpTy<'tcx, Tag>],
 766         dest: &PlaceTy<'tcx, Tag>,
 767         success: AtomicRwOp,
 768         fail: AtomicReadOp,
 769     ) -> InterpResult<'tcx> {
 770         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
 771     }
 772
 773     fn float_to_int_unchecked<F>(
 774         &self,
 775         f: F,
 776         dest_ty: ty::Ty<'tcx>,
 777     ) -> InterpResult<'tcx, Scalar<Tag>>
 778     where
 779         F: Float + Into<Scalar<Tag>>,
 780     {
 781         let this = self.eval_context_ref();
 782
 783         // Step 1: cut off the fractional part of `f`. The result of this is
 784         // guaranteed to be precisely representable in IEEE floats.
 785         let f = f.round_to_integral(Round::TowardZero).value;
 786
 787         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
 788         Ok(match dest_ty.kind() {
 789             // Unsigned
 790             ty::Uint(t) => {
 791                 let size = Integer::from_uint_ty(this, *t).size();
 792                 let res = f.to_u128(size.bits_usize());
 793                 if res.status.is_empty() {
 794                     // No status flags means there was no further rounding or other loss of precision.
 795                     Scalar::from_uint(res.value, size)
 796                 } else {
 797                     // `f` was not representable in this integer type.
 798                     throw_ub_format!(
 799                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 800                         f,
 801                         dest_ty,
 802                     );
 803                 }
 804             }
 805             // Signed
 806             ty::Int(t) => {
 807                 let size = Integer::from_int_ty(this, *t).size();
 808                 let res = f.to_i128(size.bits_usize());
 809                 if res.status.is_empty() {
 810                     // No status flags means there was no further rounding or other loss of precision.
 811                     Scalar::from_int(res.value, size)
 812                 } else {
 813                     // `f` was not representable in this integer type.
 814                     throw_ub_format!(
 815                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 816                         f,
 817                         dest_ty,
 818                     );
 819                 }
 820             }
 821             // Nothing else
 822             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
 823         })
 824     }
 825 }