src/shims/intrinsics.rs

   1 use std::iter;
   2
   3 use log::trace;
   4
   5 use rustc_apfloat::{Float, Round};
   6 use rustc_middle::ty::layout::IntegerExt;
   7 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
   8 use rustc_target::abi::{Align, Integer, LayoutOf};
   9
  10 use crate::*;
  11 use helpers::check_arg_count;
  12
  13 pub enum AtomicOp {
  14     MirOp(mir::BinOp, bool),
  15     Max,
  16     Min,
  17 }
  18
  19 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  20 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  21     fn call_intrinsic(
  22         &mut self,
  23         instance: ty::Instance<'tcx>,
  24         args: &[OpTy<'tcx, Tag>],
  25         ret: Option<(&PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  26         _unwind: Option<mir::BasicBlock>,
  27     ) -> InterpResult<'tcx> {
  28         let this = self.eval_context_mut();
  29
  30         if this.emulate_intrinsic(instance, args, ret)? {
  31             return Ok(());
  32         }
  33
  34         // All supported intrinsics have a return place.
  35         let intrinsic_name = &*this.tcx.item_name(instance.def_id()).as_str();
  36         let (dest, ret) = match ret {
  37             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  38             Some(p) => p,
  39         };
  40
  41         // Then handle terminating intrinsics.
  42         match intrinsic_name {
  43             // Miri overwriting CTFE intrinsics.
  44             "ptr_guaranteed_eq" => {
  45                 let &[ref left, ref right] = check_arg_count(args)?;
  46                 let left = this.read_immediate(left)?;
  47                 let right = this.read_immediate(right)?;
  48                 this.binop_ignore_overflow(mir::BinOp::Eq, &left, &right, dest)?;
  49             }
  50             "ptr_guaranteed_ne" => {
  51                 let &[ref left, ref right] = check_arg_count(args)?;
  52                 let left = this.read_immediate(left)?;
  53                 let right = this.read_immediate(right)?;
  54                 this.binop_ignore_overflow(mir::BinOp::Ne, &left, &right, dest)?;
  55             }
  56
  57             // Raw memory accesses
  58             "volatile_load" => {
  59                 let &[ref place] = check_arg_count(args)?;
  60                 let place = this.deref_operand(place)?;
  61                 this.copy_op(&place.into(), dest)?;
  62             }
  63             "volatile_store" => {
  64                 let &[ref place, ref dest] = check_arg_count(args)?;
  65                 let place = this.deref_operand(place)?;
  66                 this.copy_op(dest, &place.into())?;
  67             }
  68
  69             "write_bytes" => {
  70                 let &[ref ptr, ref val_byte, ref count] = check_arg_count(args)?;
  71                 let ty = instance.substs.type_at(0);
  72                 let ty_layout = this.layout_of(ty)?;
  73                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
  74                 let ptr = this.read_scalar(ptr)?.check_init()?;
  75                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  76                 let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
  77                     err_ub_format!("overflow computing total size of `write_bytes`")
  78                 })?;
  79                 this.memory
  80                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
  81             }
  82
  83             // Floating-point operations
  84             #[rustfmt::skip]
  85             | "sinf32"
  86             | "fabsf32"
  87             | "cosf32"
  88             | "sqrtf32"
  89             | "expf32"
  90             | "exp2f32"
  91             | "logf32"
  92             | "log10f32"
  93             | "log2f32"
  94             | "floorf32"
  95             | "ceilf32"
  96             | "truncf32"
  97             | "roundf32"
  98             => {
  99                 let &[ref f] = check_arg_count(args)?;
 100                 // FIXME: Using host floats.
 101                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 102                 let f = match intrinsic_name {
 103                     "sinf32" => f.sin(),
 104                     "fabsf32" => f.abs(),
 105                     "cosf32" => f.cos(),
 106                     "sqrtf32" => f.sqrt(),
 107                     "expf32" => f.exp(),
 108                     "exp2f32" => f.exp2(),
 109                     "logf32" => f.ln(),
 110                     "log10f32" => f.log10(),
 111                     "log2f32" => f.log2(),
 112                     "floorf32" => f.floor(),
 113                     "ceilf32" => f.ceil(),
 114                     "truncf32" => f.trunc(),
 115                     "roundf32" => f.round(),
 116                     _ => bug!(),
 117                 };
 118                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 119             }
 120
 121             #[rustfmt::skip]
 122             | "sinf64"
 123             | "fabsf64"
 124             | "cosf64"
 125             | "sqrtf64"
 126             | "expf64"
 127             | "exp2f64"
 128             | "logf64"
 129             | "log10f64"
 130             | "log2f64"
 131             | "floorf64"
 132             | "ceilf64"
 133             | "truncf64"
 134             | "roundf64"
 135             => {
 136                 let &[ref f] = check_arg_count(args)?;
 137                 // FIXME: Using host floats.
 138                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 139                 let f = match intrinsic_name {
 140                     "sinf64" => f.sin(),
 141                     "fabsf64" => f.abs(),
 142                     "cosf64" => f.cos(),
 143                     "sqrtf64" => f.sqrt(),
 144                     "expf64" => f.exp(),
 145                     "exp2f64" => f.exp2(),
 146                     "logf64" => f.ln(),
 147                     "log10f64" => f.log10(),
 148                     "log2f64" => f.log2(),
 149                     "floorf64" => f.floor(),
 150                     "ceilf64" => f.ceil(),
 151                     "truncf64" => f.trunc(),
 152                     "roundf64" => f.round(),
 153                     _ => bug!(),
 154                 };
 155                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 156             }
 157
 158             #[rustfmt::skip]
 159             | "fadd_fast"
 160             | "fsub_fast"
 161             | "fmul_fast"
 162             | "fdiv_fast"
 163             | "frem_fast"
 164             => {
 165                 let &[ref a, ref b] = check_arg_count(args)?;
 166                 let a = this.read_immediate(a)?;
 167                 let b = this.read_immediate(b)?;
 168                 let op = match intrinsic_name {
 169                     "fadd_fast" => mir::BinOp::Add,
 170                     "fsub_fast" => mir::BinOp::Sub,
 171                     "fmul_fast" => mir::BinOp::Mul,
 172                     "fdiv_fast" => mir::BinOp::Div,
 173                     "frem_fast" => mir::BinOp::Rem,
 174                     _ => bug!(),
 175                 };
 176                 let float_finite = |x: ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> {
 177                     Ok(match x.layout.ty.kind() {
 178                         ty::Float(FloatTy::F32) => x.to_scalar()?.to_f32()?.is_finite(),
 179                         ty::Float(FloatTy::F64) => x.to_scalar()?.to_f64()?.is_finite(),
 180                         _ => bug!(
 181                             "`{}` called with non-float input type {:?}",
 182                             intrinsic_name,
 183                             x.layout.ty
 184                         ),
 185                     })
 186                 };
 187                 match (float_finite(a)?, float_finite(b)?) {
 188                     (false, false) => throw_ub_format!(
 189                         "`{}` intrinsic called with non-finite value as both parameters",
 190                         intrinsic_name,
 191                     ),
 192                     (false, _) => throw_ub_format!(
 193                         "`{}` intrinsic called with non-finite value as first parameter",
 194                         intrinsic_name,
 195                     ),
 196                     (_, false) => throw_ub_format!(
 197                         "`{}` intrinsic called with non-finite value as second parameter",
 198                         intrinsic_name,
 199                     ),
 200                     _ => {}
 201                 }
 202                 this.binop_ignore_overflow(op, &a, &b, dest)?;
 203             }
 204
 205             #[rustfmt::skip]
 206             | "minnumf32"
 207             | "maxnumf32"
 208             | "copysignf32"
 209             => {
 210                 let &[ref a, ref b] = check_arg_count(args)?;
 211                 let a = this.read_scalar(a)?.to_f32()?;
 212                 let b = this.read_scalar(b)?.to_f32()?;
 213                 let res = match intrinsic_name {
 214                     "minnumf32" => a.min(b),
 215                     "maxnumf32" => a.max(b),
 216                     "copysignf32" => a.copy_sign(b),
 217                     _ => bug!(),
 218                 };
 219                 this.write_scalar(Scalar::from_f32(res), dest)?;
 220             }
 221
 222             #[rustfmt::skip]
 223             | "minnumf64"
 224             | "maxnumf64"
 225             | "copysignf64"
 226             => {
 227                 let &[ref a, ref b] = check_arg_count(args)?;
 228                 let a = this.read_scalar(a)?.to_f64()?;
 229                 let b = this.read_scalar(b)?.to_f64()?;
 230                 let res = match intrinsic_name {
 231                     "minnumf64" => a.min(b),
 232                     "maxnumf64" => a.max(b),
 233                     "copysignf64" => a.copy_sign(b),
 234                     _ => bug!(),
 235                 };
 236                 this.write_scalar(Scalar::from_f64(res), dest)?;
 237             }
 238
 239             "powf32" => {
 240                 let &[ref f, ref f2] = check_arg_count(args)?;
 241                 // FIXME: Using host floats.
 242                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 243                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 244                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 245             }
 246
 247             "powf64" => {
 248                 let &[ref f, ref f2] = check_arg_count(args)?;
 249                 // FIXME: Using host floats.
 250                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 251                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 252                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 253             }
 254
 255             "fmaf32" => {
 256                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 257                 let a = this.read_scalar(a)?.to_f32()?;
 258                 let b = this.read_scalar(b)?.to_f32()?;
 259                 let c = this.read_scalar(c)?.to_f32()?;
 260                 let res = a.mul_add(b, c).value;
 261                 this.write_scalar(Scalar::from_f32(res), dest)?;
 262             }
 263
 264             "fmaf64" => {
 265                 let &[ref a, ref b, ref c] = check_arg_count(args)?;
 266                 let a = this.read_scalar(a)?.to_f64()?;
 267                 let b = this.read_scalar(b)?.to_f64()?;
 268                 let c = this.read_scalar(c)?.to_f64()?;
 269                 let res = a.mul_add(b, c).value;
 270                 this.write_scalar(Scalar::from_f64(res), dest)?;
 271             }
 272
 273             "powif32" => {
 274                 let &[ref f, ref i] = check_arg_count(args)?;
 275                 // FIXME: Using host floats.
 276                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 277                 let i = this.read_scalar(i)?.to_i32()?;
 278                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 279             }
 280
 281             "powif64" => {
 282                 let &[ref f, ref i] = check_arg_count(args)?;
 283                 // FIXME: Using host floats.
 284                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 285                 let i = this.read_scalar(i)?.to_i32()?;
 286                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 287             }
 288
 289             "float_to_int_unchecked" => {
 290                 let &[ref val] = check_arg_count(args)?;
 291                 let val = this.read_immediate(val)?;
 292
 293                 let res = match val.layout.ty.kind() {
 294                     ty::Float(FloatTy::F32) =>
 295                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?,
 296                     ty::Float(FloatTy::F64) =>
 297                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?,
 298                     _ => bug!(
 299                         "`float_to_int_unchecked` called with non-float input type {:?}",
 300                         val.layout.ty
 301                     ),
 302                 };
 303
 304                 this.write_scalar(res, dest)?;
 305             }
 306
 307             // Atomic operations
 308             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
 309             "atomic_load_relaxed" => this.atomic_load(args, dest, AtomicReadOp::Relaxed)?,
 310             "atomic_load_acq" => this.atomic_load(args, dest, AtomicReadOp::Acquire)?,
 311
 312             "atomic_store" => this.atomic_store(args, AtomicWriteOp::SeqCst)?,
 313             "atomic_store_relaxed" => this.atomic_store(args, AtomicWriteOp::Relaxed)?,
 314             "atomic_store_rel" => this.atomic_store(args, AtomicWriteOp::Release)?,
 315
 316             "atomic_fence_acq" => this.atomic_fence(args, AtomicFenceOp::Acquire)?,
 317             "atomic_fence_rel" => this.atomic_fence(args, AtomicFenceOp::Release)?,
 318             "atomic_fence_acqrel" => this.atomic_fence(args, AtomicFenceOp::AcqRel)?,
 319             "atomic_fence" => this.atomic_fence(args, AtomicFenceOp::SeqCst)?,
 320
 321             "atomic_singlethreadfence_acq" => this.compiler_fence(args, AtomicFenceOp::Acquire)?,
 322             "atomic_singlethreadfence_rel" => this.compiler_fence(args, AtomicFenceOp::Release)?,
 323             "atomic_singlethreadfence_acqrel" =>
 324                 this.compiler_fence(args, AtomicFenceOp::AcqRel)?,
 325             "atomic_singlethreadfence" => this.compiler_fence(args, AtomicFenceOp::SeqCst)?,
 326
 327             "atomic_xchg" => this.atomic_exchange(args, dest, AtomicRwOp::SeqCst)?,
 328             "atomic_xchg_acq" => this.atomic_exchange(args, dest, AtomicRwOp::Acquire)?,
 329             "atomic_xchg_rel" => this.atomic_exchange(args, dest, AtomicRwOp::Release)?,
 330             "atomic_xchg_acqrel" => this.atomic_exchange(args, dest, AtomicRwOp::AcqRel)?,
 331             "atomic_xchg_relaxed" => this.atomic_exchange(args, dest, AtomicRwOp::Relaxed)?,
 332
 333             "atomic_cxchg" =>
 334                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::SeqCst)?,
 335             "atomic_cxchg_acq" => this.atomic_compare_exchange(
 336                 args,
 337                 dest,
 338                 AtomicRwOp::Acquire,
 339                 AtomicReadOp::Acquire,
 340             )?,
 341             "atomic_cxchg_rel" => this.atomic_compare_exchange(
 342                 args,
 343                 dest,
 344                 AtomicRwOp::Release,
 345                 AtomicReadOp::Relaxed,
 346             )?,
 347             "atomic_cxchg_acqrel" =>
 348                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Acquire)?,
 349             "atomic_cxchg_relaxed" => this.atomic_compare_exchange(
 350                 args,
 351                 dest,
 352                 AtomicRwOp::Relaxed,
 353                 AtomicReadOp::Relaxed,
 354             )?,
 355             "atomic_cxchg_acq_failrelaxed" => this.atomic_compare_exchange(
 356                 args,
 357                 dest,
 358                 AtomicRwOp::Acquire,
 359                 AtomicReadOp::Relaxed,
 360             )?,
 361             "atomic_cxchg_acqrel_failrelaxed" =>
 362                 this.atomic_compare_exchange(args, dest, AtomicRwOp::AcqRel, AtomicReadOp::Relaxed)?,
 363             "atomic_cxchg_failrelaxed" =>
 364                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Relaxed)?,
 365             "atomic_cxchg_failacq" =>
 366                 this.atomic_compare_exchange(args, dest, AtomicRwOp::SeqCst, AtomicReadOp::Acquire)?,
 367
 368             "atomic_cxchgweak" => this.atomic_compare_exchange_weak(
 369                 args,
 370                 dest,
 371                 AtomicRwOp::SeqCst,
 372                 AtomicReadOp::SeqCst,
 373             )?,
 374             "atomic_cxchgweak_acq" => this.atomic_compare_exchange_weak(
 375                 args,
 376                 dest,
 377                 AtomicRwOp::Acquire,
 378                 AtomicReadOp::Acquire,
 379             )?,
 380             "atomic_cxchgweak_rel" => this.atomic_compare_exchange_weak(
 381                 args,
 382                 dest,
 383                 AtomicRwOp::Release,
 384                 AtomicReadOp::Relaxed,
 385             )?,
 386             "atomic_cxchgweak_acqrel" => this.atomic_compare_exchange_weak(
 387                 args,
 388                 dest,
 389                 AtomicRwOp::AcqRel,
 390                 AtomicReadOp::Acquire,
 391             )?,
 392             "atomic_cxchgweak_relaxed" => this.atomic_compare_exchange_weak(
 393                 args,
 394                 dest,
 395                 AtomicRwOp::Relaxed,
 396                 AtomicReadOp::Relaxed,
 397             )?,
 398             "atomic_cxchgweak_acq_failrelaxed" => this.atomic_compare_exchange_weak(
 399                 args,
 400                 dest,
 401                 AtomicRwOp::Acquire,
 402                 AtomicReadOp::Relaxed,
 403             )?,
 404             "atomic_cxchgweak_acqrel_failrelaxed" => this.atomic_compare_exchange_weak(
 405                 args,
 406                 dest,
 407                 AtomicRwOp::AcqRel,
 408                 AtomicReadOp::Relaxed,
 409             )?,
 410             "atomic_cxchgweak_failrelaxed" => this.atomic_compare_exchange_weak(
 411                 args,
 412                 dest,
 413                 AtomicRwOp::SeqCst,
 414                 AtomicReadOp::Relaxed,
 415             )?,
 416             "atomic_cxchgweak_failacq" => this.atomic_compare_exchange_weak(
 417                 args,
 418                 dest,
 419                 AtomicRwOp::SeqCst,
 420                 AtomicReadOp::Acquire,
 421             )?,
 422
 423             "atomic_or" => this.atomic_op(
 424                 args,
 425                 dest,
 426                 AtomicOp::MirOp(BinOp::BitOr, false),
 427                 AtomicRwOp::SeqCst,
 428             )?,
 429             "atomic_or_acq" => this.atomic_op(
 430                 args,
 431                 dest,
 432                 AtomicOp::MirOp(BinOp::BitOr, false),
 433                 AtomicRwOp::Acquire,
 434             )?,
 435             "atomic_or_rel" => this.atomic_op(
 436                 args,
 437                 dest,
 438                 AtomicOp::MirOp(BinOp::BitOr, false),
 439                 AtomicRwOp::Release,
 440             )?,
 441             "atomic_or_acqrel" => this.atomic_op(
 442                 args,
 443                 dest,
 444                 AtomicOp::MirOp(BinOp::BitOr, false),
 445                 AtomicRwOp::AcqRel,
 446             )?,
 447             "atomic_or_relaxed" => this.atomic_op(
 448                 args,
 449                 dest,
 450                 AtomicOp::MirOp(BinOp::BitOr, false),
 451                 AtomicRwOp::Relaxed,
 452             )?,
 453             "atomic_xor" => this.atomic_op(
 454                 args,
 455                 dest,
 456                 AtomicOp::MirOp(BinOp::BitXor, false),
 457                 AtomicRwOp::SeqCst,
 458             )?,
 459             "atomic_xor_acq" => this.atomic_op(
 460                 args,
 461                 dest,
 462                 AtomicOp::MirOp(BinOp::BitXor, false),
 463                 AtomicRwOp::Acquire,
 464             )?,
 465             "atomic_xor_rel" => this.atomic_op(
 466                 args,
 467                 dest,
 468                 AtomicOp::MirOp(BinOp::BitXor, false),
 469                 AtomicRwOp::Release,
 470             )?,
 471             "atomic_xor_acqrel" => this.atomic_op(
 472                 args,
 473                 dest,
 474                 AtomicOp::MirOp(BinOp::BitXor, false),
 475                 AtomicRwOp::AcqRel,
 476             )?,
 477             "atomic_xor_relaxed" => this.atomic_op(
 478                 args,
 479                 dest,
 480                 AtomicOp::MirOp(BinOp::BitXor, false),
 481                 AtomicRwOp::Relaxed,
 482             )?,
 483             "atomic_and" => this.atomic_op(
 484                 args,
 485                 dest,
 486                 AtomicOp::MirOp(BinOp::BitAnd, false),
 487                 AtomicRwOp::SeqCst,
 488             )?,
 489             "atomic_and_acq" => this.atomic_op(
 490                 args,
 491                 dest,
 492                 AtomicOp::MirOp(BinOp::BitAnd, false),
 493                 AtomicRwOp::Acquire,
 494             )?,
 495             "atomic_and_rel" => this.atomic_op(
 496                 args,
 497                 dest,
 498                 AtomicOp::MirOp(BinOp::BitAnd, false),
 499                 AtomicRwOp::Release,
 500             )?,
 501             "atomic_and_acqrel" => this.atomic_op(
 502                 args,
 503                 dest,
 504                 AtomicOp::MirOp(BinOp::BitAnd, false),
 505                 AtomicRwOp::AcqRel,
 506             )?,
 507             "atomic_and_relaxed" => this.atomic_op(
 508                 args,
 509                 dest,
 510                 AtomicOp::MirOp(BinOp::BitAnd, false),
 511                 AtomicRwOp::Relaxed,
 512             )?,
 513             "atomic_nand" => this.atomic_op(
 514                 args,
 515                 dest,
 516                 AtomicOp::MirOp(BinOp::BitAnd, true),
 517                 AtomicRwOp::SeqCst,
 518             )?,
 519             "atomic_nand_acq" => this.atomic_op(
 520                 args,
 521                 dest,
 522                 AtomicOp::MirOp(BinOp::BitAnd, true),
 523                 AtomicRwOp::Acquire,
 524             )?,
 525             "atomic_nand_rel" => this.atomic_op(
 526                 args,
 527                 dest,
 528                 AtomicOp::MirOp(BinOp::BitAnd, true),
 529                 AtomicRwOp::Release,
 530             )?,
 531             "atomic_nand_acqrel" => this.atomic_op(
 532                 args,
 533                 dest,
 534                 AtomicOp::MirOp(BinOp::BitAnd, true),
 535                 AtomicRwOp::AcqRel,
 536             )?,
 537             "atomic_nand_relaxed" => this.atomic_op(
 538                 args,
 539                 dest,
 540                 AtomicOp::MirOp(BinOp::BitAnd, true),
 541                 AtomicRwOp::Relaxed,
 542             )?,
 543             "atomic_xadd" => this.atomic_op(
 544                 args,
 545                 dest,
 546                 AtomicOp::MirOp(BinOp::Add, false),
 547                 AtomicRwOp::SeqCst,
 548             )?,
 549             "atomic_xadd_acq" => this.atomic_op(
 550                 args,
 551                 dest,
 552                 AtomicOp::MirOp(BinOp::Add, false),
 553                 AtomicRwOp::Acquire,
 554             )?,
 555             "atomic_xadd_rel" => this.atomic_op(
 556                 args,
 557                 dest,
 558                 AtomicOp::MirOp(BinOp::Add, false),
 559                 AtomicRwOp::Release,
 560             )?,
 561             "atomic_xadd_acqrel" => this.atomic_op(
 562                 args,
 563                 dest,
 564                 AtomicOp::MirOp(BinOp::Add, false),
 565                 AtomicRwOp::AcqRel,
 566             )?,
 567             "atomic_xadd_relaxed" => this.atomic_op(
 568                 args,
 569                 dest,
 570                 AtomicOp::MirOp(BinOp::Add, false),
 571                 AtomicRwOp::Relaxed,
 572             )?,
 573             "atomic_xsub" => this.atomic_op(
 574                 args,
 575                 dest,
 576                 AtomicOp::MirOp(BinOp::Sub, false),
 577                 AtomicRwOp::SeqCst,
 578             )?,
 579             "atomic_xsub_acq" => this.atomic_op(
 580                 args,
 581                 dest,
 582                 AtomicOp::MirOp(BinOp::Sub, false),
 583                 AtomicRwOp::Acquire,
 584             )?,
 585             "atomic_xsub_rel" => this.atomic_op(
 586                 args,
 587                 dest,
 588                 AtomicOp::MirOp(BinOp::Sub, false),
 589                 AtomicRwOp::Release,
 590             )?,
 591             "atomic_xsub_acqrel" => this.atomic_op(
 592                 args,
 593                 dest,
 594                 AtomicOp::MirOp(BinOp::Sub, false),
 595                 AtomicRwOp::AcqRel,
 596             )?,
 597             "atomic_xsub_relaxed" => this.atomic_op(
 598                 args,
 599                 dest,
 600                 AtomicOp::MirOp(BinOp::Sub, false),
 601                 AtomicRwOp::Relaxed,
 602             )?,
 603             "atomic_min" =>
 604                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 605             "atomic_min_acq" =>
 606                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 607             "atomic_min_rel" =>
 608                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 609             "atomic_min_acqrel" =>
 610                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 611             "atomic_min_relaxed" =>
 612                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 613             "atomic_max" =>
 614                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 615             "atomic_max_acq" =>
 616                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 617             "atomic_max_rel" =>
 618                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 619             "atomic_max_acqrel" =>
 620                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 621             "atomic_max_relaxed" =>
 622                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 623             "atomic_umin" =>
 624                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::SeqCst)?,
 625             "atomic_umin_acq" =>
 626                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Acquire)?,
 627             "atomic_umin_rel" =>
 628                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Release)?,
 629             "atomic_umin_acqrel" =>
 630                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::AcqRel)?,
 631             "atomic_umin_relaxed" =>
 632                 this.atomic_op(args, dest, AtomicOp::Min, AtomicRwOp::Relaxed)?,
 633             "atomic_umax" =>
 634                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::SeqCst)?,
 635             "atomic_umax_acq" =>
 636                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Acquire)?,
 637             "atomic_umax_rel" =>
 638                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Release)?,
 639             "atomic_umax_acqrel" =>
 640                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::AcqRel)?,
 641             "atomic_umax_relaxed" =>
 642                 this.atomic_op(args, dest, AtomicOp::Max, AtomicRwOp::Relaxed)?,
 643
 644             // Query type information
 645             "assert_zero_valid" | "assert_uninit_valid" => {
 646                 let &[] = check_arg_count(args)?;
 647                 let ty = instance.substs.type_at(0);
 648                 let layout = this.layout_of(ty)?;
 649                 // Abort here because the caller might not be panic safe.
 650                 if layout.abi.is_uninhabited() {
 651                     // Use this message even for the other intrinsics, as that's what codegen does
 652                     throw_machine_stop!(TerminationInfo::Abort(format!(
 653                         "aborted execution: attempted to instantiate uninhabited type `{}`",
 654                         ty
 655                     )))
 656                 }
 657                 if intrinsic_name == "assert_zero_valid"
 658                     && !layout.might_permit_raw_init(this, /*zero:*/ true).unwrap()
 659                 {
 660                     throw_machine_stop!(TerminationInfo::Abort(format!(
 661                         "aborted execution: attempted to zero-initialize type `{}`, which is invalid",
 662                         ty
 663                     )))
 664                 }
 665                 if intrinsic_name == "assert_uninit_valid"
 666                     && !layout.might_permit_raw_init(this, /*zero:*/ false).unwrap()
 667                 {
 668                     throw_machine_stop!(TerminationInfo::Abort(format!(
 669                         "aborted execution: attempted to leave type `{}` uninitialized, which is invalid",
 670                         ty
 671                     )))
 672                 }
 673             }
 674
 675             // Other
 676             "exact_div" => {
 677                 let &[ref num, ref denom] = check_arg_count(args)?;
 678                 this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
 679             }
 680
 681             "try" => return this.handle_try(args, dest, ret),
 682
 683             "breakpoint" => {
 684                 let &[] = check_arg_count(args)?;
 685                 // normally this would raise a SIGTRAP, which aborts if no debugger is connected
 686                 throw_machine_stop!(TerminationInfo::Abort("Trace/breakpoint trap".to_string()))
 687             }
 688
 689             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
 690         }
 691
 692         trace!("{:?}", this.dump_place(**dest));
 693         this.go_to_block(ret);
 694         Ok(())
 695     }
 696
 697     fn atomic_load(
 698         &mut self,
 699         args: &[OpTy<'tcx, Tag>],
 700         dest: &PlaceTy<'tcx, Tag>,
 701         atomic: AtomicReadOp,
 702     ) -> InterpResult<'tcx> {
 703         let this = self.eval_context_mut();
 704
 705         let &[ref place] = check_arg_count(args)?;
 706         let place = this.deref_operand(place)?;
 707
 708         // make sure it fits into a scalar; otherwise it cannot be atomic
 709         let val = this.read_scalar_atomic(&place, atomic)?;
 710
 711         // Check alignment requirements. Atomics must always be aligned to their size,
 712         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 713         // be 8-aligned).
 714         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 715         this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 716         this.write_scalar(val, dest)?;
 717         Ok(())
 718     }
 719
 720     fn atomic_store(
 721         &mut self,
 722         args: &[OpTy<'tcx, Tag>],
 723         atomic: AtomicWriteOp,
 724     ) -> InterpResult<'tcx> {
 725         let this = self.eval_context_mut();
 726
 727         let &[ref place, ref val] = check_arg_count(args)?;
 728         let place = this.deref_operand(place)?;
 729         let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
 730
 731         // Check alignment requirements. Atomics must always be aligned to their size,
 732         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 733         // be 8-aligned).
 734         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 735         this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 736
 737         // Perform atomic store
 738         this.write_scalar_atomic(val, &place, atomic)?;
 739         Ok(())
 740     }
 741
 742     fn compiler_fence(
 743         &mut self,
 744         args: &[OpTy<'tcx, Tag>],
 745         atomic: AtomicFenceOp,
 746     ) -> InterpResult<'tcx> {
 747         let &[] = check_arg_count(args)?;
 748         let _ = atomic;
 749         //FIXME: compiler fences are currently ignored
 750         Ok(())
 751     }
 752
 753     fn atomic_fence(
 754         &mut self,
 755         args: &[OpTy<'tcx, Tag>],
 756         atomic: AtomicFenceOp,
 757     ) -> InterpResult<'tcx> {
 758         let this = self.eval_context_mut();
 759         let &[] = check_arg_count(args)?;
 760         this.validate_atomic_fence(atomic)?;
 761         Ok(())
 762     }
 763
 764     fn atomic_op(
 765         &mut self,
 766         args: &[OpTy<'tcx, Tag>],
 767         dest: &PlaceTy<'tcx, Tag>,
 768         atomic_op: AtomicOp,
 769         atomic: AtomicRwOp,
 770     ) -> InterpResult<'tcx> {
 771         let this = self.eval_context_mut();
 772
 773         let &[ref place, ref rhs] = check_arg_count(args)?;
 774         let place = this.deref_operand(place)?;
 775
 776         if !place.layout.ty.is_integral() {
 777             bug!("Atomic arithmetic operations only work on integer types");
 778         }
 779         let rhs = this.read_immediate(rhs)?;
 780
 781         // Check alignment requirements. Atomics must always be aligned to their size,
 782         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 783         // be 8-aligned).
 784         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 785         this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 786
 787         match atomic_op {
 788             AtomicOp::Min => {
 789                 let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?;
 790                 this.write_immediate(*old, &dest)?; // old value is returned
 791                 Ok(())
 792             }
 793             AtomicOp::Max => {
 794                 let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?;
 795                 this.write_immediate(*old, &dest)?; // old value is returned
 796                 Ok(())
 797             }
 798             AtomicOp::MirOp(op, neg) => {
 799                 let old = this.atomic_op_immediate(&place, &rhs, op, neg, atomic)?;
 800                 this.write_immediate(*old, dest)?; // old value is returned
 801                 Ok(())
 802             }
 803         }
 804     }
 805
 806     fn atomic_exchange(
 807         &mut self,
 808         args: &[OpTy<'tcx, Tag>],
 809         dest: &PlaceTy<'tcx, Tag>,
 810         atomic: AtomicRwOp,
 811     ) -> InterpResult<'tcx> {
 812         let this = self.eval_context_mut();
 813
 814         let &[ref place, ref new] = check_arg_count(args)?;
 815         let place = this.deref_operand(place)?;
 816         let new = this.read_scalar(new)?;
 817
 818         // Check alignment requirements. Atomics must always be aligned to their size,
 819         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 820         // be 8-aligned).
 821         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 822         this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 823
 824         let old = this.atomic_exchange_scalar(&place, new, atomic)?;
 825         this.write_scalar(old, dest)?; // old value is returned
 826         Ok(())
 827     }
 828
 829     fn atomic_compare_exchange_impl(
 830         &mut self,
 831         args: &[OpTy<'tcx, Tag>],
 832         dest: &PlaceTy<'tcx, Tag>,
 833         success: AtomicRwOp,
 834         fail: AtomicReadOp,
 835         can_fail_spuriously: bool,
 836     ) -> InterpResult<'tcx> {
 837         let this = self.eval_context_mut();
 838
 839         let &[ref place, ref expect_old, ref new] = check_arg_count(args)?;
 840         let place = this.deref_operand(place)?;
 841         let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
 842         let new = this.read_scalar(new)?;
 843
 844         // Check alignment requirements. Atomics must always be aligned to their size,
 845         // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 846         // be 8-aligned).
 847         let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 848         this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 849
 850         let old = this.atomic_compare_exchange_scalar(
 851             &place,
 852             &expect_old,
 853             new,
 854             success,
 855             fail,
 856             can_fail_spuriously,
 857         )?;
 858
 859         // Return old value.
 860         this.write_immediate(old, dest)?;
 861         Ok(())
 862     }
 863
 864     fn atomic_compare_exchange(
 865         &mut self,
 866         args: &[OpTy<'tcx, Tag>],
 867         dest: &PlaceTy<'tcx, Tag>,
 868         success: AtomicRwOp,
 869         fail: AtomicReadOp,
 870     ) -> InterpResult<'tcx> {
 871         self.atomic_compare_exchange_impl(args, dest, success, fail, false)
 872     }
 873
 874     fn atomic_compare_exchange_weak(
 875         &mut self,
 876         args: &[OpTy<'tcx, Tag>],
 877         dest: &PlaceTy<'tcx, Tag>,
 878         success: AtomicRwOp,
 879         fail: AtomicReadOp,
 880     ) -> InterpResult<'tcx> {
 881         self.atomic_compare_exchange_impl(args, dest, success, fail, true)
 882     }
 883
 884     fn float_to_int_unchecked<F>(
 885         &self,
 886         f: F,
 887         dest_ty: ty::Ty<'tcx>,
 888     ) -> InterpResult<'tcx, Scalar<Tag>>
 889     where
 890         F: Float + Into<Scalar<Tag>>,
 891     {
 892         let this = self.eval_context_ref();
 893
 894         // Step 1: cut off the fractional part of `f`. The result of this is
 895         // guaranteed to be precisely representable in IEEE floats.
 896         let f = f.round_to_integral(Round::TowardZero).value;
 897
 898         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
 899         Ok(match dest_ty.kind() {
 900             // Unsigned
 901             ty::Uint(t) => {
 902                 let size = Integer::from_uint_ty(this, *t).size();
 903                 let res = f.to_u128(size.bits_usize());
 904                 if res.status.is_empty() {
 905                     // No status flags means there was no further rounding or other loss of precision.
 906                     Scalar::from_uint(res.value, size)
 907                 } else {
 908                     // `f` was not representable in this integer type.
 909                     throw_ub_format!(
 910                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 911                         f,
 912                         dest_ty,
 913                     );
 914                 }
 915             }
 916             // Signed
 917             ty::Int(t) => {
 918                 let size = Integer::from_int_ty(this, *t).size();
 919                 let res = f.to_i128(size.bits_usize());
 920                 if res.status.is_empty() {
 921                     // No status flags means there was no further rounding or other loss of precision.
 922                     Scalar::from_int(res.value, size)
 923                 } else {
 924                     // `f` was not representable in this integer type.
 925                     throw_ub_format!(
 926                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 927                         f,
 928                         dest_ty,
 929                     );
 930                 }
 931             }
 932             // Nothing else
 933             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
 934         })
 935     }
 936 }