src/shims/intrinsics.rs

   1 use std::iter;
   2
   3 use log::trace;
   4
   5 use rustc_attr as attr;
   6 use rustc_ast::ast::FloatTy;
   7 use rustc_middle::{mir, ty};
   8 use rustc_middle::ty::layout::IntegerExt;
   9 use rustc_apfloat::{Float, Round};
  10 use rustc_target::abi::{Align, Integer, LayoutOf};
  11
  12 use crate::*;
  13 use helpers::check_arg_count;
  14
  15 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  16 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  17     fn call_intrinsic(
  18         &mut self,
  19         instance: ty::Instance<'tcx>,
  20         args: &[OpTy<'tcx, Tag>],
  21         ret: Option<(PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  22         _unwind: Option<mir::BasicBlock>,
  23     ) -> InterpResult<'tcx> {
  24         let this = self.eval_context_mut();
  25
  26         if this.emulate_intrinsic(instance, args, ret)? {
  27             return Ok(());
  28         }
  29
  30         // All supported intrinsics have a return place.
  31         let intrinsic_name = &*this.tcx.item_name(instance.def_id()).as_str();
  32         let (dest, ret) = match ret {
  33             None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  34             Some(p) => p,
  35         };
  36
  37         // Then handle terminating intrinsics.
  38         match intrinsic_name {
  39             // Miri overwriting CTFE intrinsics.
  40             "ptr_guaranteed_eq" => {
  41                 let &[left, right] = check_arg_count(args)?;
  42                 let left = this.read_immediate(left)?;
  43                 let right = this.read_immediate(right)?;
  44                 this.binop_ignore_overflow(mir::BinOp::Eq, left, right, dest)?;
  45             }
  46             "ptr_guaranteed_ne" => {
  47                 let &[left, right] = check_arg_count(args)?;
  48                 let left = this.read_immediate(left)?;
  49                 let right = this.read_immediate(right)?;
  50                 this.binop_ignore_overflow(mir::BinOp::Ne, left, right, dest)?;
  51             }
  52
  53             // Raw memory accesses
  54             #[rustfmt::skip]
  55             | "copy"
  56             | "copy_nonoverlapping"
  57             => {
  58                 let &[src, dest, count] = check_arg_count(args)?;
  59                 let elem_ty = instance.substs.type_at(0);
  60                 let elem_layout = this.layout_of(elem_ty)?;
  61                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
  62                 let elem_align = elem_layout.align.abi;
  63
  64                 let size = elem_layout.size.checked_mul(count, this)
  65                     .ok_or_else(|| err_ub_format!("overflow computing total size of `{}`", intrinsic_name))?;
  66                 let src = this.read_scalar(src)?.check_init()?;
  67                 let src = this.memory.check_ptr_access(src, size, elem_align)?;
  68                 let dest = this.read_scalar(dest)?.check_init()?;
  69                 let dest = this.memory.check_ptr_access(dest, size, elem_align)?;
  70
  71                 if let (Some(src), Some(dest)) = (src, dest) {
  72                     this.memory.copy(
  73                         src,
  74                         dest,
  75                         size,
  76                         intrinsic_name.ends_with("_nonoverlapping"),
  77                     )?;
  78                 }
  79             }
  80
  81             "move_val_init" => {
  82                 let &[place, dest] = check_arg_count(args)?;
  83                 let place = this.deref_operand(place)?;
  84                 this.copy_op(dest, place.into())?;
  85             }
  86
  87             "volatile_load" => {
  88                 let &[place] = check_arg_count(args)?;
  89                 let place = this.deref_operand(place)?;
  90                 this.copy_op(place.into(), dest)?;
  91             }
  92             "volatile_store" => {
  93                 let &[place, dest] = check_arg_count(args)?;
  94                 let place = this.deref_operand(place)?;
  95                 this.copy_op(dest, place.into())?;
  96             }
  97
  98             "write_bytes" => {
  99                 let &[ptr, val_byte, count] = check_arg_count(args)?;
 100                 let ty = instance.substs.type_at(0);
 101                 let ty_layout = this.layout_of(ty)?;
 102                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
 103                 let ptr = this.read_scalar(ptr)?.check_init()?;
 104                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
 105                 let byte_count = ty_layout.size.checked_mul(count, this)
 106                     .ok_or_else(|| err_ub_format!("overflow computing total size of `write_bytes`"))?;
 107                 this.memory
 108                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
 109             }
 110
 111             // Floating-point operations
 112             #[rustfmt::skip]
 113             | "sinf32"
 114             | "fabsf32"
 115             | "cosf32"
 116             | "sqrtf32"
 117             | "expf32"
 118             | "exp2f32"
 119             | "logf32"
 120             | "log10f32"
 121             | "log2f32"
 122             | "floorf32"
 123             | "ceilf32"
 124             | "truncf32"
 125             | "roundf32"
 126             => {
 127                 let &[f] = check_arg_count(args)?;
 128                 // FIXME: Using host floats.
 129                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 130                 let f = match intrinsic_name {
 131                     "sinf32" => f.sin(),
 132                     "fabsf32" => f.abs(),
 133                     "cosf32" => f.cos(),
 134                     "sqrtf32" => f.sqrt(),
 135                     "expf32" => f.exp(),
 136                     "exp2f32" => f.exp2(),
 137                     "logf32" => f.ln(),
 138                     "log10f32" => f.log10(),
 139                     "log2f32" => f.log2(),
 140                     "floorf32" => f.floor(),
 141                     "ceilf32" => f.ceil(),
 142                     "truncf32" => f.trunc(),
 143                     "roundf32" => f.round(),
 144                     _ => bug!(),
 145                 };
 146                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 147             }
 148
 149             #[rustfmt::skip]
 150             | "sinf64"
 151             | "fabsf64"
 152             | "cosf64"
 153             | "sqrtf64"
 154             | "expf64"
 155             | "exp2f64"
 156             | "logf64"
 157             | "log10f64"
 158             | "log2f64"
 159             | "floorf64"
 160             | "ceilf64"
 161             | "truncf64"
 162             | "roundf64"
 163             => {
 164                 let &[f] = check_arg_count(args)?;
 165                 // FIXME: Using host floats.
 166                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 167                 let f = match intrinsic_name {
 168                     "sinf64" => f.sin(),
 169                     "fabsf64" => f.abs(),
 170                     "cosf64" => f.cos(),
 171                     "sqrtf64" => f.sqrt(),
 172                     "expf64" => f.exp(),
 173                     "exp2f64" => f.exp2(),
 174                     "logf64" => f.ln(),
 175                     "log10f64" => f.log10(),
 176                     "log2f64" => f.log2(),
 177                     "floorf64" => f.floor(),
 178                     "ceilf64" => f.ceil(),
 179                     "truncf64" => f.trunc(),
 180                     "roundf64" => f.round(),
 181                     _ => bug!(),
 182                 };
 183                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 184             }
 185
 186             #[rustfmt::skip]
 187             | "fadd_fast"
 188             | "fsub_fast"
 189             | "fmul_fast"
 190             | "fdiv_fast"
 191             | "frem_fast"
 192             => {
 193                 let &[a, b] = check_arg_count(args)?;
 194                 let a = this.read_immediate(a)?;
 195                 let b = this.read_immediate(b)?;
 196                 let op = match intrinsic_name {
 197                     "fadd_fast" => mir::BinOp::Add,
 198                     "fsub_fast" => mir::BinOp::Sub,
 199                     "fmul_fast" => mir::BinOp::Mul,
 200                     "fdiv_fast" => mir::BinOp::Div,
 201                     "frem_fast" => mir::BinOp::Rem,
 202                     _ => bug!(),
 203                 };
 204                 this.binop_ignore_overflow(op, a, b, dest)?;
 205             }
 206
 207             #[rustfmt::skip]
 208             | "minnumf32"
 209             | "maxnumf32"
 210             | "copysignf32"
 211             => {
 212                 let &[a, b] = check_arg_count(args)?;
 213                 let a = this.read_scalar(a)?.to_f32()?;
 214                 let b = this.read_scalar(b)?.to_f32()?;
 215                 let res = match intrinsic_name {
 216                     "minnumf32" => a.min(b),
 217                     "maxnumf32" => a.max(b),
 218                     "copysignf32" => a.copy_sign(b),
 219                     _ => bug!(),
 220                 };
 221                 this.write_scalar(Scalar::from_f32(res), dest)?;
 222             }
 223
 224             #[rustfmt::skip]
 225             | "minnumf64"
 226             | "maxnumf64"
 227             | "copysignf64"
 228             => {
 229                 let &[a, b] = check_arg_count(args)?;
 230                 let a = this.read_scalar(a)?.to_f64()?;
 231                 let b = this.read_scalar(b)?.to_f64()?;
 232                 let res = match intrinsic_name {
 233                     "minnumf64" => a.min(b),
 234                     "maxnumf64" => a.max(b),
 235                     "copysignf64" => a.copy_sign(b),
 236                     _ => bug!(),
 237                 };
 238                 this.write_scalar(Scalar::from_f64(res), dest)?;
 239             }
 240
 241             "powf32" => {
 242                 let &[f, f2] = check_arg_count(args)?;
 243                 // FIXME: Using host floats.
 244                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 245                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
 246                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 247             }
 248
 249             "powf64" => {
 250                 let &[f, f2] = check_arg_count(args)?;
 251                 // FIXME: Using host floats.
 252                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 253                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
 254                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 255             }
 256
 257             "fmaf32" => {
 258                 let &[a, b, c] = check_arg_count(args)?;
 259                 let a = this.read_scalar(a)?.to_f32()?;
 260                 let b = this.read_scalar(b)?.to_f32()?;
 261                 let c = this.read_scalar(c)?.to_f32()?;
 262                 let res = a.mul_add(b, c).value;
 263                 this.write_scalar(Scalar::from_f32(res), dest)?;
 264             }
 265
 266             "fmaf64" => {
 267                 let &[a, b, c] = check_arg_count(args)?;
 268                 let a = this.read_scalar(a)?.to_f64()?;
 269                 let b = this.read_scalar(b)?.to_f64()?;
 270                 let c = this.read_scalar(c)?.to_f64()?;
 271                 let res = a.mul_add(b, c).value;
 272                 this.write_scalar(Scalar::from_f64(res), dest)?;
 273             }
 274
 275             "powif32" => {
 276                 let &[f, i] = check_arg_count(args)?;
 277                 // FIXME: Using host floats.
 278                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
 279                 let i = this.read_scalar(i)?.to_i32()?;
 280                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 281             }
 282
 283             "powif64" => {
 284                 let &[f, i] = check_arg_count(args)?;
 285                 // FIXME: Using host floats.
 286                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
 287                 let i = this.read_scalar(i)?.to_i32()?;
 288                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 289             }
 290
 291             "float_to_int_unchecked" => {
 292                 let &[val] = check_arg_count(args)?;
 293                 let val = this.read_immediate(val)?;
 294
 295                 let res = match val.layout.ty.kind() {
 296                     ty::Float(FloatTy::F32) => {
 297                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?
 298                     }
 299                     ty::Float(FloatTy::F64) => {
 300                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?
 301                     }
 302                     _ => bug!("`float_to_int_unchecked` called with non-float input type {:?}", val.layout.ty),
 303                 };
 304
 305                 this.write_scalar(res, dest)?;
 306             }
 307
 308             // Atomic operations
 309             #[rustfmt::skip]
 310             | "atomic_load"
 311             | "atomic_load_relaxed"
 312             | "atomic_load_acq"
 313             => {
 314                 let &[place] = check_arg_count(args)?;
 315                 let place = this.deref_operand(place)?;
 316                 let val = this.read_scalar(place.into())?; // make sure it fits into a scalar; otherwise it cannot be atomic
 317
 318                 // Check alignment requirements. Atomics must always be aligned to their size,
 319                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 320                 // be 8-aligned).
 321                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 322                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 323
 324                 this.write_scalar(val, dest)?;
 325             }
 326
 327             #[rustfmt::skip]
 328             | "atomic_store"
 329             | "atomic_store_relaxed"
 330             | "atomic_store_rel"
 331             => {
 332                 let &[place, val] = check_arg_count(args)?;
 333                 let place = this.deref_operand(place)?;
 334                 let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
 335
 336                 // Check alignment requirements. Atomics must always be aligned to their size,
 337                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 338                 // be 8-aligned).
 339                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 340                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 341
 342                 this.write_scalar(val, place.into())?;
 343             }
 344
 345             #[rustfmt::skip]
 346             | "atomic_fence_acq"
 347             | "atomic_fence_rel"
 348             | "atomic_fence_acqrel"
 349             | "atomic_fence"
 350             | "atomic_singlethreadfence_acq"
 351             | "atomic_singlethreadfence_rel"
 352             | "atomic_singlethreadfence_acqrel"
 353             | "atomic_singlethreadfence"
 354             => {
 355                 let &[] = check_arg_count(args)?;
 356                 // FIXME: this will become relevant once we try to detect data races.
 357             }
 358
 359             _ if intrinsic_name.starts_with("atomic_xchg") => {
 360                 let &[place, new] = check_arg_count(args)?;
 361                 let place = this.deref_operand(place)?;
 362                 let new = this.read_scalar(new)?;
 363                 let old = this.read_scalar(place.into())?;
 364
 365                 // Check alignment requirements. Atomics must always be aligned to their size,
 366                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 367                 // be 8-aligned).
 368                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 369                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 370
 371                 this.write_scalar(old, dest)?; // old value is returned
 372                 this.write_scalar(new, place.into())?;
 373             }
 374
 375             _ if intrinsic_name.starts_with("atomic_cxchg") => {
 376                 let &[place, expect_old, new] = check_arg_count(args)?;
 377                 let place = this.deref_operand(place)?;
 378                 let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
 379                 let new = this.read_scalar(new)?;
 380                 let old = this.read_immediate(place.into())?; // read as immediate for the sake of `binary_op()`
 381
 382                 // Check alignment requirements. Atomics must always be aligned to their size,
 383                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 384                 // be 8-aligned).
 385                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 386                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 387
 388                 // `binary_op` will bail if either of them is not a scalar.
 389                 let eq = this.overflowing_binary_op(mir::BinOp::Eq, old, expect_old)?.0;
 390                 let res = Immediate::ScalarPair(old.to_scalar_or_uninit(), eq.into());
 391                 // Return old value.
 392                 this.write_immediate(res, dest)?;
 393                 // Update ptr depending on comparison.
 394                 if eq.to_bool()? {
 395                     this.write_scalar(new, place.into())?;
 396                 }
 397             }
 398
 399             #[rustfmt::skip]
 400             | "atomic_or"
 401             | "atomic_or_acq"
 402             | "atomic_or_rel"
 403             | "atomic_or_acqrel"
 404             | "atomic_or_relaxed"
 405             | "atomic_xor"
 406             | "atomic_xor_acq"
 407             | "atomic_xor_rel"
 408             | "atomic_xor_acqrel"
 409             | "atomic_xor_relaxed"
 410             | "atomic_and"
 411             | "atomic_and_acq"
 412             | "atomic_and_rel"
 413             | "atomic_and_acqrel"
 414             | "atomic_and_relaxed"
 415             | "atomic_nand"
 416             | "atomic_nand_acq"
 417             | "atomic_nand_rel"
 418             | "atomic_nand_acqrel"
 419             | "atomic_nand_relaxed"
 420             | "atomic_xadd"
 421             | "atomic_xadd_acq"
 422             | "atomic_xadd_rel"
 423             | "atomic_xadd_acqrel"
 424             | "atomic_xadd_relaxed"
 425             | "atomic_xsub"
 426             | "atomic_xsub_acq"
 427             | "atomic_xsub_rel"
 428             | "atomic_xsub_acqrel"
 429             | "atomic_xsub_relaxed"
 430             => {
 431                 let &[place, rhs] = check_arg_count(args)?;
 432                 let place = this.deref_operand(place)?;
 433                 if !place.layout.ty.is_integral() {
 434                     bug!("Atomic arithmetic operations only work on integer types");
 435                 }
 436                 let rhs = this.read_immediate(rhs)?;
 437                 let old = this.read_immediate(place.into())?;
 438
 439                 // Check alignment requirements. Atomics must always be aligned to their size,
 440                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 441                 // be 8-aligned).
 442                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 443                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 444
 445                 this.write_immediate(*old, dest)?; // old value is returned
 446                 let (op, neg) = match intrinsic_name.split('_').nth(1).unwrap() {
 447                     "or" => (mir::BinOp::BitOr, false),
 448                     "xor" => (mir::BinOp::BitXor, false),
 449                     "and" => (mir::BinOp::BitAnd, false),
 450                     "xadd" => (mir::BinOp::Add, false),
 451                     "xsub" => (mir::BinOp::Sub, false),
 452                     "nand" => (mir::BinOp::BitAnd, true),
 453                     _ => bug!(),
 454                 };
 455                 // Atomics wrap around on overflow.
 456                 let val = this.binary_op(op, old, rhs)?;
 457                 let val = if neg { this.unary_op(mir::UnOp::Not, val)? } else { val };
 458                 this.write_immediate(*val, place.into())?;
 459             }
 460
 461             // Query type information
 462             "assert_inhabited" |
 463             "assert_zero_valid" |
 464             "assert_uninit_valid" => {
 465                 let &[] = check_arg_count(args)?;
 466                 let ty = instance.substs.type_at(0);
 467                 let layout = this.layout_of(ty)?;
 468                 // Abort here because the caller might not be panic safe.
 469                 if layout.abi.is_uninhabited() {
 470                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to instantiate uninhabited type `{}`", ty))))
 471                 }
 472                 if intrinsic_name == "assert_zero_valid" && !layout.might_permit_raw_init(this, /*zero:*/ true).unwrap() {
 473                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to zero-initialize type `{}`, which is invalid", ty))))
 474                 }
 475                 if intrinsic_name == "assert_uninit_valid" && !layout.might_permit_raw_init(this, /*zero:*/ false).unwrap() {
 476                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to leave type `{}` uninitialized, which is invalid", ty))))
 477                 }
 478             }
 479
 480             // Other
 481             "assume" => {
 482                 let &[cond] = check_arg_count(args)?;
 483                 let cond = this.read_scalar(cond)?.check_init()?.to_bool()?;
 484                 if !cond {
 485                     throw_ub_format!("`assume` intrinsic called with `false`");
 486                 }
 487             }
 488
 489             "exact_div" => {
 490                 let &[num, denom] = check_arg_count(args)?;
 491                 this.exact_div(this.read_immediate(num)?, this.read_immediate(denom)?, dest)?;
 492             }
 493
 494             "forget" => {
 495                 // We get an argument... and forget about it.
 496                 let &[_] = check_arg_count(args)?;
 497             }
 498
 499             "try" => return this.handle_try(args, dest, ret),
 500
 501             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
 502         }
 503
 504         trace!("{:?}", this.dump_place(*dest));
 505         this.go_to_block(ret);
 506         Ok(())
 507     }
 508
 509     fn float_to_int_unchecked<F>(
 510         &self,
 511         f: F,
 512         dest_ty: ty::Ty<'tcx>,
 513     ) -> InterpResult<'tcx, Scalar<Tag>>
 514     where
 515         F: Float + Into<Scalar<Tag>>
 516     {
 517         let this = self.eval_context_ref();
 518
 519         // Step 1: cut off the fractional part of `f`. The result of this is
 520         // guaranteed to be precisely representable in IEEE floats.
 521         let f = f.round_to_integral(Round::TowardZero).value;
 522
 523         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
 524         Ok(match dest_ty.kind() {
 525             // Unsigned
 526             ty::Uint(t) => {
 527                 let size = Integer::from_attr(this, attr::IntType::UnsignedInt(*t)).size();
 528                 let res = f.to_u128(size.bits_usize());
 529                 if res.status.is_empty() {
 530                     // No status flags means there was no further rounding or other loss of precision.
 531                     Scalar::from_uint(res.value, size)
 532                 } else {
 533                     // `f` was not representable in this integer type.
 534                     throw_ub_format!(
 535                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 536                         f, dest_ty,
 537                     );
 538                 }
 539             }
 540             // Signed
 541             ty::Int(t) => {
 542                 let size = Integer::from_attr(this, attr::IntType::SignedInt(*t)).size();
 543                 let res = f.to_i128(size.bits_usize());
 544                 if res.status.is_empty() {
 545                     // No status flags means there was no further rounding or other loss of precision.
 546                     Scalar::from_int(res.value, size)
 547                 } else {
 548                     // `f` was not representable in this integer type.
 549                     throw_ub_format!(
 550                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
 551                         f, dest_ty,
 552                     );
 553                 }
 554             }
 555             // Nothing else
 556             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
 557         })
 558     }
 559 }