src/shims/intrinsics.rs

   1 use rustc_apfloat::Float;
   2 use rustc::mir;
   3 use rustc::mir::interpret::{InterpResult, PointerArithmetic};
   4 use rustc::ty::layout::{self, LayoutOf, Size, Align};
   5 use rustc::ty;
   6
   7 use crate::{
   8     PlaceTy, OpTy, ImmTy, Immediate, Scalar, Tag,
   9     OperatorEvalContextExt
  10 };
  11
  12 impl<'mir, 'tcx> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  13 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  14     fn call_intrinsic(
  15         &mut self,
  16         instance: ty::Instance<'tcx>,
  17         args: &[OpTy<'tcx, Tag>],
  18         dest: PlaceTy<'tcx, Tag>,
  19     ) -> InterpResult<'tcx> {
  20         let this = self.eval_context_mut();
  21         if this.emulate_intrinsic(instance, args, dest)? {
  22             return Ok(());
  23         }
  24         let tcx = &{this.tcx.tcx};
  25         let substs = instance.substs;
  26
  27         // All these intrinsics take raw pointers, so if we access memory directly
  28         // (as opposed to through a place), we have to remember to erase any tag
  29         // that might still hang around!
  30
  31         let intrinsic_name = this.tcx.item_name(instance.def_id()).as_str();
  32         match intrinsic_name.get() {
  33             "arith_offset" => {
  34                 let offset = this.read_scalar(args[1])?.to_isize(this)?;
  35                 let ptr = this.read_scalar(args[0])?.not_undef()?;
  36
  37                 let pointee_ty = substs.type_at(0);
  38                 let pointee_size = this.layout_of(pointee_ty)?.size.bytes() as i64;
  39                 let offset = offset.overflowing_mul(pointee_size).0;
  40                 let result_ptr = ptr.ptr_wrapping_signed_offset(offset, this);
  41                 this.write_scalar(result_ptr, dest)?;
  42             }
  43
  44             "assume" => {
  45                 let cond = this.read_scalar(args[0])?.to_bool()?;
  46                 if !cond {
  47                     throw_unsup!(AssumptionNotHeld);
  48                 }
  49             }
  50
  51             "volatile_load" => {
  52                 let place = this.deref_operand(args[0])?;
  53                 this.copy_op(place.into(), dest)?;
  54             }
  55
  56             "volatile_store" => {
  57                 let place = this.deref_operand(args[0])?;
  58                 this.copy_op(args[1], place.into())?;
  59             }
  60
  61             "atomic_load" |
  62             "atomic_load_relaxed" |
  63             "atomic_load_acq" => {
  64                 let place = this.deref_operand(args[0])?;
  65                 let val = this.read_scalar(place.into())?; // make sure it fits into a scalar; otherwise it cannot be atomic
  66
  67                 // Check alignment requirements. Atomics must always be aligned to their size,
  68                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
  69                 // be 8-aligned).
  70                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
  71                 this.memory().check_ptr_access(place.ptr, place.layout.size, align)?;
  72
  73                 this.write_scalar(val, dest)?;
  74             }
  75
  76             "atomic_store" |
  77             "atomic_store_relaxed" |
  78             "atomic_store_rel" => {
  79                 let place = this.deref_operand(args[0])?;
  80                 let val = this.read_scalar(args[1])?; // make sure it fits into a scalar; otherwise it cannot be atomic
  81
  82                 // Check alignment requirements. Atomics must always be aligned to their size,
  83                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
  84                 // be 8-aligned).
  85                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
  86                 this.memory().check_ptr_access(place.ptr, place.layout.size, align)?;
  87
  88                 this.write_scalar(val, place.into())?;
  89             }
  90
  91             "atomic_fence_acq" => {
  92                 // we are inherently singlethreaded and singlecored, this is a nop
  93             }
  94
  95             _ if intrinsic_name.starts_with("atomic_xchg") => {
  96                 let place = this.deref_operand(args[0])?;
  97                 let new = this.read_scalar(args[1])?;
  98                 let old = this.read_scalar(place.into())?;
  99
 100                 // Check alignment requirements. Atomics must always be aligned to their size,
 101                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 102                 // be 8-aligned).
 103                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 104                 this.memory().check_ptr_access(place.ptr, place.layout.size, align)?;
 105
 106                 this.write_scalar(old, dest)?; // old value is returned
 107                 this.write_scalar(new, place.into())?;
 108             }
 109
 110             _ if intrinsic_name.starts_with("atomic_cxchg") => {
 111                 let place = this.deref_operand(args[0])?;
 112                 let expect_old = this.read_immediate(args[1])?; // read as immediate for the sake of `binary_op()`
 113                 let new = this.read_scalar(args[2])?;
 114                 let old = this.read_immediate(place.into())?; // read as immediate for the sake of `binary_op()`
 115
 116                 // Check alignment requirements. Atomics must always be aligned to their size,
 117                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 118                 // be 8-aligned).
 119                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 120                 this.memory().check_ptr_access(place.ptr, place.layout.size, align)?;
 121
 122                 // binary_op will bail if either of them is not a scalar
 123                 let (eq, _) = this.binary_op(mir::BinOp::Eq, old, expect_old)?;
 124                 let res = Immediate::ScalarPair(old.to_scalar_or_undef(), eq.into());
 125                 this.write_immediate(res, dest)?; // old value is returned
 126                 // update ptr depending on comparison
 127                 if eq.to_bool()? {
 128                     this.write_scalar(new, place.into())?;
 129                 }
 130             }
 131
 132             "atomic_or" |
 133             "atomic_or_acq" |
 134             "atomic_or_rel" |
 135             "atomic_or_acqrel" |
 136             "atomic_or_relaxed" |
 137             "atomic_xor" |
 138             "atomic_xor_acq" |
 139             "atomic_xor_rel" |
 140             "atomic_xor_acqrel" |
 141             "atomic_xor_relaxed" |
 142             "atomic_and" |
 143             "atomic_and_acq" |
 144             "atomic_and_rel" |
 145             "atomic_and_acqrel" |
 146             "atomic_and_relaxed" |
 147             "atomic_nand" |
 148             "atomic_nand_acq" |
 149             "atomic_nand_rel" |
 150             "atomic_nand_acqrel" |
 151             "atomic_nand_relaxed" |
 152             "atomic_xadd" |
 153             "atomic_xadd_acq" |
 154             "atomic_xadd_rel" |
 155             "atomic_xadd_acqrel" |
 156             "atomic_xadd_relaxed" |
 157             "atomic_xsub" |
 158             "atomic_xsub_acq" |
 159             "atomic_xsub_rel" |
 160             "atomic_xsub_acqrel" |
 161             "atomic_xsub_relaxed" => {
 162                 let place = this.deref_operand(args[0])?;
 163                 if !place.layout.ty.is_integral() {
 164                     bug!("Atomic arithmetic operations only work on integer types");
 165                 }
 166                 let rhs = this.read_immediate(args[1])?;
 167                 let old = this.read_immediate(place.into())?;
 168
 169                 // Check alignment requirements. Atomics must always be aligned to their size,
 170                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 171                 // be 8-aligned).
 172                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 173                 this.memory().check_ptr_access(place.ptr, place.layout.size, align)?;
 174
 175                 this.write_immediate(*old, dest)?; // old value is returned
 176                 let (op, neg) = match intrinsic_name.split('_').nth(1).unwrap() {
 177                     "or" => (mir::BinOp::BitOr, false),
 178                     "xor" => (mir::BinOp::BitXor, false),
 179                     "and" => (mir::BinOp::BitAnd, false),
 180                     "xadd" => (mir::BinOp::Add, false),
 181                     "xsub" => (mir::BinOp::Sub, false),
 182                     "nand" => (mir::BinOp::BitAnd, true),
 183                     _ => bug!(),
 184                 };
 185                 // Atomics wrap around on overflow.
 186                 let (val, _overflowed) = this.binary_op(op, old, rhs)?;
 187                 let val = if neg {
 188                     this.unary_op(mir::UnOp::Not, ImmTy::from_scalar(val, old.layout))?
 189                 } else {
 190                     val
 191                 };
 192                 this.write_scalar(val, place.into())?;
 193             }
 194
 195             "breakpoint" => unimplemented!(), // halt miri
 196
 197             "copy" |
 198             "copy_nonoverlapping" => {
 199                 let elem_ty = substs.type_at(0);
 200                 let elem_layout = this.layout_of(elem_ty)?;
 201                 let elem_size = elem_layout.size.bytes();
 202                 let count = this.read_scalar(args[2])?.to_usize(this)?;
 203                 let elem_align = elem_layout.align.abi;
 204
 205                 let size = Size::from_bytes(count * elem_size);
 206                 let src = this.read_scalar(args[0])?.not_undef()?;
 207                 let src = this.memory().check_ptr_access(src, size, elem_align)?;
 208                 let dest = this.read_scalar(args[1])?.not_undef()?;
 209                 let dest = this.memory().check_ptr_access(dest, size, elem_align)?;
 210
 211                 if let (Some(src), Some(dest)) = (src, dest) {
 212                     this.memory_mut().copy(
 213                         src,
 214                         dest,
 215                         size,
 216                         intrinsic_name.ends_with("_nonoverlapping"),
 217                     )?;
 218                 }
 219             }
 220
 221             "discriminant_value" => {
 222                 let place = this.deref_operand(args[0])?;
 223                 let discr_val = this.read_discriminant(place.into())?.0;
 224                 this.write_scalar(Scalar::from_uint(discr_val, dest.layout.size), dest)?;
 225             }
 226
 227             "sinf32" | "fabsf32" | "cosf32" | "sqrtf32" | "expf32" | "exp2f32" | "logf32" |
 228             "log10f32" | "log2f32" | "floorf32" | "ceilf32" | "truncf32" => {
 229                 // FIXME: Using host floats.
 230                 let f = f32::from_bits(this.read_scalar(args[0])?.to_u32()?);
 231                 let f = match intrinsic_name.get() {
 232                     "sinf32" => f.sin(),
 233                     "fabsf32" => f.abs(),
 234                     "cosf32" => f.cos(),
 235                     "sqrtf32" => f.sqrt(),
 236                     "expf32" => f.exp(),
 237                     "exp2f32" => f.exp2(),
 238                     "logf32" => f.ln(),
 239                     "log10f32" => f.log10(),
 240                     "log2f32" => f.log2(),
 241                     "floorf32" => f.floor(),
 242                     "ceilf32" => f.ceil(),
 243                     "truncf32" => f.trunc(),
 244                     _ => bug!(),
 245                 };
 246                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 247             }
 248
 249             "sinf64" | "fabsf64" | "cosf64" | "sqrtf64" | "expf64" | "exp2f64" | "logf64" |
 250             "log10f64" | "log2f64" | "floorf64" | "ceilf64" | "truncf64" => {
 251                 // FIXME: Using host floats.
 252                 let f = f64::from_bits(this.read_scalar(args[0])?.to_u64()?);
 253                 let f = match intrinsic_name.get() {
 254                     "sinf64" => f.sin(),
 255                     "fabsf64" => f.abs(),
 256                     "cosf64" => f.cos(),
 257                     "sqrtf64" => f.sqrt(),
 258                     "expf64" => f.exp(),
 259                     "exp2f64" => f.exp2(),
 260                     "logf64" => f.ln(),
 261                     "log10f64" => f.log10(),
 262                     "log2f64" => f.log2(),
 263                     "floorf64" => f.floor(),
 264                     "ceilf64" => f.ceil(),
 265                     "truncf64" => f.trunc(),
 266                     _ => bug!(),
 267                 };
 268                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 269             }
 270
 271             "fadd_fast" | "fsub_fast" | "fmul_fast" | "fdiv_fast" | "frem_fast" => {
 272                 let a = this.read_immediate(args[0])?;
 273                 let b = this.read_immediate(args[1])?;
 274                 let op = match intrinsic_name.get() {
 275                     "fadd_fast" => mir::BinOp::Add,
 276                     "fsub_fast" => mir::BinOp::Sub,
 277                     "fmul_fast" => mir::BinOp::Mul,
 278                     "fdiv_fast" => mir::BinOp::Div,
 279                     "frem_fast" => mir::BinOp::Rem,
 280                     _ => bug!(),
 281                 };
 282                 this.binop_ignore_overflow(op, a, b, dest)?;
 283             }
 284
 285             "minnumf32" | "maxnumf32" => {
 286                 let a = this.read_scalar(args[0])?.to_f32()?;
 287                 let b = this.read_scalar(args[1])?.to_f32()?;
 288                 let res = if intrinsic_name.get().starts_with("min") {
 289                     a.min(b)
 290                 } else {
 291                     a.max(b)
 292                 };
 293                 this.write_scalar(Scalar::from_f32(res), dest)?;
 294             }
 295
 296             "minnumf64" | "maxnumf64" => {
 297                 let a = this.read_scalar(args[0])?.to_f64()?;
 298                 let b = this.read_scalar(args[1])?.to_f64()?;
 299                 let res = if intrinsic_name.get().starts_with("min") {
 300                     a.min(b)
 301                 } else {
 302                     a.max(b)
 303                 };
 304                 this.write_scalar(Scalar::from_f64(res), dest)?;
 305             }
 306
 307             "exact_div" => {
 308                 // Performs an exact division, resulting in undefined behavior where
 309                 // `x % y != 0` or `y == 0` or `x == T::min_value() && y == -1`
 310                 let a = this.read_immediate(args[0])?;
 311                 let b = this.read_immediate(args[1])?;
 312                 // check x % y != 0
 313                 if this.binary_op(mir::BinOp::Rem, a, b)?.0.to_bits(dest.layout.size)? != 0 {
 314                     // Check if `b` is -1, which is the "min_value / -1" case.
 315                     let minus1 = Scalar::from_int(-1, dest.layout.size);
 316                     return Err(if b.to_scalar().unwrap() == minus1 {
 317                         err_ub!(Ub(format!("exact_div: result of dividing MIN by -1 cannot be represented")))
 318                     } else {
 319                         err_ub!(Ub(format!("exact_div: {:?} cannot be divided by {:?} without remainder", *a, *b)))
 320                     }.into());
 321                 }
 322                 this.binop_ignore_overflow(mir::BinOp::Div, a, b, dest)?;
 323             },
 324
 325             "forget" => {}
 326
 327             "likely" | "unlikely" => {
 328                 // These just return their argument
 329                 let b = this.read_immediate(args[0])?;
 330                 this.write_immediate(*b, dest)?;
 331             }
 332
 333             "init" => {
 334                 // Check fast path: we don't want to force an allocation in case the destination is a simple value,
 335                 // but we also do not want to create a new allocation with 0s and then copy that over.
 336                 // FIXME: We do not properly validate in case of ZSTs and when doing it in memory!
 337                 // However, this only affects direct calls of the intrinsic; calls to the stable
 338                 // functions wrapping them do get their validation.
 339                 // FIXME: should we check that the destination pointer is aligned even for ZSTs?
 340                 if !dest.layout.is_zst() {
 341                     match dest.layout.abi {
 342                         layout::Abi::Scalar(ref s) => {
 343                             let x = Scalar::from_int(0, s.value.size(this));
 344                             this.write_scalar(x, dest)?;
 345                         }
 346                         layout::Abi::ScalarPair(ref s1, ref s2) => {
 347                             let x = Scalar::from_int(0, s1.value.size(this));
 348                             let y = Scalar::from_int(0, s2.value.size(this));
 349                             this.write_immediate(Immediate::ScalarPair(x.into(), y.into()), dest)?;
 350                         }
 351                         _ => {
 352                             // Do it in memory
 353                             let mplace = this.force_allocation(dest)?;
 354                             assert!(mplace.meta.is_none());
 355                             // not a zst, must be valid pointer
 356                             let ptr = mplace.ptr.to_ptr()?;
 357                             this.memory_mut().get_mut(ptr.alloc_id)?.write_repeat(tcx, ptr, 0, dest.layout.size)?;
 358                         }
 359                     }
 360                 }
 361             }
 362
 363             "pref_align_of" => {
 364                 let ty = substs.type_at(0);
 365                 let layout = this.layout_of(ty)?;
 366                 let align = layout.align.pref.bytes();
 367                 let ptr_size = this.pointer_size();
 368                 let align_val = Scalar::from_uint(align as u128, ptr_size);
 369                 this.write_scalar(align_val, dest)?;
 370             }
 371
 372             "move_val_init" => {
 373                 let place = this.deref_operand(args[0])?;
 374                 this.copy_op(args[1], place.into())?;
 375             }
 376
 377             "offset" => {
 378                 let offset = this.read_scalar(args[1])?.to_isize(this)?;
 379                 let ptr = this.read_scalar(args[0])?.not_undef()?;
 380                 let result_ptr = this.pointer_offset_inbounds(ptr, substs.type_at(0), offset)?;
 381                 this.write_scalar(result_ptr, dest)?;
 382             }
 383
 384             "panic_if_uninhabited" => {
 385                 let ty = substs.type_at(0);
 386                 let layout = this.layout_of(ty)?;
 387                 if layout.abi.is_uninhabited() {
 388                     throw_ub_format!("Trying to instantiate uninhabited type {}", ty)
 389                 }
 390             }
 391
 392             "powf32" => {
 393                 // FIXME: Using host floats.
 394                 let f = f32::from_bits(this.read_scalar(args[0])?.to_u32()?);
 395                 let f2 = f32::from_bits(this.read_scalar(args[1])?.to_u32()?);
 396                 this.write_scalar(
 397                     Scalar::from_u32(f.powf(f2).to_bits()),
 398                     dest,
 399                 )?;
 400             }
 401
 402             "powf64" => {
 403                 // FIXME: Using host floats.
 404                 let f = f64::from_bits(this.read_scalar(args[0])?.to_u64()?);
 405                 let f2 = f64::from_bits(this.read_scalar(args[1])?.to_u64()?);
 406                 this.write_scalar(
 407                     Scalar::from_u64(f.powf(f2).to_bits()),
 408                     dest,
 409                 )?;
 410             }
 411
 412             "fmaf32" => {
 413                 let a = this.read_scalar(args[0])?.to_f32()?;
 414                 let b = this.read_scalar(args[1])?.to_f32()?;
 415                 let c = this.read_scalar(args[2])?.to_f32()?;
 416                 let res = a.mul_add(b, c).value;
 417                 this.write_scalar(
 418                     Scalar::from_f32(res),
 419                     dest,
 420                 )?;
 421             }
 422
 423             "fmaf64" => {
 424                 let a = this.read_scalar(args[0])?.to_f64()?;
 425                 let b = this.read_scalar(args[1])?.to_f64()?;
 426                 let c = this.read_scalar(args[2])?.to_f64()?;
 427                 let res = a.mul_add(b, c).value;
 428                 this.write_scalar(
 429                     Scalar::from_f64(res),
 430                     dest,
 431                 )?;
 432             }
 433
 434             "powif32" => {
 435                 // FIXME: Using host floats.
 436                 let f = f32::from_bits(this.read_scalar(args[0])?.to_u32()?);
 437                 let i = this.read_scalar(args[1])?.to_i32()?;
 438                 this.write_scalar(
 439                     Scalar::from_u32(f.powi(i).to_bits()),
 440                     dest,
 441                 )?;
 442             }
 443
 444             "powif64" => {
 445                 // FIXME: Using host floats.
 446                 let f = f64::from_bits(this.read_scalar(args[0])?.to_u64()?);
 447                 let i = this.read_scalar(args[1])?.to_i32()?;
 448                 this.write_scalar(
 449                     Scalar::from_u64(f.powi(i).to_bits()),
 450                     dest,
 451                 )?;
 452             }
 453
 454             "size_of_val" => {
 455                 let mplace = this.deref_operand(args[0])?;
 456                 let (size, _) = this.size_and_align_of_mplace(mplace)?
 457                     .expect("size_of_val called on extern type");
 458                 let ptr_size = this.pointer_size();
 459                 this.write_scalar(
 460                     Scalar::from_uint(size.bytes() as u128, ptr_size),
 461                     dest,
 462                 )?;
 463             }
 464
 465             "min_align_of_val" |
 466             "align_of_val" => {
 467                 let mplace = this.deref_operand(args[0])?;
 468                 let (_, align) = this.size_and_align_of_mplace(mplace)?
 469                     .expect("size_of_val called on extern type");
 470                 let ptr_size = this.pointer_size();
 471                 this.write_scalar(
 472                     Scalar::from_uint(align.bytes(), ptr_size),
 473                     dest,
 474                 )?;
 475             }
 476
 477             "unchecked_div" => {
 478                 let l = this.read_immediate(args[0])?;
 479                 let r = this.read_immediate(args[1])?;
 480                 let rval = r.to_scalar()?.to_bits(args[1].layout.size)?;
 481                 if rval == 0 {
 482                     throw_ub_format!("Division by 0 in unchecked_div");
 483                 }
 484                 this.binop_ignore_overflow(
 485                     mir::BinOp::Div,
 486                     l,
 487                     r,
 488                     dest,
 489                 )?;
 490             }
 491
 492             "unchecked_rem" => {
 493                 let l = this.read_immediate(args[0])?;
 494                 let r = this.read_immediate(args[1])?;
 495                 let rval = r.to_scalar()?.to_bits(args[1].layout.size)?;
 496                 if rval == 0 {
 497                     throw_ub_format!("Division by 0 in unchecked_rem");
 498                 }
 499                 this.binop_ignore_overflow(
 500                     mir::BinOp::Rem,
 501                     l,
 502                     r,
 503                     dest,
 504                 )?;
 505             }
 506
 507             "unchecked_add" | "unchecked_sub" | "unchecked_mul" => {
 508                 let l = this.read_immediate(args[0])?;
 509                 let r = this.read_immediate(args[1])?;
 510                 let op = match intrinsic_name.get() {
 511                     "unchecked_add" => mir::BinOp::Add,
 512                     "unchecked_sub" => mir::BinOp::Sub,
 513                     "unchecked_mul" => mir::BinOp::Mul,
 514                     _ => bug!(),
 515                 };
 516                 let (res, overflowed) = this.binary_op(op, l, r)?;
 517                 if overflowed {
 518                     throw_ub_format!("Overflowing arithmetic in {}", intrinsic_name.get());
 519                 }
 520                 this.write_scalar(res, dest)?;
 521             }
 522
 523             "write_bytes" => {
 524                 let ty = substs.type_at(0);
 525                 let ty_layout = this.layout_of(ty)?;
 526                 let val_byte = this.read_scalar(args[1])?.to_u8()?;
 527                 let ptr = this.read_scalar(args[0])?.not_undef()?;
 528                 let count = this.read_scalar(args[2])?.to_usize(this)?;
 529                 let byte_count = ty_layout.size * count;
 530                 match this.memory().check_ptr_access(ptr, byte_count, ty_layout.align.abi)? {
 531                     Some(ptr) => {
 532                         this.memory_mut()
 533                             .get_mut(ptr.alloc_id)?
 534                             .write_repeat(tcx, ptr, val_byte, byte_count)?;
 535                     }
 536                     None => {
 537                         // Size is 0, nothing to do.
 538                     }
 539                 }
 540             }
 541
 542             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
 543         }
 544
 545         Ok(())
 546     }
 547 }