src/shims/intrinsics.rs

   1 use rustc_apfloat::Float;
   2 use rustc::mir;
   3 use rustc::mir::interpret::{InterpResult, PointerArithmetic};
   4 use rustc::ty::layout::{self, LayoutOf, Size};
   5 use rustc::ty;
   6
   7 use crate::{
   8     PlaceTy, OpTy, ImmTy, Immediate, Scalar, Tag,
   9     OperatorEvalContextExt
  10 };
  11
  12 impl<'mir, 'tcx> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  13 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  14     fn call_intrinsic(
  15         &mut self,
  16         instance: ty::Instance<'tcx>,
  17         args: &[OpTy<'tcx, Tag>],
  18         dest: PlaceTy<'tcx, Tag>,
  19     ) -> InterpResult<'tcx> {
  20         let this = self.eval_context_mut();
  21         if this.emulate_intrinsic(instance, args, dest)? {
  22             return Ok(());
  23         }
  24         let tcx = &{this.tcx.tcx};
  25         let substs = instance.substs;
  26
  27         // All these intrinsics take raw pointers, so if we access memory directly
  28         // (as opposed to through a place), we have to remember to erase any tag
  29         // that might still hang around!
  30
  31         let intrinsic_name = this.tcx.item_name(instance.def_id()).as_str();
  32         match intrinsic_name.get() {
  33             "arith_offset" => {
  34                 let offset = this.read_scalar(args[1])?.to_isize(this)?;
  35                 let ptr = this.read_scalar(args[0])?.not_undef()?;
  36
  37                 let pointee_ty = substs.type_at(0);
  38                 let pointee_size = this.layout_of(pointee_ty)?.size.bytes() as i64;
  39                 let offset = offset.overflowing_mul(pointee_size).0;
  40                 let result_ptr = ptr.ptr_wrapping_signed_offset(offset, this);
  41                 this.write_scalar(result_ptr, dest)?;
  42             }
  43
  44             "assume" => {
  45                 let cond = this.read_scalar(args[0])?.to_bool()?;
  46                 if !cond {
  47                     return err!(AssumptionNotHeld);
  48                 }
  49             }
  50
  51             "atomic_load" |
  52             "atomic_load_relaxed" |
  53             "atomic_load_acq" => {
  54                 let ptr = this.deref_operand(args[0])?;
  55                 let val = this.read_scalar(ptr.into())?; // make sure it fits into a scalar; otherwise it cannot be atomic
  56                 this.write_scalar(val, dest)?;
  57             }
  58
  59             "volatile_load" => {
  60                 let ptr = this.deref_operand(args[0])?;
  61                 this.copy_op(ptr.into(), dest)?;
  62             }
  63
  64             "atomic_store" |
  65             "atomic_store_relaxed" |
  66             "atomic_store_rel" => {
  67                 let ptr = this.deref_operand(args[0])?;
  68                 let val = this.read_scalar(args[1])?; // make sure it fits into a scalar; otherwise it cannot be atomic
  69                 this.write_scalar(val, ptr.into())?;
  70             }
  71
  72             "volatile_store" => {
  73                 let ptr = this.deref_operand(args[0])?;
  74                 this.copy_op(args[1], ptr.into())?;
  75             }
  76
  77             "atomic_fence_acq" => {
  78                 // we are inherently singlethreaded and singlecored, this is a nop
  79             }
  80
  81             _ if intrinsic_name.starts_with("atomic_xchg") => {
  82                 let ptr = this.deref_operand(args[0])?;
  83                 let new = this.read_scalar(args[1])?;
  84                 let old = this.read_scalar(ptr.into())?;
  85                 this.write_scalar(old, dest)?; // old value is returned
  86                 this.write_scalar(new, ptr.into())?;
  87             }
  88
  89             _ if intrinsic_name.starts_with("atomic_cxchg") => {
  90                 let ptr = this.deref_operand(args[0])?;
  91                 let expect_old = this.read_immediate(args[1])?; // read as immediate for the sake of `binary_op()`
  92                 let new = this.read_scalar(args[2])?;
  93                 let old = this.read_immediate(ptr.into())?; // read as immediate for the sake of `binary_op()`
  94                 // binary_op will bail if either of them is not a scalar
  95                 let (eq, _) = this.binary_op(mir::BinOp::Eq, old, expect_old)?;
  96                 let res = Immediate::ScalarPair(old.to_scalar_or_undef(), eq.into());
  97                 this.write_immediate(res, dest)?; // old value is returned
  98                 // update ptr depending on comparison
  99                 if eq.to_bool()? {
 100                     this.write_scalar(new, ptr.into())?;
 101                 }
 102             }
 103
 104             "atomic_or" |
 105             "atomic_or_acq" |
 106             "atomic_or_rel" |
 107             "atomic_or_acqrel" |
 108             "atomic_or_relaxed" |
 109             "atomic_xor" |
 110             "atomic_xor_acq" |
 111             "atomic_xor_rel" |
 112             "atomic_xor_acqrel" |
 113             "atomic_xor_relaxed" |
 114             "atomic_and" |
 115             "atomic_and_acq" |
 116             "atomic_and_rel" |
 117             "atomic_and_acqrel" |
 118             "atomic_and_relaxed" |
 119             "atomic_nand" |
 120             "atomic_nand_acq" |
 121             "atomic_nand_rel" |
 122             "atomic_nand_acqrel" |
 123             "atomic_nand_relaxed" |
 124             "atomic_xadd" |
 125             "atomic_xadd_acq" |
 126             "atomic_xadd_rel" |
 127             "atomic_xadd_acqrel" |
 128             "atomic_xadd_relaxed" |
 129             "atomic_xsub" |
 130             "atomic_xsub_acq" |
 131             "atomic_xsub_rel" |
 132             "atomic_xsub_acqrel" |
 133             "atomic_xsub_relaxed" => {
 134                 let ptr = this.deref_operand(args[0])?;
 135                 if !ptr.layout.ty.is_integral() {
 136                     return err!(Unimplemented(format!("Atomic arithmetic operations only work on integer types")));
 137                 }
 138                 let rhs = this.read_immediate(args[1])?;
 139                 let old = this.read_immediate(ptr.into())?;
 140                 this.write_immediate(*old, dest)?; // old value is returned
 141                 let (op, neg) = match intrinsic_name.split('_').nth(1).unwrap() {
 142                     "or" => (mir::BinOp::BitOr, false),
 143                     "xor" => (mir::BinOp::BitXor, false),
 144                     "and" => (mir::BinOp::BitAnd, false),
 145                     "xadd" => (mir::BinOp::Add, false),
 146                     "xsub" => (mir::BinOp::Sub, false),
 147                     "nand" => (mir::BinOp::BitAnd, true),
 148                     _ => bug!(),
 149                 };
 150                 // Atomics wrap around on overflow.
 151                 let (val, _overflowed) = this.binary_op(op, old, rhs)?;
 152                 let val = if neg {
 153                     this.unary_op(mir::UnOp::Not, ImmTy::from_scalar(val, old.layout))?
 154                 } else {
 155                     val
 156                 };
 157                 this.write_scalar(val, ptr.into())?;
 158             }
 159
 160             "breakpoint" => unimplemented!(), // halt miri
 161
 162             "copy" |
 163             "copy_nonoverlapping" => {
 164                 let elem_ty = substs.type_at(0);
 165                 let elem_layout = this.layout_of(elem_ty)?;
 166                 let elem_size = elem_layout.size.bytes();
 167                 let count = this.read_scalar(args[2])?.to_usize(this)?;
 168                 let elem_align = elem_layout.align.abi;
 169                 // erase tags: this is a raw ptr operation
 170                 let src = this.read_scalar(args[0])?.not_undef()?;
 171                 let dest = this.read_scalar(args[1])?.not_undef()?;
 172                 this.memory_mut().copy(
 173                     src,
 174                     elem_align,
 175                     dest,
 176                     elem_align,
 177                     Size::from_bytes(count * elem_size),
 178                     intrinsic_name.ends_with("_nonoverlapping"),
 179                 )?;
 180             }
 181
 182             "discriminant_value" => {
 183                 let place = this.deref_operand(args[0])?;
 184                 let discr_val = this.read_discriminant(place.into())?.0;
 185                 this.write_scalar(Scalar::from_uint(discr_val, dest.layout.size), dest)?;
 186             }
 187
 188             "sinf32" | "fabsf32" | "cosf32" | "sqrtf32" | "expf32" | "exp2f32" | "logf32" |
 189             "log10f32" | "log2f32" | "floorf32" | "ceilf32" | "truncf32" => {
 190                 // FIXME: Using host floats.
 191                 let f = f32::from_bits(this.read_scalar(args[0])?.to_u32()?);
 192                 let f = match intrinsic_name.get() {
 193                     "sinf32" => f.sin(),
 194                     "fabsf32" => f.abs(),
 195                     "cosf32" => f.cos(),
 196                     "sqrtf32" => f.sqrt(),
 197                     "expf32" => f.exp(),
 198                     "exp2f32" => f.exp2(),
 199                     "logf32" => f.ln(),
 200                     "log10f32" => f.log10(),
 201                     "log2f32" => f.log2(),
 202                     "floorf32" => f.floor(),
 203                     "ceilf32" => f.ceil(),
 204                     "truncf32" => f.trunc(),
 205                     _ => bug!(),
 206                 };
 207                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 208             }
 209
 210             "sinf64" | "fabsf64" | "cosf64" | "sqrtf64" | "expf64" | "exp2f64" | "logf64" |
 211             "log10f64" | "log2f64" | "floorf64" | "ceilf64" | "truncf64" => {
 212                 // FIXME: Using host floats.
 213                 let f = f64::from_bits(this.read_scalar(args[0])?.to_u64()?);
 214                 let f = match intrinsic_name.get() {
 215                     "sinf64" => f.sin(),
 216                     "fabsf64" => f.abs(),
 217                     "cosf64" => f.cos(),
 218                     "sqrtf64" => f.sqrt(),
 219                     "expf64" => f.exp(),
 220                     "exp2f64" => f.exp2(),
 221                     "logf64" => f.ln(),
 222                     "log10f64" => f.log10(),
 223                     "log2f64" => f.log2(),
 224                     "floorf64" => f.floor(),
 225                     "ceilf64" => f.ceil(),
 226                     "truncf64" => f.trunc(),
 227                     _ => bug!(),
 228                 };
 229                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 230             }
 231
 232             "fadd_fast" | "fsub_fast" | "fmul_fast" | "fdiv_fast" | "frem_fast" => {
 233                 let a = this.read_immediate(args[0])?;
 234                 let b = this.read_immediate(args[1])?;
 235                 let op = match intrinsic_name.get() {
 236                     "fadd_fast" => mir::BinOp::Add,
 237                     "fsub_fast" => mir::BinOp::Sub,
 238                     "fmul_fast" => mir::BinOp::Mul,
 239                     "fdiv_fast" => mir::BinOp::Div,
 240                     "frem_fast" => mir::BinOp::Rem,
 241                     _ => bug!(),
 242                 };
 243                 this.binop_ignore_overflow(op, a, b, dest)?;
 244             }
 245
 246             "minnumf32" | "maxnumf32" => {
 247                 let a = this.read_scalar(args[0])?.to_f32()?;
 248                 let b = this.read_scalar(args[1])?.to_f32()?;
 249                 let res = if intrinsic_name.get().starts_with("min") {
 250                     a.min(b)
 251                 } else {
 252                     a.max(b)
 253                 };
 254                 this.write_scalar(Scalar::from_f32(res), dest)?;
 255             }
 256
 257             "minnumf64" | "maxnumf64" => {
 258                 let a = this.read_scalar(args[0])?.to_f64()?;
 259                 let b = this.read_scalar(args[1])?.to_f64()?;
 260                 let res = if intrinsic_name.get().starts_with("min") {
 261                     a.min(b)
 262                 } else {
 263                     a.max(b)
 264                 };
 265                 this.write_scalar(Scalar::from_f64(res), dest)?;
 266             }
 267
 268             "exact_div" => {
 269                 // Performs an exact division, resulting in undefined behavior where
 270                 // `x % y != 0` or `y == 0` or `x == T::min_value() && y == -1`
 271                 let a = this.read_immediate(args[0])?;
 272                 let b = this.read_immediate(args[1])?;
 273                 // check x % y != 0
 274                 if this.binary_op(mir::BinOp::Rem, a, b)?.0.to_bits(dest.layout.size)? != 0 {
 275                     // Check if `b` is -1, which is the "min_value / -1" case.
 276                     let minus1 = Scalar::from_int(-1, dest.layout.size);
 277                     return if b.to_scalar().unwrap() == minus1 {
 278                         err!(Intrinsic(format!("exact_div: result of dividing MIN by -1 cannot be represented")))
 279                     } else {
 280                         err!(Intrinsic(format!("exact_div: {:?} cannot be divided by {:?} without remainder", *a, *b)))
 281                     };
 282                 }
 283                 this.binop_ignore_overflow(mir::BinOp::Div, a, b, dest)?;
 284             },
 285
 286             "forget" => {}
 287
 288             "likely" | "unlikely" => {
 289                 // These just return their argument
 290                 let b = this.read_immediate(args[0])?;
 291                 this.write_immediate(*b, dest)?;
 292             }
 293
 294             "init" => {
 295                 // Check fast path: we don't want to force an allocation in case the destination is a simple value,
 296                 // but we also do not want to create a new allocation with 0s and then copy that over.
 297                 // FIXME: We do not properly validate in case of ZSTs and when doing it in memory!
 298                 // However, this only affects direct calls of the intrinsic; calls to the stable
 299                 // functions wrapping them do get their validation.
 300                 // FIXME: should we check that the destination pointer is aligned even for ZSTs?
 301                 if !dest.layout.is_zst() {
 302                     match dest.layout.abi {
 303                         layout::Abi::Scalar(ref s) => {
 304                             let x = Scalar::from_int(0, s.value.size(this));
 305                             this.write_immediate(Immediate::Scalar(x.into()), dest)?;
 306                         }
 307                         layout::Abi::ScalarPair(ref s1, ref s2) => {
 308                             let x = Scalar::from_int(0, s1.value.size(this));
 309                             let y = Scalar::from_int(0, s2.value.size(this));
 310                             this.write_immediate(Immediate::ScalarPair(x.into(), y.into()), dest)?;
 311                         }
 312                         _ => {
 313                             // Do it in memory
 314                             let mplace = this.force_allocation(dest)?;
 315                             assert!(mplace.meta.is_none());
 316                             // not a zst, must be valid pointer
 317                             let ptr = mplace.ptr.to_ptr()?;
 318                             this.memory_mut().get_mut(ptr.alloc_id)?.write_repeat(tcx, ptr, 0, dest.layout.size)?;
 319                         }
 320                     }
 321                 }
 322             }
 323
 324             "pref_align_of" => {
 325                 let ty = substs.type_at(0);
 326                 let layout = this.layout_of(ty)?;
 327                 let align = layout.align.pref.bytes();
 328                 let ptr_size = this.pointer_size();
 329                 let align_val = Scalar::from_uint(align as u128, ptr_size);
 330                 this.write_scalar(align_val, dest)?;
 331             }
 332
 333             "move_val_init" => {
 334                 let ptr = this.deref_operand(args[0])?;
 335                 this.copy_op(args[1], ptr.into())?;
 336             }
 337
 338             "offset" => {
 339                 let offset = this.read_scalar(args[1])?.to_isize(this)?;
 340                 let ptr = this.read_scalar(args[0])?.not_undef()?;
 341                 let result_ptr = this.pointer_offset_inbounds(ptr, substs.type_at(0), offset)?;
 342                 this.write_scalar(result_ptr, dest)?;
 343             }
 344
 345             "panic_if_uninhabited" => {
 346                 let ty = substs.type_at(0);
 347                 let layout = this.layout_of(ty)?;
 348                 if layout.abi.is_uninhabited() {
 349                     return err!(Intrinsic(format!("Trying to instantiate uninhabited type {}", ty)))
 350                 }
 351             }
 352
 353             "powf32" => {
 354                 // FIXME: Using host floats.
 355                 let f = f32::from_bits(this.read_scalar(args[0])?.to_u32()?);
 356                 let f2 = f32::from_bits(this.read_scalar(args[1])?.to_u32()?);
 357                 this.write_scalar(
 358                     Scalar::from_u32(f.powf(f2).to_bits()),
 359                     dest,
 360                 )?;
 361             }
 362
 363             "powf64" => {
 364                 // FIXME: Using host floats.
 365                 let f = f64::from_bits(this.read_scalar(args[0])?.to_u64()?);
 366                 let f2 = f64::from_bits(this.read_scalar(args[1])?.to_u64()?);
 367                 this.write_scalar(
 368                     Scalar::from_u64(f.powf(f2).to_bits()),
 369                     dest,
 370                 )?;
 371             }
 372
 373             "fmaf32" => {
 374                 let a = this.read_scalar(args[0])?.to_f32()?;
 375                 let b = this.read_scalar(args[1])?.to_f32()?;
 376                 let c = this.read_scalar(args[2])?.to_f32()?;
 377                 let res = a.mul_add(b, c).value;
 378                 this.write_scalar(
 379                     Scalar::from_f32(res),
 380                     dest,
 381                 )?;
 382             }
 383
 384             "fmaf64" => {
 385                 let a = this.read_scalar(args[0])?.to_f64()?;
 386                 let b = this.read_scalar(args[1])?.to_f64()?;
 387                 let c = this.read_scalar(args[2])?.to_f64()?;
 388                 let res = a.mul_add(b, c).value;
 389                 this.write_scalar(
 390                     Scalar::from_f64(res),
 391                     dest,
 392                 )?;
 393             }
 394
 395             "powif32" => {
 396                 // FIXME: Using host floats.
 397                 let f = f32::from_bits(this.read_scalar(args[0])?.to_u32()?);
 398                 let i = this.read_scalar(args[1])?.to_i32()?;
 399                 this.write_scalar(
 400                     Scalar::from_u32(f.powi(i).to_bits()),
 401                     dest,
 402                 )?;
 403             }
 404
 405             "powif64" => {
 406                 // FIXME: Using host floats.
 407                 let f = f64::from_bits(this.read_scalar(args[0])?.to_u64()?);
 408                 let i = this.read_scalar(args[1])?.to_i32()?;
 409                 this.write_scalar(
 410                     Scalar::from_u64(f.powi(i).to_bits()),
 411                     dest,
 412                 )?;
 413             }
 414
 415             "size_of_val" => {
 416                 let mplace = this.deref_operand(args[0])?;
 417                 let (size, _) = this.size_and_align_of_mplace(mplace)?
 418                     .expect("size_of_val called on extern type");
 419                 let ptr_size = this.pointer_size();
 420                 this.write_scalar(
 421                     Scalar::from_uint(size.bytes() as u128, ptr_size),
 422                     dest,
 423                 )?;
 424             }
 425
 426             "min_align_of_val" |
 427             "align_of_val" => {
 428                 let mplace = this.deref_operand(args[0])?;
 429                 let (_, align) = this.size_and_align_of_mplace(mplace)?
 430                     .expect("size_of_val called on extern type");
 431                 let ptr_size = this.pointer_size();
 432                 this.write_scalar(
 433                     Scalar::from_uint(align.bytes(), ptr_size),
 434                     dest,
 435                 )?;
 436             }
 437
 438             "unchecked_div" => {
 439                 let l = this.read_immediate(args[0])?;
 440                 let r = this.read_immediate(args[1])?;
 441                 let rval = r.to_scalar()?.to_bits(args[1].layout.size)?;
 442                 if rval == 0 {
 443                     return err!(Intrinsic(format!("Division by 0 in unchecked_div")));
 444                 }
 445                 this.binop_ignore_overflow(
 446                     mir::BinOp::Div,
 447                     l,
 448                     r,
 449                     dest,
 450                 )?;
 451             }
 452
 453             "unchecked_rem" => {
 454                 let l = this.read_immediate(args[0])?;
 455                 let r = this.read_immediate(args[1])?;
 456                 let rval = r.to_scalar()?.to_bits(args[1].layout.size)?;
 457                 if rval == 0 {
 458                     return err!(Intrinsic(format!("Division by 0 in unchecked_rem")));
 459                 }
 460                 this.binop_ignore_overflow(
 461                     mir::BinOp::Rem,
 462                     l,
 463                     r,
 464                     dest,
 465                 )?;
 466             }
 467
 468             "unchecked_add" | "unchecked_sub" | "unchecked_mul" => {
 469                 let l = this.read_immediate(args[0])?;
 470                 let r = this.read_immediate(args[1])?;
 471                 let op = match intrinsic_name.get() {
 472                     "unchecked_add" => mir::BinOp::Add,
 473                     "unchecked_sub" => mir::BinOp::Sub,
 474                     "unchecked_mul" => mir::BinOp::Mul,
 475                     _ => bug!(),
 476                 };
 477                 let (res, overflowed) = this.binary_op(op, l, r)?;
 478                 if overflowed {
 479                     return err!(Intrinsic(format!("Overflowing arithmetic in {}", intrinsic_name.get())));
 480                 }
 481                 this.write_scalar(res, dest)?;
 482             }
 483
 484             "write_bytes" => {
 485                 let ty = substs.type_at(0);
 486                 let ty_layout = this.layout_of(ty)?;
 487                 let val_byte = this.read_scalar(args[1])?.to_u8()?;
 488                 let ptr = this.read_scalar(args[0])?.not_undef()?;
 489                 let count = this.read_scalar(args[2])?.to_usize(this)?;
 490                 let byte_count = ty_layout.size * count;
 491                 match this.memory().check_ptr_access(ptr, byte_count, ty_layout.align.abi)? {
 492                     Some(ptr) => {
 493                         this.memory_mut()
 494                             .get_mut(ptr.alloc_id)?
 495                             .write_repeat(tcx, ptr, val_byte, byte_count)?;
 496                     }
 497                     None => {
 498                         // Size is 0, nothing to do.
 499                     }
 500                 }
 501             }
 502
 503             name => return err!(Unimplemented(format!("unimplemented intrinsic: {}", name))),
 504         }
 505
 506         Ok(())
 507     }
 508 }