src/shims/intrinsics.rs

   1 use std::iter;
   2 use std::convert::TryFrom;
   3
   4 use rustc::mir;
   5 use rustc::mir::interpret::{InterpResult, PointerArithmetic};
   6 use rustc::ty;
   7 use rustc::ty::layout::{Align, LayoutOf, Size};
   8 use rustc_apfloat::Float;
   9 use rustc_span::source_map::Span;
  10
  11 use crate::*;
  12
  13 impl<'mir, 'tcx> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
  14 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
  15     fn call_intrinsic(
  16         &mut self,
  17         span: Span,
  18         instance: ty::Instance<'tcx>,
  19         args: &[OpTy<'tcx, Tag>],
  20         ret: Option<(PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
  21         unwind: Option<mir::BasicBlock>,
  22     ) -> InterpResult<'tcx> {
  23         let this = self.eval_context_mut();
  24         if this.emulate_intrinsic(span, instance, args, ret)? {
  25             return Ok(());
  26         }
  27         let tcx = &{ this.tcx.tcx };
  28         let substs = instance.substs;
  29
  30         // All these intrinsics take raw pointers, so if we access memory directly
  31         // (as opposed to through a place), we have to remember to erase any tag
  32         // that might still hang around!
  33         let intrinsic_name = &*tcx.item_name(instance.def_id()).as_str();
  34
  35         // First handle intrinsics without return place.
  36         let (dest, ret) = match ret {
  37             None => match intrinsic_name {
  38                 "miri_start_panic" => return this.handle_miri_start_panic(args, unwind),
  39                 _ => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
  40             },
  41             Some(p) => p,
  42         };
  43
  44         match intrinsic_name {
  45             "try" => return this.handle_try(args, dest, ret),
  46
  47             "arith_offset" => {
  48                 let offset = this.read_scalar(args[1])?.to_machine_isize(this)?;
  49                 let ptr = this.read_scalar(args[0])?.not_undef()?;
  50
  51                 let pointee_ty = substs.type_at(0);
  52                 let pointee_size = i64::try_from(this.layout_of(pointee_ty)?.size.bytes()).unwrap();
  53                 let offset = offset.overflowing_mul(pointee_size).0;
  54                 let result_ptr = ptr.ptr_wrapping_signed_offset(offset, this);
  55                 this.write_scalar(result_ptr, dest)?;
  56             }
  57
  58             "assume" => {
  59                 let cond = this.read_scalar(args[0])?.to_bool()?;
  60                 if !cond {
  61                     throw_ub_format!("`assume` intrinsic called with `false`");
  62                 }
  63             }
  64
  65             "volatile_load" => {
  66                 let place = this.deref_operand(args[0])?;
  67                 this.copy_op(place.into(), dest)?;
  68             }
  69
  70             "volatile_store" => {
  71                 let place = this.deref_operand(args[0])?;
  72                 this.copy_op(args[1], place.into())?;
  73             }
  74
  75             #[rustfmt::skip]
  76             | "atomic_load"
  77             | "atomic_load_relaxed"
  78             | "atomic_load_acq"
  79             => {
  80                 let place = this.deref_operand(args[0])?;
  81                 let val = this.read_scalar(place.into())?; // make sure it fits into a scalar; otherwise it cannot be atomic
  82
  83                 // Check alignment requirements. Atomics must always be aligned to their size,
  84                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
  85                 // be 8-aligned).
  86                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
  87                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
  88
  89                 this.write_scalar(val, dest)?;
  90             }
  91
  92             #[rustfmt::skip]
  93             | "atomic_store"
  94             | "atomic_store_relaxed"
  95             | "atomic_store_rel"
  96             => {
  97                 let place = this.deref_operand(args[0])?;
  98                 let val = this.read_scalar(args[1])?; // make sure it fits into a scalar; otherwise it cannot be atomic
  99
 100                 // Check alignment requirements. Atomics must always be aligned to their size,
 101                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 102                 // be 8-aligned).
 103                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 104                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 105
 106                 this.write_scalar(val, place.into())?;
 107             }
 108
 109             #[rustfmt::skip]
 110             | "atomic_fence_acq"
 111             | "atomic_fence_rel"
 112             | "atomic_fence_acqrel"
 113             | "atomic_fence"
 114             | "atomic_singlethreadfence_acq"
 115             | "atomic_singlethreadfence_rel"
 116             | "atomic_singlethreadfence_acqrel"
 117             | "atomic_singlethreadfence"
 118             => {
 119                 // we are inherently singlethreaded and singlecored, this is a nop
 120             }
 121
 122             _ if intrinsic_name.starts_with("atomic_xchg") => {
 123                 let place = this.deref_operand(args[0])?;
 124                 let new = this.read_scalar(args[1])?;
 125                 let old = this.read_scalar(place.into())?;
 126
 127                 // Check alignment requirements. Atomics must always be aligned to their size,
 128                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 129                 // be 8-aligned).
 130                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 131                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 132
 133                 this.write_scalar(old, dest)?; // old value is returned
 134                 this.write_scalar(new, place.into())?;
 135             }
 136
 137             _ if intrinsic_name.starts_with("atomic_cxchg") => {
 138                 let place = this.deref_operand(args[0])?;
 139                 let expect_old = this.read_immediate(args[1])?; // read as immediate for the sake of `binary_op()`
 140                 let new = this.read_scalar(args[2])?;
 141                 let old = this.read_immediate(place.into())?; // read as immediate for the sake of `binary_op()`
 142
 143                 // Check alignment requirements. Atomics must always be aligned to their size,
 144                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 145                 // be 8-aligned).
 146                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 147                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 148
 149                 // `binary_op` will bail if either of them is not a scalar.
 150                 let eq = this.overflowing_binary_op(mir::BinOp::Eq, old, expect_old)?.0;
 151                 let res = Immediate::ScalarPair(old.to_scalar_or_undef(), eq.into());
 152                 // Return old value.
 153                 this.write_immediate(res, dest)?;
 154                 // Update ptr depending on comparison.
 155                 if eq.to_bool()? {
 156                     this.write_scalar(new, place.into())?;
 157                 }
 158             }
 159
 160             #[rustfmt::skip]
 161             | "atomic_or"
 162             | "atomic_or_acq"
 163             | "atomic_or_rel"
 164             | "atomic_or_acqrel"
 165             | "atomic_or_relaxed"
 166             | "atomic_xor"
 167             | "atomic_xor_acq"
 168             | "atomic_xor_rel"
 169             | "atomic_xor_acqrel"
 170             | "atomic_xor_relaxed"
 171             | "atomic_and"
 172             | "atomic_and_acq"
 173             | "atomic_and_rel"
 174             | "atomic_and_acqrel"
 175             | "atomic_and_relaxed"
 176             | "atomic_nand"
 177             | "atomic_nand_acq"
 178             | "atomic_nand_rel"
 179             | "atomic_nand_acqrel"
 180             | "atomic_nand_relaxed"
 181             | "atomic_xadd"
 182             | "atomic_xadd_acq"
 183             | "atomic_xadd_rel"
 184             | "atomic_xadd_acqrel"
 185             | "atomic_xadd_relaxed"
 186             | "atomic_xsub"
 187             | "atomic_xsub_acq"
 188             | "atomic_xsub_rel"
 189             | "atomic_xsub_acqrel"
 190             | "atomic_xsub_relaxed"
 191             => {
 192                 let place = this.deref_operand(args[0])?;
 193                 if !place.layout.ty.is_integral() {
 194                     bug!("Atomic arithmetic operations only work on integer types");
 195                 }
 196                 let rhs = this.read_immediate(args[1])?;
 197                 let old = this.read_immediate(place.into())?;
 198
 199                 // Check alignment requirements. Atomics must always be aligned to their size,
 200                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
 201                 // be 8-aligned).
 202                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
 203                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
 204
 205                 this.write_immediate(*old, dest)?; // old value is returned
 206                 let (op, neg) = match intrinsic_name.split('_').nth(1).unwrap() {
 207                     "or" => (mir::BinOp::BitOr, false),
 208                     "xor" => (mir::BinOp::BitXor, false),
 209                     "and" => (mir::BinOp::BitAnd, false),
 210                     "xadd" => (mir::BinOp::Add, false),
 211                     "xsub" => (mir::BinOp::Sub, false),
 212                     "nand" => (mir::BinOp::BitAnd, true),
 213                     _ => bug!(),
 214                 };
 215                 // Atomics wrap around on overflow.
 216                 let val = this.binary_op(op, old, rhs)?;
 217                 let val = if neg { this.unary_op(mir::UnOp::Not, val)? } else { val };
 218                 this.write_immediate(*val, place.into())?;
 219             }
 220
 221             "breakpoint" => unimplemented!(), // halt miri
 222
 223             #[rustfmt::skip]
 224             | "copy"
 225             | "copy_nonoverlapping"
 226             => {
 227                 let elem_ty = substs.type_at(0);
 228                 let elem_layout = this.layout_of(elem_ty)?;
 229                 let elem_size = elem_layout.size.bytes();
 230                 let count = this.read_scalar(args[2])?.to_machine_usize(this)?;
 231                 let elem_align = elem_layout.align.abi;
 232
 233                 let size = Size::from_bytes(count) * elem_size;
 234                 let src = this.read_scalar(args[0])?.not_undef()?;
 235                 let src = this.memory.check_ptr_access(src, size, elem_align)?;
 236                 let dest = this.read_scalar(args[1])?.not_undef()?;
 237                 let dest = this.memory.check_ptr_access(dest, size, elem_align)?;
 238
 239                 if let (Some(src), Some(dest)) = (src, dest) {
 240                     this.memory.copy(
 241                         src,
 242                         dest,
 243                         size,
 244                         intrinsic_name.ends_with("_nonoverlapping"),
 245                     )?;
 246                 }
 247             }
 248
 249             #[rustfmt::skip]
 250             | "sinf32"
 251             | "fabsf32"
 252             | "cosf32"
 253             | "sqrtf32"
 254             | "expf32"
 255             | "exp2f32"
 256             | "logf32"
 257             | "log10f32"
 258             | "log2f32"
 259             | "floorf32"
 260             | "ceilf32"
 261             | "truncf32"
 262             | "roundf32"
 263             => {
 264                 // FIXME: Using host floats.
 265                 let f = f32::from_bits(this.read_scalar(args[0])?.to_u32()?);
 266                 let f = match intrinsic_name {
 267                     "sinf32" => f.sin(),
 268                     "fabsf32" => f.abs(),
 269                     "cosf32" => f.cos(),
 270                     "sqrtf32" => f.sqrt(),
 271                     "expf32" => f.exp(),
 272                     "exp2f32" => f.exp2(),
 273                     "logf32" => f.ln(),
 274                     "log10f32" => f.log10(),
 275                     "log2f32" => f.log2(),
 276                     "floorf32" => f.floor(),
 277                     "ceilf32" => f.ceil(),
 278                     "truncf32" => f.trunc(),
 279                     "roundf32" => f.round(),
 280                     _ => bug!(),
 281                 };
 282                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
 283             }
 284
 285             #[rustfmt::skip]
 286             | "sinf64"
 287             | "fabsf64"
 288             | "cosf64"
 289             | "sqrtf64"
 290             | "expf64"
 291             | "exp2f64"
 292             | "logf64"
 293             | "log10f64"
 294             | "log2f64"
 295             | "floorf64"
 296             | "ceilf64"
 297             | "truncf64"
 298             | "roundf64"
 299             => {
 300                 // FIXME: Using host floats.
 301                 let f = f64::from_bits(this.read_scalar(args[0])?.to_u64()?);
 302                 let f = match intrinsic_name {
 303                     "sinf64" => f.sin(),
 304                     "fabsf64" => f.abs(),
 305                     "cosf64" => f.cos(),
 306                     "sqrtf64" => f.sqrt(),
 307                     "expf64" => f.exp(),
 308                     "exp2f64" => f.exp2(),
 309                     "logf64" => f.ln(),
 310                     "log10f64" => f.log10(),
 311                     "log2f64" => f.log2(),
 312                     "floorf64" => f.floor(),
 313                     "ceilf64" => f.ceil(),
 314                     "truncf64" => f.trunc(),
 315                     "roundf64" => f.round(),
 316                     _ => bug!(),
 317                 };
 318                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
 319             }
 320
 321             #[rustfmt::skip]
 322             | "fadd_fast"
 323             | "fsub_fast"
 324             | "fmul_fast"
 325             | "fdiv_fast"
 326             | "frem_fast"
 327             => {
 328                 let a = this.read_immediate(args[0])?;
 329                 let b = this.read_immediate(args[1])?;
 330                 let op = match intrinsic_name {
 331                     "fadd_fast" => mir::BinOp::Add,
 332                     "fsub_fast" => mir::BinOp::Sub,
 333                     "fmul_fast" => mir::BinOp::Mul,
 334                     "fdiv_fast" => mir::BinOp::Div,
 335                     "frem_fast" => mir::BinOp::Rem,
 336                     _ => bug!(),
 337                 };
 338                 this.binop_ignore_overflow(op, a, b, dest)?;
 339             }
 340
 341             #[rustfmt::skip]
 342             | "minnumf32"
 343             | "maxnumf32"
 344             | "copysignf32"
 345             => {
 346                 let a = this.read_scalar(args[0])?.to_f32()?;
 347                 let b = this.read_scalar(args[1])?.to_f32()?;
 348                 let res = match intrinsic_name {
 349                     "minnumf32" => a.min(b),
 350                     "maxnumf32" => a.max(b),
 351                     "copysignf32" => a.copy_sign(b),
 352                     _ => bug!(),
 353                 };
 354                 this.write_scalar(Scalar::from_f32(res), dest)?;
 355             }
 356
 357             #[rustfmt::skip]
 358             | "minnumf64"
 359             | "maxnumf64"
 360             | "copysignf64"
 361             => {
 362                 let a = this.read_scalar(args[0])?.to_f64()?;
 363                 let b = this.read_scalar(args[1])?.to_f64()?;
 364                 let res = match intrinsic_name {
 365                     "minnumf64" => a.min(b),
 366                     "maxnumf64" => a.max(b),
 367                     "copysignf64" => a.copy_sign(b),
 368                     _ => bug!(),
 369                 };
 370                 this.write_scalar(Scalar::from_f64(res), dest)?;
 371             }
 372
 373             "exact_div" =>
 374                 this.exact_div(this.read_immediate(args[0])?, this.read_immediate(args[1])?, dest)?,
 375
 376             "forget" => {}
 377
 378             #[rustfmt::skip]
 379             | "likely"
 380             | "unlikely"
 381             => {
 382                 // These just return their argument
 383                 let b = this.read_immediate(args[0])?;
 384                 this.write_immediate(*b, dest)?;
 385             }
 386
 387             "pref_align_of" => {
 388                 let ty = substs.type_at(0);
 389                 let layout = this.layout_of(ty)?;
 390                 let align = layout.align.pref.bytes();
 391                 let ptr_size = this.pointer_size();
 392                 let align_val = Scalar::from_uint(align, ptr_size);
 393                 this.write_scalar(align_val, dest)?;
 394             }
 395
 396             "move_val_init" => {
 397                 let place = this.deref_operand(args[0])?;
 398                 this.copy_op(args[1], place.into())?;
 399             }
 400
 401             "offset" => {
 402                 let offset = this.read_scalar(args[1])?.to_machine_isize(this)?;
 403                 let ptr = this.read_scalar(args[0])?.not_undef()?;
 404                 let result_ptr = this.pointer_offset_inbounds(ptr, substs.type_at(0), offset)?;
 405                 this.write_scalar(result_ptr, dest)?;
 406             }
 407
 408             "assert_inhabited" |
 409             "assert_zero_valid" |
 410             "assert_uninit_valid" => {
 411                 let ty = substs.type_at(0);
 412                 let layout = this.layout_of(ty)?;
 413                 // Return here because we panicked instead of returning normally from the intrinsic.
 414                 if layout.abi.is_uninhabited() {
 415                     return this.start_panic(&format!("attempted to instantiate uninhabited type `{}`", ty), unwind);
 416                 }
 417                 if intrinsic_name == "assert_zero_valid" && !layout.might_permit_raw_init(this, /*zero:*/ true).unwrap() {
 418                     return this.start_panic(&format!("attempted to zero-initialize type `{}`, which is invalid", ty), unwind);
 419                 }
 420                 if intrinsic_name == "assert_uninit_valid" && !layout.might_permit_raw_init(this, /*zero:*/ false).unwrap() {
 421                     return this.start_panic(&format!("attempted to leave type `{}` uninitialized, which is invalid", ty), unwind);
 422                 }
 423             }
 424
 425             "powf32" => {
 426                 // FIXME: Using host floats.
 427                 let f = f32::from_bits(this.read_scalar(args[0])?.to_u32()?);
 428                 let f2 = f32::from_bits(this.read_scalar(args[1])?.to_u32()?);
 429                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
 430             }
 431
 432             "powf64" => {
 433                 // FIXME: Using host floats.
 434                 let f = f64::from_bits(this.read_scalar(args[0])?.to_u64()?);
 435                 let f2 = f64::from_bits(this.read_scalar(args[1])?.to_u64()?);
 436                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
 437             }
 438
 439             "fmaf32" => {
 440                 let a = this.read_scalar(args[0])?.to_f32()?;
 441                 let b = this.read_scalar(args[1])?.to_f32()?;
 442                 let c = this.read_scalar(args[2])?.to_f32()?;
 443                 let res = a.mul_add(b, c).value;
 444                 this.write_scalar(Scalar::from_f32(res), dest)?;
 445             }
 446
 447             "fmaf64" => {
 448                 let a = this.read_scalar(args[0])?.to_f64()?;
 449                 let b = this.read_scalar(args[1])?.to_f64()?;
 450                 let c = this.read_scalar(args[2])?.to_f64()?;
 451                 let res = a.mul_add(b, c).value;
 452                 this.write_scalar(Scalar::from_f64(res), dest)?;
 453             }
 454
 455             "powif32" => {
 456                 // FIXME: Using host floats.
 457                 let f = f32::from_bits(this.read_scalar(args[0])?.to_u32()?);
 458                 let i = this.read_scalar(args[1])?.to_i32()?;
 459                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
 460             }
 461
 462             "powif64" => {
 463                 // FIXME: Using host floats.
 464                 let f = f64::from_bits(this.read_scalar(args[0])?.to_u64()?);
 465                 let i = this.read_scalar(args[1])?.to_i32()?;
 466                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
 467             }
 468
 469             "size_of_val" => {
 470                 let mplace = this.deref_operand(args[0])?;
 471                 let (size, _) = this
 472                     .size_and_align_of_mplace(mplace)?
 473                     .expect("size_of_val called on extern type");
 474                 let ptr_size = this.pointer_size();
 475                 this.write_scalar(Scalar::from_uint(size.bytes(), ptr_size), dest)?;
 476             }
 477
 478             #[rustfmt::skip]
 479             | "min_align_of_val"
 480             | "align_of_val"
 481             => {
 482                 let mplace = this.deref_operand(args[0])?;
 483                 let (_, align) = this
 484                     .size_and_align_of_mplace(mplace)?
 485                     .expect("size_of_val called on extern type");
 486                 let ptr_size = this.pointer_size();
 487                 this.write_scalar(Scalar::from_uint(align.bytes(), ptr_size), dest)?;
 488             }
 489
 490             "write_bytes" => {
 491                 let ty = substs.type_at(0);
 492                 let ty_layout = this.layout_of(ty)?;
 493                 let val_byte = this.read_scalar(args[1])?.to_u8()?;
 494                 let ptr = this.read_scalar(args[0])?.not_undef()?;
 495                 let count = this.read_scalar(args[2])?.to_machine_usize(this)?;
 496                 let byte_count = ty_layout.size * count;
 497                 this.memory
 498                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
 499             }
 500
 501             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
 502         }
 503
 504         this.dump_place(*dest);
 505         this.go_to_block(ret);
 506         Ok(())
 507     }
 508 }