5 use rustc_attr as attr;
6 use rustc_ast::ast::FloatTy;
7 use rustc_middle::{mir, ty};
8 use rustc_middle::ty::layout::IntegerExt;
9 use rustc_apfloat::{Float, Round};
10 use rustc_target::abi::{Align, Integer, LayoutOf};
11 use rustc_span::symbol::sym;
14 use helpers::check_arg_count;
16 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
17 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
20 instance: ty::Instance<'tcx>,
21 args: &[OpTy<'tcx, Tag>],
22 ret: Option<(PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
23 _unwind: Option<mir::BasicBlock>,
24 ) -> InterpResult<'tcx> {
25 let this = self.eval_context_mut();
26 let intrinsic_name = this.tcx.item_name(instance.def_id());
27 // We want to overwrite some of the intrinsic implementations that CTFE uses.
28 let prefer_miri_intrinsic = match intrinsic_name {
29 sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => true,
33 if !prefer_miri_intrinsic && this.emulate_intrinsic(instance, args, ret)? {
37 // All supported intrinsics have a return place.
38 let intrinsic_name = &*intrinsic_name.as_str();
39 let (dest, ret) = match ret {
40 None => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
44 // Then handle terminating intrinsics.
45 match intrinsic_name {
46 // Miri overwriting CTFE intrinsics.
47 "ptr_guaranteed_eq" => {
48 let &[left, right] = check_arg_count(args)?;
49 let left = this.read_immediate(left)?;
50 let right = this.read_immediate(right)?;
51 this.binop_ignore_overflow(mir::BinOp::Eq, left, right, dest)?;
53 "ptr_guaranteed_ne" => {
54 let &[left, right] = check_arg_count(args)?;
55 let left = this.read_immediate(left)?;
56 let right = this.read_immediate(right)?;
57 this.binop_ignore_overflow(mir::BinOp::Ne, left, right, dest)?;
60 // Raw memory accesses
63 | "copy_nonoverlapping"
65 let &[src, dest, count] = check_arg_count(args)?;
66 let elem_ty = instance.substs.type_at(0);
67 let elem_layout = this.layout_of(elem_ty)?;
68 let count = this.read_scalar(count)?.to_machine_usize(this)?;
69 let elem_align = elem_layout.align.abi;
71 let size = elem_layout.size.checked_mul(count, this)
72 .ok_or_else(|| err_ub_format!("overflow computing total size of `{}`", intrinsic_name))?;
73 let src = this.read_scalar(src)?.check_init()?;
74 let src = this.memory.check_ptr_access(src, size, elem_align)?;
75 let dest = this.read_scalar(dest)?.check_init()?;
76 let dest = this.memory.check_ptr_access(dest, size, elem_align)?;
78 if let (Some(src), Some(dest)) = (src, dest) {
83 intrinsic_name.ends_with("_nonoverlapping"),
89 let &[place, dest] = check_arg_count(args)?;
90 let place = this.deref_operand(place)?;
91 this.copy_op(dest, place.into())?;
95 let &[place] = check_arg_count(args)?;
96 let place = this.deref_operand(place)?;
97 this.copy_op(place.into(), dest)?;
100 let &[place, dest] = check_arg_count(args)?;
101 let place = this.deref_operand(place)?;
102 this.copy_op(dest, place.into())?;
106 let &[ptr, val_byte, count] = check_arg_count(args)?;
107 let ty = instance.substs.type_at(0);
108 let ty_layout = this.layout_of(ty)?;
109 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
110 let ptr = this.read_scalar(ptr)?.check_init()?;
111 let count = this.read_scalar(count)?.to_machine_usize(this)?;
112 let byte_count = ty_layout.size.checked_mul(count, this)
113 .ok_or_else(|| err_ub_format!("overflow computing total size of `write_bytes`"))?;
115 .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
118 // Floating-point operations
134 let &[f] = check_arg_count(args)?;
135 // FIXME: Using host floats.
136 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
137 let f = match intrinsic_name {
139 "fabsf32" => f.abs(),
141 "sqrtf32" => f.sqrt(),
143 "exp2f32" => f.exp2(),
145 "log10f32" => f.log10(),
146 "log2f32" => f.log2(),
147 "floorf32" => f.floor(),
148 "ceilf32" => f.ceil(),
149 "truncf32" => f.trunc(),
150 "roundf32" => f.round(),
153 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
171 let &[f] = check_arg_count(args)?;
172 // FIXME: Using host floats.
173 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
174 let f = match intrinsic_name {
176 "fabsf64" => f.abs(),
178 "sqrtf64" => f.sqrt(),
180 "exp2f64" => f.exp2(),
182 "log10f64" => f.log10(),
183 "log2f64" => f.log2(),
184 "floorf64" => f.floor(),
185 "ceilf64" => f.ceil(),
186 "truncf64" => f.trunc(),
187 "roundf64" => f.round(),
190 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
200 let &[a, b] = check_arg_count(args)?;
201 let a = this.read_immediate(a)?;
202 let b = this.read_immediate(b)?;
203 let op = match intrinsic_name {
204 "fadd_fast" => mir::BinOp::Add,
205 "fsub_fast" => mir::BinOp::Sub,
206 "fmul_fast" => mir::BinOp::Mul,
207 "fdiv_fast" => mir::BinOp::Div,
208 "frem_fast" => mir::BinOp::Rem,
211 this.binop_ignore_overflow(op, a, b, dest)?;
219 let &[a, b] = check_arg_count(args)?;
220 let a = this.read_scalar(a)?.to_f32()?;
221 let b = this.read_scalar(b)?.to_f32()?;
222 let res = match intrinsic_name {
223 "minnumf32" => a.min(b),
224 "maxnumf32" => a.max(b),
225 "copysignf32" => a.copy_sign(b),
228 this.write_scalar(Scalar::from_f32(res), dest)?;
236 let &[a, b] = check_arg_count(args)?;
237 let a = this.read_scalar(a)?.to_f64()?;
238 let b = this.read_scalar(b)?.to_f64()?;
239 let res = match intrinsic_name {
240 "minnumf64" => a.min(b),
241 "maxnumf64" => a.max(b),
242 "copysignf64" => a.copy_sign(b),
245 this.write_scalar(Scalar::from_f64(res), dest)?;
249 let &[f, f2] = check_arg_count(args)?;
250 // FIXME: Using host floats.
251 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
252 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
253 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
257 let &[f, f2] = check_arg_count(args)?;
258 // FIXME: Using host floats.
259 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
260 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
261 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
265 let &[a, b, c] = check_arg_count(args)?;
266 let a = this.read_scalar(a)?.to_f32()?;
267 let b = this.read_scalar(b)?.to_f32()?;
268 let c = this.read_scalar(c)?.to_f32()?;
269 let res = a.mul_add(b, c).value;
270 this.write_scalar(Scalar::from_f32(res), dest)?;
274 let &[a, b, c] = check_arg_count(args)?;
275 let a = this.read_scalar(a)?.to_f64()?;
276 let b = this.read_scalar(b)?.to_f64()?;
277 let c = this.read_scalar(c)?.to_f64()?;
278 let res = a.mul_add(b, c).value;
279 this.write_scalar(Scalar::from_f64(res), dest)?;
283 let &[f, i] = check_arg_count(args)?;
284 // FIXME: Using host floats.
285 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
286 let i = this.read_scalar(i)?.to_i32()?;
287 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
291 let &[f, i] = check_arg_count(args)?;
292 // FIXME: Using host floats.
293 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
294 let i = this.read_scalar(i)?.to_i32()?;
295 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
298 "float_to_int_unchecked" => {
299 let &[val] = check_arg_count(args)?;
300 let val = this.read_immediate(val)?;
302 let res = match val.layout.ty.kind {
303 ty::Float(FloatTy::F32) => {
304 this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?
306 ty::Float(FloatTy::F64) => {
307 this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?
309 _ => bug!("`float_to_int_unchecked` called with non-float input type {:?}", val.layout.ty),
312 this.write_scalar(res, dest)?;
318 | "atomic_load_relaxed"
321 let &[place] = check_arg_count(args)?;
322 let place = this.deref_operand(place)?;
323 let val = this.read_scalar(place.into())?; // make sure it fits into a scalar; otherwise it cannot be atomic
325 // Check alignment requirements. Atomics must always be aligned to their size,
326 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
328 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
329 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
331 this.write_scalar(val, dest)?;
336 | "atomic_store_relaxed"
339 let &[place, val] = check_arg_count(args)?;
340 let place = this.deref_operand(place)?;
341 let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
343 // Check alignment requirements. Atomics must always be aligned to their size,
344 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
346 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
347 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
349 this.write_scalar(val, place.into())?;
355 | "atomic_fence_acqrel"
357 | "atomic_singlethreadfence_acq"
358 | "atomic_singlethreadfence_rel"
359 | "atomic_singlethreadfence_acqrel"
360 | "atomic_singlethreadfence"
362 let &[] = check_arg_count(args)?;
363 // FIXME: this will become relevant once we try to detect data races.
366 _ if intrinsic_name.starts_with("atomic_xchg") => {
367 let &[place, new] = check_arg_count(args)?;
368 let place = this.deref_operand(place)?;
369 let new = this.read_scalar(new)?;
370 let old = this.read_scalar(place.into())?;
372 // Check alignment requirements. Atomics must always be aligned to their size,
373 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
375 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
376 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
378 this.write_scalar(old, dest)?; // old value is returned
379 this.write_scalar(new, place.into())?;
382 _ if intrinsic_name.starts_with("atomic_cxchg") => {
383 let &[place, expect_old, new] = check_arg_count(args)?;
384 let place = this.deref_operand(place)?;
385 let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
386 let new = this.read_scalar(new)?;
387 let old = this.read_immediate(place.into())?; // read as immediate for the sake of `binary_op()`
389 // Check alignment requirements. Atomics must always be aligned to their size,
390 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
392 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
393 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
395 // `binary_op` will bail if either of them is not a scalar.
396 let eq = this.overflowing_binary_op(mir::BinOp::Eq, old, expect_old)?.0;
397 let res = Immediate::ScalarPair(old.to_scalar_or_uninit(), eq.into());
399 this.write_immediate(res, dest)?;
400 // Update ptr depending on comparison.
402 this.write_scalar(new, place.into())?;
411 | "atomic_or_relaxed"
415 | "atomic_xor_acqrel"
416 | "atomic_xor_relaxed"
420 | "atomic_and_acqrel"
421 | "atomic_and_relaxed"
425 | "atomic_nand_acqrel"
426 | "atomic_nand_relaxed"
430 | "atomic_xadd_acqrel"
431 | "atomic_xadd_relaxed"
435 | "atomic_xsub_acqrel"
436 | "atomic_xsub_relaxed"
438 let &[place, rhs] = check_arg_count(args)?;
439 let place = this.deref_operand(place)?;
440 if !place.layout.ty.is_integral() {
441 bug!("Atomic arithmetic operations only work on integer types");
443 let rhs = this.read_immediate(rhs)?;
444 let old = this.read_immediate(place.into())?;
446 // Check alignment requirements. Atomics must always be aligned to their size,
447 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
449 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
450 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
452 this.write_immediate(*old, dest)?; // old value is returned
453 let (op, neg) = match intrinsic_name.split('_').nth(1).unwrap() {
454 "or" => (mir::BinOp::BitOr, false),
455 "xor" => (mir::BinOp::BitXor, false),
456 "and" => (mir::BinOp::BitAnd, false),
457 "xadd" => (mir::BinOp::Add, false),
458 "xsub" => (mir::BinOp::Sub, false),
459 "nand" => (mir::BinOp::BitAnd, true),
462 // Atomics wrap around on overflow.
463 let val = this.binary_op(op, old, rhs)?;
464 let val = if neg { this.unary_op(mir::UnOp::Not, val)? } else { val };
465 this.write_immediate(*val, place.into())?;
468 // Query type information
470 "assert_zero_valid" |
471 "assert_uninit_valid" => {
472 let &[] = check_arg_count(args)?;
473 let ty = instance.substs.type_at(0);
474 let layout = this.layout_of(ty)?;
475 // Abort here because the caller might not be panic safe.
476 if layout.abi.is_uninhabited() {
477 throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to instantiate uninhabited type `{}`", ty))))
479 if intrinsic_name == "assert_zero_valid" && !layout.might_permit_raw_init(this, /*zero:*/ true).unwrap() {
480 throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to zero-initialize type `{}`, which is invalid", ty))))
482 if intrinsic_name == "assert_uninit_valid" && !layout.might_permit_raw_init(this, /*zero:*/ false).unwrap() {
483 throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to leave type `{}` uninitialized, which is invalid", ty))))
487 "min_align_of_val" => {
488 let &[mplace] = check_arg_count(args)?;
489 let mplace = this.deref_operand(mplace)?;
490 let (_, align) = this
491 .size_and_align_of_mplace(mplace)?
492 .expect("size_of_val called on extern type");
493 this.write_scalar(Scalar::from_machine_usize(align.bytes(), this), dest)?;
497 let &[mplace] = check_arg_count(args)?;
498 let mplace = this.deref_operand(mplace)?;
500 .size_and_align_of_mplace(mplace)?
501 .expect("size_of_val called on extern type");
502 this.write_scalar(Scalar::from_machine_usize(size.bytes(), this), dest)?;
507 let &[cond] = check_arg_count(args)?;
508 let cond = this.read_scalar(cond)?.check_init()?.to_bool()?;
510 throw_ub_format!("`assume` intrinsic called with `false`");
515 let &[num, denom] = check_arg_count(args)?;
516 this.exact_div(this.read_immediate(num)?, this.read_immediate(denom)?, dest)?;
520 // We get an argument... and forget about it.
521 let &[_] = check_arg_count(args)?;
524 "try" => return this.handle_try(args, dest, ret),
526 name => throw_unsup_format!("unimplemented intrinsic: {}", name),
529 trace!("{:?}", this.dump_place(*dest));
530 this.go_to_block(ret);
534 fn float_to_int_unchecked<F>(
537 dest_ty: ty::Ty<'tcx>,
538 ) -> InterpResult<'tcx, Scalar<Tag>>
540 F: Float + Into<Scalar<Tag>>
542 let this = self.eval_context_ref();
544 // Step 1: cut off the fractional part of `f`. The result of this is
545 // guaranteed to be precisely representable in IEEE floats.
546 let f = f.round_to_integral(Round::TowardZero).value;
548 // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
549 Ok(match dest_ty.kind {
552 let size = Integer::from_attr(this, attr::IntType::UnsignedInt(t)).size();
553 let res = f.to_u128(size.bits_usize());
554 if res.status.is_empty() {
555 // No status flags means there was no further rounding or other loss of precision.
556 Scalar::from_uint(res.value, size)
558 // `f` was not representable in this integer type.
560 "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
567 let size = Integer::from_attr(this, attr::IntType::SignedInt(t)).size();
568 let res = f.to_i128(size.bits_usize());
569 if res.status.is_empty() {
570 // No status flags means there was no further rounding or other loss of precision.
571 Scalar::from_int(res.value, size)
573 // `f` was not representable in this integer type.
575 "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
581 _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),