3 use rustc_attr as attr;
4 use rustc_ast::ast::FloatTy;
5 use rustc_middle::{mir, ty};
6 use rustc_middle::ty::layout::IntegerExt;
7 use rustc_apfloat::{Float, Round};
8 use rustc_target::abi::{Align, Integer, LayoutOf};
11 use helpers::check_arg_count;
13 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
14 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
17 instance: ty::Instance<'tcx>,
18 args: &[OpTy<'tcx, Tag>],
19 ret: Option<(PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
20 unwind: Option<mir::BasicBlock>,
21 ) -> InterpResult<'tcx> {
22 let this = self.eval_context_mut();
23 if this.emulate_intrinsic(instance, args, ret)? {
26 let substs = instance.substs;
28 // All these intrinsics take raw pointers, so if we access memory directly
29 // (as opposed to through a place), we have to remember to erase any tag
30 // that might still hang around!
31 let intrinsic_name = &*this.tcx.item_name(instance.def_id()).as_str();
33 // First handle intrinsics without return place.
34 let (dest, ret) = match ret {
35 None => match intrinsic_name {
36 "miri_start_panic" => return this.handle_miri_start_panic(args, unwind),
37 "unreachable" => throw_ub!(Unreachable),
38 _ => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
43 // Then handle terminating intrinsics.
44 match intrinsic_name {
45 // Raw memory accesses
48 | "copy_nonoverlapping"
50 let &[src, dest, count] = check_arg_count(args)?;
51 let elem_ty = substs.type_at(0);
52 let elem_layout = this.layout_of(elem_ty)?;
53 let count = this.read_scalar(count)?.to_machine_usize(this)?;
54 let elem_align = elem_layout.align.abi;
56 let size = elem_layout.size.checked_mul(count, this)
57 .ok_or_else(|| err_ub_format!("overflow computing total size of `{}`", intrinsic_name))?;
58 let src = this.read_scalar(src)?.not_undef()?;
59 let src = this.memory.check_ptr_access(src, size, elem_align)?;
60 let dest = this.read_scalar(dest)?.not_undef()?;
61 let dest = this.memory.check_ptr_access(dest, size, elem_align)?;
63 if let (Some(src), Some(dest)) = (src, dest) {
68 intrinsic_name.ends_with("_nonoverlapping"),
74 let &[place, dest] = check_arg_count(args)?;
75 let place = this.deref_operand(place)?;
76 this.copy_op(dest, place.into())?;
80 let &[place] = check_arg_count(args)?;
81 let place = this.deref_operand(place)?;
82 this.copy_op(place.into(), dest)?;
85 let &[place, dest] = check_arg_count(args)?;
86 let place = this.deref_operand(place)?;
87 this.copy_op(dest, place.into())?;
91 let &[ptr, val_byte, count] = check_arg_count(args)?;
92 let ty = substs.type_at(0);
93 let ty_layout = this.layout_of(ty)?;
94 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
95 let ptr = this.read_scalar(ptr)?.not_undef()?;
96 let count = this.read_scalar(count)?.to_machine_usize(this)?;
97 let byte_count = ty_layout.size.checked_mul(count, this)
98 .ok_or_else(|| err_ub_format!("overflow computing total size of `write_bytes`"))?;
100 .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
103 // Floating-point operations
119 let &[f] = check_arg_count(args)?;
120 // FIXME: Using host floats.
121 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
122 let f = match intrinsic_name {
124 "fabsf32" => f.abs(),
126 "sqrtf32" => f.sqrt(),
128 "exp2f32" => f.exp2(),
130 "log10f32" => f.log10(),
131 "log2f32" => f.log2(),
132 "floorf32" => f.floor(),
133 "ceilf32" => f.ceil(),
134 "truncf32" => f.trunc(),
135 "roundf32" => f.round(),
138 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
156 let &[f] = check_arg_count(args)?;
157 // FIXME: Using host floats.
158 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
159 let f = match intrinsic_name {
161 "fabsf64" => f.abs(),
163 "sqrtf64" => f.sqrt(),
165 "exp2f64" => f.exp2(),
167 "log10f64" => f.log10(),
168 "log2f64" => f.log2(),
169 "floorf64" => f.floor(),
170 "ceilf64" => f.ceil(),
171 "truncf64" => f.trunc(),
172 "roundf64" => f.round(),
175 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
185 let &[a, b] = check_arg_count(args)?;
186 let a = this.read_immediate(a)?;
187 let b = this.read_immediate(b)?;
188 let op = match intrinsic_name {
189 "fadd_fast" => mir::BinOp::Add,
190 "fsub_fast" => mir::BinOp::Sub,
191 "fmul_fast" => mir::BinOp::Mul,
192 "fdiv_fast" => mir::BinOp::Div,
193 "frem_fast" => mir::BinOp::Rem,
196 this.binop_ignore_overflow(op, a, b, dest)?;
204 let &[a, b] = check_arg_count(args)?;
205 let a = this.read_scalar(a)?.to_f32()?;
206 let b = this.read_scalar(b)?.to_f32()?;
207 let res = match intrinsic_name {
208 "minnumf32" => a.min(b),
209 "maxnumf32" => a.max(b),
210 "copysignf32" => a.copy_sign(b),
213 this.write_scalar(Scalar::from_f32(res), dest)?;
221 let &[a, b] = check_arg_count(args)?;
222 let a = this.read_scalar(a)?.to_f64()?;
223 let b = this.read_scalar(b)?.to_f64()?;
224 let res = match intrinsic_name {
225 "minnumf64" => a.min(b),
226 "maxnumf64" => a.max(b),
227 "copysignf64" => a.copy_sign(b),
230 this.write_scalar(Scalar::from_f64(res), dest)?;
234 let &[f, f2] = check_arg_count(args)?;
235 // FIXME: Using host floats.
236 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
237 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
238 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
242 let &[f, f2] = check_arg_count(args)?;
243 // FIXME: Using host floats.
244 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
245 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
246 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
250 let &[a, b, c] = check_arg_count(args)?;
251 let a = this.read_scalar(a)?.to_f32()?;
252 let b = this.read_scalar(b)?.to_f32()?;
253 let c = this.read_scalar(c)?.to_f32()?;
254 let res = a.mul_add(b, c).value;
255 this.write_scalar(Scalar::from_f32(res), dest)?;
259 let &[a, b, c] = check_arg_count(args)?;
260 let a = this.read_scalar(a)?.to_f64()?;
261 let b = this.read_scalar(b)?.to_f64()?;
262 let c = this.read_scalar(c)?.to_f64()?;
263 let res = a.mul_add(b, c).value;
264 this.write_scalar(Scalar::from_f64(res), dest)?;
268 let &[f, i] = check_arg_count(args)?;
269 // FIXME: Using host floats.
270 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
271 let i = this.read_scalar(i)?.to_i32()?;
272 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
276 let &[f, i] = check_arg_count(args)?;
277 // FIXME: Using host floats.
278 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
279 let i = this.read_scalar(i)?.to_i32()?;
280 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
283 "float_to_int_unchecked" => {
284 let &[val] = check_arg_count(args)?;
285 let val = this.read_immediate(val)?;
287 let res = match val.layout.ty.kind {
288 ty::Float(FloatTy::F32) => {
289 this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?
291 ty::Float(FloatTy::F64) => {
292 this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?
294 _ => bug!("`float_to_int_unchecked` called with non-float input type {:?}", val.layout.ty),
297 this.write_scalar(res, dest)?;
303 | "atomic_load_relaxed"
306 let &[place] = check_arg_count(args)?;
307 let place = this.deref_operand(place)?;
308 let val = this.read_scalar(place.into())?; // make sure it fits into a scalar; otherwise it cannot be atomic
310 // Check alignment requirements. Atomics must always be aligned to their size,
311 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
313 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
314 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
316 this.write_scalar(val, dest)?;
321 | "atomic_store_relaxed"
324 let &[place, val] = check_arg_count(args)?;
325 let place = this.deref_operand(place)?;
326 let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
328 // Check alignment requirements. Atomics must always be aligned to their size,
329 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
331 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
332 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
334 this.write_scalar(val, place.into())?;
340 | "atomic_fence_acqrel"
342 | "atomic_singlethreadfence_acq"
343 | "atomic_singlethreadfence_rel"
344 | "atomic_singlethreadfence_acqrel"
345 | "atomic_singlethreadfence"
347 let &[] = check_arg_count(args)?;
348 // FIXME: this will become relevant once we try to detect data races.
351 _ if intrinsic_name.starts_with("atomic_xchg") => {
352 let &[place, new] = check_arg_count(args)?;
353 let place = this.deref_operand(place)?;
354 let new = this.read_scalar(new)?;
355 let old = this.read_scalar(place.into())?;
357 // Check alignment requirements. Atomics must always be aligned to their size,
358 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
360 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
361 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
363 this.write_scalar(old, dest)?; // old value is returned
364 this.write_scalar(new, place.into())?;
367 _ if intrinsic_name.starts_with("atomic_cxchg") => {
368 let &[place, expect_old, new] = check_arg_count(args)?;
369 let place = this.deref_operand(place)?;
370 let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
371 let new = this.read_scalar(new)?;
372 let old = this.read_immediate(place.into())?; // read as immediate for the sake of `binary_op()`
374 // Check alignment requirements. Atomics must always be aligned to their size,
375 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
377 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
378 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
380 // `binary_op` will bail if either of them is not a scalar.
381 let eq = this.overflowing_binary_op(mir::BinOp::Eq, old, expect_old)?.0;
382 let res = Immediate::ScalarPair(old.to_scalar_or_undef(), eq.into());
384 this.write_immediate(res, dest)?;
385 // Update ptr depending on comparison.
387 this.write_scalar(new, place.into())?;
396 | "atomic_or_relaxed"
400 | "atomic_xor_acqrel"
401 | "atomic_xor_relaxed"
405 | "atomic_and_acqrel"
406 | "atomic_and_relaxed"
410 | "atomic_nand_acqrel"
411 | "atomic_nand_relaxed"
415 | "atomic_xadd_acqrel"
416 | "atomic_xadd_relaxed"
420 | "atomic_xsub_acqrel"
421 | "atomic_xsub_relaxed"
423 let &[place, rhs] = check_arg_count(args)?;
424 let place = this.deref_operand(place)?;
425 if !place.layout.ty.is_integral() {
426 bug!("Atomic arithmetic operations only work on integer types");
428 let rhs = this.read_immediate(rhs)?;
429 let old = this.read_immediate(place.into())?;
431 // Check alignment requirements. Atomics must always be aligned to their size,
432 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
434 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
435 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
437 this.write_immediate(*old, dest)?; // old value is returned
438 let (op, neg) = match intrinsic_name.split('_').nth(1).unwrap() {
439 "or" => (mir::BinOp::BitOr, false),
440 "xor" => (mir::BinOp::BitXor, false),
441 "and" => (mir::BinOp::BitAnd, false),
442 "xadd" => (mir::BinOp::Add, false),
443 "xsub" => (mir::BinOp::Sub, false),
444 "nand" => (mir::BinOp::BitAnd, true),
447 // Atomics wrap around on overflow.
448 let val = this.binary_op(op, old, rhs)?;
449 let val = if neg { this.unary_op(mir::UnOp::Not, val)? } else { val };
450 this.write_immediate(*val, place.into())?;
453 // Query type information
455 "assert_zero_valid" |
456 "assert_uninit_valid" => {
457 let &[] = check_arg_count(args)?;
458 let ty = substs.type_at(0);
459 let layout = this.layout_of(ty)?;
460 // Abort here because the caller might not be panic safe.
461 if layout.abi.is_uninhabited() {
462 throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to instantiate uninhabited type `{}`", ty))))
464 if intrinsic_name == "assert_zero_valid" && !layout.might_permit_raw_init(this, /*zero:*/ true).unwrap() {
465 throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to zero-initialize type `{}`, which is invalid", ty))))
467 if intrinsic_name == "assert_uninit_valid" && !layout.might_permit_raw_init(this, /*zero:*/ false).unwrap() {
468 throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to leave type `{}` uninitialized, which is invalid", ty))))
472 "min_align_of_val" => {
473 let &[mplace] = check_arg_count(args)?;
474 let mplace = this.deref_operand(mplace)?;
475 let (_, align) = this
476 .size_and_align_of_mplace(mplace)?
477 .expect("size_of_val called on extern type");
478 this.write_scalar(Scalar::from_machine_usize(align.bytes(), this), dest)?;
482 let &[mplace] = check_arg_count(args)?;
483 let mplace = this.deref_operand(mplace)?;
485 .size_and_align_of_mplace(mplace)?
486 .expect("size_of_val called on extern type");
487 this.write_scalar(Scalar::from_machine_usize(size.bytes(), this), dest)?;
492 let &[cond] = check_arg_count(args)?;
493 let cond = this.read_scalar(cond)?.not_undef()?.to_bool()?;
495 throw_ub_format!("`assume` intrinsic called with `false`");
500 let &[num, denom] = check_arg_count(args)?;
501 this.exact_div(this.read_immediate(num)?, this.read_immediate(denom)?, dest)?;
505 // We get an argument... and forget about it.
506 let &[_] = check_arg_count(args)?;
513 // These just return their argument
514 let &[b] = check_arg_count(args)?;
515 let b = this.read_immediate(b)?;
516 this.write_immediate(*b, dest)?;
519 "try" => return this.handle_try(args, dest, ret),
521 name => throw_unsup_format!("unimplemented intrinsic: {}", name),
524 this.dump_place(*dest);
525 this.go_to_block(ret);
529 fn float_to_int_unchecked<F>(
532 dest_ty: ty::Ty<'tcx>,
533 ) -> InterpResult<'tcx, Scalar<Tag>>
535 F: Float + Into<Scalar<Tag>>
537 let this = self.eval_context_ref();
539 // Step 1: cut off the fractional part of `f`. The result of this is
540 // guaranteed to be precisely representable in IEEE floats.
541 let f = f.round_to_integral(Round::TowardZero).value;
543 // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
544 Ok(match dest_ty.kind {
547 let size = Integer::from_attr(this, attr::IntType::UnsignedInt(t)).size();
548 let res = f.to_u128(size.bits_usize());
549 if res.status.is_empty() {
550 // No status flags means there was no further rounding or other loss of precision.
551 Scalar::from_uint(res.value, size)
553 // `f` was not representable in this integer type.
555 "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
562 let size = Integer::from_attr(this, attr::IntType::SignedInt(t)).size();
563 let res = f.to_i128(size.bits_usize());
564 if res.status.is_empty() {
565 // No status flags means there was no further rounding or other loss of precision.
566 Scalar::from_int(res.value, size)
568 // `f` was not representable in this integer type.
570 "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
576 _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),