]> git.lizzy.rs Git - rust.git/blob - src/shims/intrinsics.rs
rustup (and account for stabilization)
[rust.git] / src / shims / intrinsics.rs
1 use std::iter;
2
3 use rustc_attr as attr;
4 use rustc_ast::ast::FloatTy;
5 use rustc_middle::{mir, ty};
6 use rustc_middle::ty::layout::IntegerExt;
7 use rustc_apfloat::{Float, Round};
8 use rustc_target::abi::{Align, Integer, LayoutOf};
9
10 use crate::*;
11 use helpers::check_arg_count;
12
13 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {}
14 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> {
15     fn call_intrinsic(
16         &mut self,
17         instance: ty::Instance<'tcx>,
18         args: &[OpTy<'tcx, Tag>],
19         ret: Option<(PlaceTy<'tcx, Tag>, mir::BasicBlock)>,
20         unwind: Option<mir::BasicBlock>,
21     ) -> InterpResult<'tcx> {
22         let this = self.eval_context_mut();
23         if this.emulate_intrinsic(instance, args, ret)? {
24             return Ok(());
25         }
26         let substs = instance.substs;
27
28         // All these intrinsics take raw pointers, so if we access memory directly
29         // (as opposed to through a place), we have to remember to erase any tag
30         // that might still hang around!
31         let intrinsic_name = &*this.tcx.item_name(instance.def_id()).as_str();
32
33         // First handle intrinsics without return place.
34         let (dest, ret) = match ret {
35             None => match intrinsic_name {
36                 "miri_start_panic" => return this.handle_miri_start_panic(args, unwind),
37                 "unreachable" => throw_ub!(Unreachable),
38                 _ => throw_unsup_format!("unimplemented (diverging) intrinsic: {}", intrinsic_name),
39             },
40             Some(p) => p,
41         };
42
43         // Then handle terminating intrinsics.
44         match intrinsic_name {
45             // Raw memory accesses
46             #[rustfmt::skip]
47             | "copy"
48             | "copy_nonoverlapping"
49             => {
50                 let &[src, dest, count] = check_arg_count(args)?;
51                 let elem_ty = substs.type_at(0);
52                 let elem_layout = this.layout_of(elem_ty)?;
53                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
54                 let elem_align = elem_layout.align.abi;
55
56                 let size = elem_layout.size.checked_mul(count, this)
57                     .ok_or_else(|| err_ub_format!("overflow computing total size of `{}`", intrinsic_name))?;
58                 let src = this.read_scalar(src)?.not_undef()?;
59                 let src = this.memory.check_ptr_access(src, size, elem_align)?;
60                 let dest = this.read_scalar(dest)?.not_undef()?;
61                 let dest = this.memory.check_ptr_access(dest, size, elem_align)?;
62
63                 if let (Some(src), Some(dest)) = (src, dest) {
64                     this.memory.copy(
65                         src,
66                         dest,
67                         size,
68                         intrinsic_name.ends_with("_nonoverlapping"),
69                     )?;
70                 }
71             }
72
73             "move_val_init" => {
74                 let &[place, dest] = check_arg_count(args)?;
75                 let place = this.deref_operand(place)?;
76                 this.copy_op(dest, place.into())?;
77             }
78
79             "volatile_load" => {
80                 let &[place] = check_arg_count(args)?;
81                 let place = this.deref_operand(place)?;
82                 this.copy_op(place.into(), dest)?;
83             }
84             "volatile_store" => {
85                 let &[place, dest] = check_arg_count(args)?;
86                 let place = this.deref_operand(place)?;
87                 this.copy_op(dest, place.into())?;
88             }
89
90             "write_bytes" => {
91                 let &[ptr, val_byte, count] = check_arg_count(args)?;
92                 let ty = substs.type_at(0);
93                 let ty_layout = this.layout_of(ty)?;
94                 let val_byte = this.read_scalar(val_byte)?.to_u8()?;
95                 let ptr = this.read_scalar(ptr)?.not_undef()?;
96                 let count = this.read_scalar(count)?.to_machine_usize(this)?;
97                 let byte_count = ty_layout.size.checked_mul(count, this)
98                     .ok_or_else(|| err_ub_format!("overflow computing total size of `write_bytes`"))?;
99                 this.memory
100                     .write_bytes(ptr, iter::repeat(val_byte).take(byte_count.bytes() as usize))?;
101             }
102
103             // Floating-point operations
104             #[rustfmt::skip]
105             | "sinf32"
106             | "fabsf32"
107             | "cosf32"
108             | "sqrtf32"
109             | "expf32"
110             | "exp2f32"
111             | "logf32"
112             | "log10f32"
113             | "log2f32"
114             | "floorf32"
115             | "ceilf32"
116             | "truncf32"
117             | "roundf32"
118             => {
119                 let &[f] = check_arg_count(args)?;
120                 // FIXME: Using host floats.
121                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
122                 let f = match intrinsic_name {
123                     "sinf32" => f.sin(),
124                     "fabsf32" => f.abs(),
125                     "cosf32" => f.cos(),
126                     "sqrtf32" => f.sqrt(),
127                     "expf32" => f.exp(),
128                     "exp2f32" => f.exp2(),
129                     "logf32" => f.ln(),
130                     "log10f32" => f.log10(),
131                     "log2f32" => f.log2(),
132                     "floorf32" => f.floor(),
133                     "ceilf32" => f.ceil(),
134                     "truncf32" => f.trunc(),
135                     "roundf32" => f.round(),
136                     _ => bug!(),
137                 };
138                 this.write_scalar(Scalar::from_u32(f.to_bits()), dest)?;
139             }
140
141             #[rustfmt::skip]
142             | "sinf64"
143             | "fabsf64"
144             | "cosf64"
145             | "sqrtf64"
146             | "expf64"
147             | "exp2f64"
148             | "logf64"
149             | "log10f64"
150             | "log2f64"
151             | "floorf64"
152             | "ceilf64"
153             | "truncf64"
154             | "roundf64"
155             => {
156                 let &[f] = check_arg_count(args)?;
157                 // FIXME: Using host floats.
158                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
159                 let f = match intrinsic_name {
160                     "sinf64" => f.sin(),
161                     "fabsf64" => f.abs(),
162                     "cosf64" => f.cos(),
163                     "sqrtf64" => f.sqrt(),
164                     "expf64" => f.exp(),
165                     "exp2f64" => f.exp2(),
166                     "logf64" => f.ln(),
167                     "log10f64" => f.log10(),
168                     "log2f64" => f.log2(),
169                     "floorf64" => f.floor(),
170                     "ceilf64" => f.ceil(),
171                     "truncf64" => f.trunc(),
172                     "roundf64" => f.round(),
173                     _ => bug!(),
174                 };
175                 this.write_scalar(Scalar::from_u64(f.to_bits()), dest)?;
176             }
177
178             #[rustfmt::skip]
179             | "fadd_fast"
180             | "fsub_fast"
181             | "fmul_fast"
182             | "fdiv_fast"
183             | "frem_fast"
184             => {
185                 let &[a, b] = check_arg_count(args)?;
186                 let a = this.read_immediate(a)?;
187                 let b = this.read_immediate(b)?;
188                 let op = match intrinsic_name {
189                     "fadd_fast" => mir::BinOp::Add,
190                     "fsub_fast" => mir::BinOp::Sub,
191                     "fmul_fast" => mir::BinOp::Mul,
192                     "fdiv_fast" => mir::BinOp::Div,
193                     "frem_fast" => mir::BinOp::Rem,
194                     _ => bug!(),
195                 };
196                 this.binop_ignore_overflow(op, a, b, dest)?;
197             }
198
199             #[rustfmt::skip]
200             | "minnumf32"
201             | "maxnumf32"
202             | "copysignf32"
203             => {
204                 let &[a, b] = check_arg_count(args)?;
205                 let a = this.read_scalar(a)?.to_f32()?;
206                 let b = this.read_scalar(b)?.to_f32()?;
207                 let res = match intrinsic_name {
208                     "minnumf32" => a.min(b),
209                     "maxnumf32" => a.max(b),
210                     "copysignf32" => a.copy_sign(b),
211                     _ => bug!(),
212                 };
213                 this.write_scalar(Scalar::from_f32(res), dest)?;
214             }
215
216             #[rustfmt::skip]
217             | "minnumf64"
218             | "maxnumf64"
219             | "copysignf64"
220             => {
221                 let &[a, b] = check_arg_count(args)?;
222                 let a = this.read_scalar(a)?.to_f64()?;
223                 let b = this.read_scalar(b)?.to_f64()?;
224                 let res = match intrinsic_name {
225                     "minnumf64" => a.min(b),
226                     "maxnumf64" => a.max(b),
227                     "copysignf64" => a.copy_sign(b),
228                     _ => bug!(),
229                 };
230                 this.write_scalar(Scalar::from_f64(res), dest)?;
231             }
232
233             "powf32" => {
234                 let &[f, f2] = check_arg_count(args)?;
235                 // FIXME: Using host floats.
236                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
237                 let f2 = f32::from_bits(this.read_scalar(f2)?.to_u32()?);
238                 this.write_scalar(Scalar::from_u32(f.powf(f2).to_bits()), dest)?;
239             }
240
241             "powf64" => {
242                 let &[f, f2] = check_arg_count(args)?;
243                 // FIXME: Using host floats.
244                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
245                 let f2 = f64::from_bits(this.read_scalar(f2)?.to_u64()?);
246                 this.write_scalar(Scalar::from_u64(f.powf(f2).to_bits()), dest)?;
247             }
248
249             "fmaf32" => {
250                 let &[a, b, c] = check_arg_count(args)?;
251                 let a = this.read_scalar(a)?.to_f32()?;
252                 let b = this.read_scalar(b)?.to_f32()?;
253                 let c = this.read_scalar(c)?.to_f32()?;
254                 let res = a.mul_add(b, c).value;
255                 this.write_scalar(Scalar::from_f32(res), dest)?;
256             }
257
258             "fmaf64" => {
259                 let &[a, b, c] = check_arg_count(args)?;
260                 let a = this.read_scalar(a)?.to_f64()?;
261                 let b = this.read_scalar(b)?.to_f64()?;
262                 let c = this.read_scalar(c)?.to_f64()?;
263                 let res = a.mul_add(b, c).value;
264                 this.write_scalar(Scalar::from_f64(res), dest)?;
265             }
266
267             "powif32" => {
268                 let &[f, i] = check_arg_count(args)?;
269                 // FIXME: Using host floats.
270                 let f = f32::from_bits(this.read_scalar(f)?.to_u32()?);
271                 let i = this.read_scalar(i)?.to_i32()?;
272                 this.write_scalar(Scalar::from_u32(f.powi(i).to_bits()), dest)?;
273             }
274
275             "powif64" => {
276                 let &[f, i] = check_arg_count(args)?;
277                 // FIXME: Using host floats.
278                 let f = f64::from_bits(this.read_scalar(f)?.to_u64()?);
279                 let i = this.read_scalar(i)?.to_i32()?;
280                 this.write_scalar(Scalar::from_u64(f.powi(i).to_bits()), dest)?;
281             }
282
283             "float_to_int_unchecked" => {
284                 let &[val] = check_arg_count(args)?;
285                 let val = this.read_immediate(val)?;
286
287                 let res = match val.layout.ty.kind {
288                     ty::Float(FloatTy::F32) => {
289                         this.float_to_int_unchecked(val.to_scalar()?.to_f32()?, dest.layout.ty)?
290                     }
291                     ty::Float(FloatTy::F64) => {
292                         this.float_to_int_unchecked(val.to_scalar()?.to_f64()?, dest.layout.ty)?
293                     }
294                     _ => bug!("`float_to_int_unchecked` called with non-float input type {:?}", val.layout.ty),
295                 };
296
297                 this.write_scalar(res, dest)?;
298             }
299
300             // Atomic operations
301             #[rustfmt::skip]
302             | "atomic_load"
303             | "atomic_load_relaxed"
304             | "atomic_load_acq"
305             => {
306                 let &[place] = check_arg_count(args)?;
307                 let place = this.deref_operand(place)?;
308                 let val = this.read_scalar(place.into())?; // make sure it fits into a scalar; otherwise it cannot be atomic
309
310                 // Check alignment requirements. Atomics must always be aligned to their size,
311                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
312                 // be 8-aligned).
313                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
314                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
315
316                 this.write_scalar(val, dest)?;
317             }
318
319             #[rustfmt::skip]
320             | "atomic_store"
321             | "atomic_store_relaxed"
322             | "atomic_store_rel"
323             => {
324                 let &[place, val] = check_arg_count(args)?;
325                 let place = this.deref_operand(place)?;
326                 let val = this.read_scalar(val)?; // make sure it fits into a scalar; otherwise it cannot be atomic
327
328                 // Check alignment requirements. Atomics must always be aligned to their size,
329                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
330                 // be 8-aligned).
331                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
332                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
333
334                 this.write_scalar(val, place.into())?;
335             }
336
337             #[rustfmt::skip]
338             | "atomic_fence_acq"
339             | "atomic_fence_rel"
340             | "atomic_fence_acqrel"
341             | "atomic_fence"
342             | "atomic_singlethreadfence_acq"
343             | "atomic_singlethreadfence_rel"
344             | "atomic_singlethreadfence_acqrel"
345             | "atomic_singlethreadfence"
346             => {
347                 let &[] = check_arg_count(args)?;
348                 // FIXME: this will become relevant once we try to detect data races.
349             }
350
351             _ if intrinsic_name.starts_with("atomic_xchg") => {
352                 let &[place, new] = check_arg_count(args)?;
353                 let place = this.deref_operand(place)?;
354                 let new = this.read_scalar(new)?;
355                 let old = this.read_scalar(place.into())?;
356
357                 // Check alignment requirements. Atomics must always be aligned to their size,
358                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
359                 // be 8-aligned).
360                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
361                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
362
363                 this.write_scalar(old, dest)?; // old value is returned
364                 this.write_scalar(new, place.into())?;
365             }
366
367             _ if intrinsic_name.starts_with("atomic_cxchg") => {
368                 let &[place, expect_old, new] = check_arg_count(args)?;
369                 let place = this.deref_operand(place)?;
370                 let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
371                 let new = this.read_scalar(new)?;
372                 let old = this.read_immediate(place.into())?; // read as immediate for the sake of `binary_op()`
373
374                 // Check alignment requirements. Atomics must always be aligned to their size,
375                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
376                 // be 8-aligned).
377                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
378                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
379
380                 // `binary_op` will bail if either of them is not a scalar.
381                 let eq = this.overflowing_binary_op(mir::BinOp::Eq, old, expect_old)?.0;
382                 let res = Immediate::ScalarPair(old.to_scalar_or_undef(), eq.into());
383                 // Return old value.
384                 this.write_immediate(res, dest)?;
385                 // Update ptr depending on comparison.
386                 if eq.to_bool()? {
387                     this.write_scalar(new, place.into())?;
388                 }
389             }
390
391             #[rustfmt::skip]
392             | "atomic_or"
393             | "atomic_or_acq"
394             | "atomic_or_rel"
395             | "atomic_or_acqrel"
396             | "atomic_or_relaxed"
397             | "atomic_xor"
398             | "atomic_xor_acq"
399             | "atomic_xor_rel"
400             | "atomic_xor_acqrel"
401             | "atomic_xor_relaxed"
402             | "atomic_and"
403             | "atomic_and_acq"
404             | "atomic_and_rel"
405             | "atomic_and_acqrel"
406             | "atomic_and_relaxed"
407             | "atomic_nand"
408             | "atomic_nand_acq"
409             | "atomic_nand_rel"
410             | "atomic_nand_acqrel"
411             | "atomic_nand_relaxed"
412             | "atomic_xadd"
413             | "atomic_xadd_acq"
414             | "atomic_xadd_rel"
415             | "atomic_xadd_acqrel"
416             | "atomic_xadd_relaxed"
417             | "atomic_xsub"
418             | "atomic_xsub_acq"
419             | "atomic_xsub_rel"
420             | "atomic_xsub_acqrel"
421             | "atomic_xsub_relaxed"
422             => {
423                 let &[place, rhs] = check_arg_count(args)?;
424                 let place = this.deref_operand(place)?;
425                 if !place.layout.ty.is_integral() {
426                     bug!("Atomic arithmetic operations only work on integer types");
427                 }
428                 let rhs = this.read_immediate(rhs)?;
429                 let old = this.read_immediate(place.into())?;
430
431                 // Check alignment requirements. Atomics must always be aligned to their size,
432                 // even if the type they wrap would be less aligned (e.g. AtomicU64 on 32bit must
433                 // be 8-aligned).
434                 let align = Align::from_bytes(place.layout.size.bytes()).unwrap();
435                 this.memory.check_ptr_access(place.ptr, place.layout.size, align)?;
436
437                 this.write_immediate(*old, dest)?; // old value is returned
438                 let (op, neg) = match intrinsic_name.split('_').nth(1).unwrap() {
439                     "or" => (mir::BinOp::BitOr, false),
440                     "xor" => (mir::BinOp::BitXor, false),
441                     "and" => (mir::BinOp::BitAnd, false),
442                     "xadd" => (mir::BinOp::Add, false),
443                     "xsub" => (mir::BinOp::Sub, false),
444                     "nand" => (mir::BinOp::BitAnd, true),
445                     _ => bug!(),
446                 };
447                 // Atomics wrap around on overflow.
448                 let val = this.binary_op(op, old, rhs)?;
449                 let val = if neg { this.unary_op(mir::UnOp::Not, val)? } else { val };
450                 this.write_immediate(*val, place.into())?;
451             }
452
453             // Query type information
454             "assert_inhabited" |
455             "assert_zero_valid" |
456             "assert_uninit_valid" => {
457                 let &[] = check_arg_count(args)?;
458                 let ty = substs.type_at(0);
459                 let layout = this.layout_of(ty)?;
460                 // Abort here because the caller might not be panic safe.
461                 if layout.abi.is_uninhabited() {
462                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to instantiate uninhabited type `{}`", ty))))
463                 }
464                 if intrinsic_name == "assert_zero_valid" && !layout.might_permit_raw_init(this, /*zero:*/ true).unwrap() {
465                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to zero-initialize type `{}`, which is invalid", ty))))
466                 }
467                 if intrinsic_name == "assert_uninit_valid" && !layout.might_permit_raw_init(this, /*zero:*/ false).unwrap() {
468                     throw_machine_stop!(TerminationInfo::Abort(Some(format!("attempted to leave type `{}` uninitialized, which is invalid", ty))))
469                 }
470             }
471
472             "min_align_of_val" => {
473                 let &[mplace] = check_arg_count(args)?;
474                 let mplace = this.deref_operand(mplace)?;
475                 let (_, align) = this
476                     .size_and_align_of_mplace(mplace)?
477                     .expect("size_of_val called on extern type");
478                 this.write_scalar(Scalar::from_machine_usize(align.bytes(), this), dest)?;
479             }
480
481             "size_of_val" => {
482                 let &[mplace] = check_arg_count(args)?;
483                 let mplace = this.deref_operand(mplace)?;
484                 let (size, _) = this
485                     .size_and_align_of_mplace(mplace)?
486                     .expect("size_of_val called on extern type");
487                 this.write_scalar(Scalar::from_machine_usize(size.bytes(), this), dest)?;
488             }
489
490             // Other
491             "assume" => {
492                 let &[cond] = check_arg_count(args)?;
493                 let cond = this.read_scalar(cond)?.not_undef()?.to_bool()?;
494                 if !cond {
495                     throw_ub_format!("`assume` intrinsic called with `false`");
496                 }
497             }
498
499             "exact_div" => {
500                 let &[num, denom] = check_arg_count(args)?;
501                 this.exact_div(this.read_immediate(num)?, this.read_immediate(denom)?, dest)?;
502             }
503
504             "forget" => {
505                 // We get an argument... and forget about it.
506                 let &[_] = check_arg_count(args)?;
507             }
508
509             #[rustfmt::skip]
510             | "likely"
511             | "unlikely"
512             => {
513                 // These just return their argument
514                 let &[b] = check_arg_count(args)?;
515                 let b = this.read_immediate(b)?;
516                 this.write_immediate(*b, dest)?;
517             }
518
519             "try" => return this.handle_try(args, dest, ret),
520
521             name => throw_unsup_format!("unimplemented intrinsic: {}", name),
522         }
523
524         this.dump_place(*dest);
525         this.go_to_block(ret);
526         Ok(())
527     }
528
529     fn float_to_int_unchecked<F>(
530         &self,
531         f: F,
532         dest_ty: ty::Ty<'tcx>,
533     ) -> InterpResult<'tcx, Scalar<Tag>>
534     where
535         F: Float + Into<Scalar<Tag>>
536     {
537         let this = self.eval_context_ref();
538
539         // Step 1: cut off the fractional part of `f`. The result of this is
540         // guaranteed to be precisely representable in IEEE floats.
541         let f = f.round_to_integral(Round::TowardZero).value;
542
543         // Step 2: Cast the truncated float to the target integer type and see if we lose any information in this step.
544         Ok(match dest_ty.kind {
545             // Unsigned
546             ty::Uint(t) => {
547                 let size = Integer::from_attr(this, attr::IntType::UnsignedInt(t)).size();
548                 let res = f.to_u128(size.bits_usize());
549                 if res.status.is_empty() {
550                     // No status flags means there was no further rounding or other loss of precision.
551                     Scalar::from_uint(res.value, size)
552                 } else {
553                     // `f` was not representable in this integer type.
554                     throw_ub_format!(
555                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
556                         f, dest_ty,
557                     );
558                 }
559             }
560             // Signed
561             ty::Int(t) => {
562                 let size = Integer::from_attr(this, attr::IntType::SignedInt(t)).size();
563                 let res = f.to_i128(size.bits_usize());
564                 if res.status.is_empty() {
565                     // No status flags means there was no further rounding or other loss of precision.
566                     Scalar::from_int(res.value, size)
567                 } else {
568                     // `f` was not representable in this integer type.
569                     throw_ub_format!(
570                         "`float_to_int_unchecked` intrinsic called on {} which cannot be represented in target type `{:?}`",
571                         f, dest_ty,
572                     );
573                 }
574             }
575             // Nothing else
576             _ => bug!("`float_to_int_unchecked` called with non-int output type {:?}", dest_ty),
577         })
578     }
579 }